@Override public void close() throws IOException { if (orcRowsReader != null) { this.orcRowsReader.close(); } this.orcRowsReader = null; }
/** * Checks if there is at least one row left in the batch to return. * If no more row are available, it reads another batch of rows. * * @return Returns true if there is one more row to return, false otherwise. * @throws IOException throw if an exception happens while reading a batch. */ private boolean ensureBatch() throws IOException { if (nextRow >= rowsInBatch) { // No more rows available in the Rows array. nextRow = 0; // Try to read the next batch if rows from the ORC file. boolean moreRows = orcRowsReader.nextBatch(rowBatch); if (moreRows) { // Load the data into the Rows array. rowsInBatch = fillRows(rows, schema, rowBatch, selectedFields); } return moreRows; } // there is at least one Row left in the Rows array. return true; }
@Override public KeyValue next() throws IOException { boolean endOfBatch = false; StringWriter sw = new StringWriter(); if (rowIndex > batch.size - 1) { endOfBatch = !rows.nextBatch(batch); rowIndex = 0; } if (endOfBatch) { rows.close(); return null; } try { JsonFieldFiller.processRow(new JSONWriter(sw), batch, schema, rowIndex); } catch (JSONException e) { LOG.error("Unable to parse json {}", sw.toString()); return null; } rowIndex++; return new KeyValue(offset++, sw.toString().getBytes("UTF-8")); }
@Override public float getProgress() throws IOException { return recordReader.getProgress(); }
@Override public float getProgress() throws IOException { return recordReader.getProgress(); }
@Override public void close() throws IOException { rows.close(); } }
@SuppressWarnings("deprecation") public JsonORCFileReader(LogFilePath logFilePath, CompressionCodec codec) throws IOException { schema = schemaProvider.getSchema(logFilePath.getTopic(), logFilePath); Path path = new Path(logFilePath.getLogFilePath()); Reader reader = OrcFile.createReader(path, OrcFile.readerOptions(new Configuration(true))); offset = logFilePath.getOffset(); rows = reader.rows(); batch = reader.getSchema().createRowBatch(); rows.nextBatch(batch); }
@Override public float getProgress() throws IOException, InterruptedException { return in.getProgress(); }
@Override public void close() throws IOException { if (columnarBatch != null) { columnarBatch.close(); columnarBatch = null; } if (recordReader != null) { recordReader.close(); recordReader = null; } }
/** * If the current batch is empty, get a new one. * @return true if we have rows available. * @throws IOException */ boolean ensureBatch() throws IOException { if (rowInBatch >= batch.size) { rowInBatch = 0; return batchReader.nextBatch(batch); } return true; }
@Override public float getProgress() throws IOException { return batchReader.getProgress(); } }
@Override public void close() throws IOException { if (columnarBatch != null) { columnarBatch.close(); columnarBatch = null; } if (recordReader != null) { recordReader.close(); recordReader = null; } }
/** * If the current batch is empty, get a new one. * @return true if we have rows available. * @throws IOException */ boolean ensureBatch() throws IOException { if (rowInBatch >= batch.size) { rowInBatch = 0; return batchReader.nextBatch(batch); } return true; }
@Override public void close() throws IOException { if (recordReader != null) { recordReader.close(); recordReader = null; } }
recordReader.nextBatch(batch); int batchSize = batch.size; if (batchSize == 0) {
@Override public void close() throws IOException { in.close(); } }
recordReader.nextBatch(batch); int batchSize = batch.size; if (batchSize == 0) {
@Override public void close() throws IOException { rows.close(); }
/** * If the current batch is empty, get a new one. * * @return true if we have rows available. * @throws IOException */ boolean ensureBatch() throws IOException { if (rowInBatch >= batch.size) { rowInBatch = 0; return in.nextBatch(batch); } return true; }