Tabnine Logo
DataFileReader.next
Code IndexAdd Tabnine to your IDE (free)

How to use
next
method
in
org.apache.avro.file.DataFileReader

Best Java code snippets using org.apache.avro.file.DataFileReader.next (Showing top 20 results out of 495)

origin: apache/flink

@Override
public E nextRecord(E reuseValue) throws IOException {
  if (reachedEnd()) {
    return null;
  }
  // if we start a new block, then register the event, and
  // restart the counter.
  if (dataFileReader.previousSync() != lastSync) {
    lastSync = dataFileReader.previousSync();
    recordsReadSinceLastSync = 0;
  }
  recordsReadSinceLastSync++;
  if (reuseAvroValue) {
    return dataFileReader.next(reuseValue);
  } else {
    if (GenericRecord.class == avroValueType) {
      return dataFileReader.next();
    } else {
      return dataFileReader.next(InstantiationUtil.instantiate(avroValueType, Object.class));
    }
  }
}
origin: apache/avro

/** Reads and returns the first datum in a data file. */
static Object datumFromFile(Schema schema, String file) throws IOException {
 DataFileReader<Object> in =
   new DataFileReader<>(new File(file),
              new GenericDatumReader<>(schema));
 try {
  return in.next();
 } finally {
  in.close();
 }
}
origin: apache/flume

/**
 * Read the last record in the file.
 */
private void initReader() throws IOException {
 long syncPos = trackerFile.length() - 256L;
 if (syncPos < 0) syncPos = 0L;
 reader.sync(syncPos);
 while (reader.hasNext()) {
  reader.next(metaCache);
 }
}
origin: pinterest/secor

@Override
public KeyValue next() throws IOException {
  GenericRecord record = reader.next();
  if (record != null) {
    return new KeyValue(offset++, serializeAvroRecord(writer, record));
  }
  return null;
}
origin: apache/avro

/** {@inheritDoc} */
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
 assert null != mAvroFileReader;
 if (mAvroFileReader.hasNext() && !mAvroFileReader.pastSync(mEndPosition)) {
  mCurrentRecord = mAvroFileReader.next(mCurrentRecord);
  return true;
 }
 return false;
}
origin: apache/incubator-gobblin

@Override
public String readRecord(@Deprecated String reuse) throws IOException {
 if (this.dataFileReader == null) {
  return null;
 }
 if (this.dataFileReader.hasNext()) {
  return this.dataFileReader.next().toString();
 }
 return null;
}
origin: apache/flume

@Override
public Event readEvent() throws IOException {
 if (fileReader.hasNext()) {
  record = fileReader.next(record);
  out.reset();
  datumWriter.write(record, encoder);
  encoder.flush();
  // annotate header with 64-bit schema CRC hash in hex
  Event event = EventBuilder.withBody(out.toByteArray());
  if (schemaType == AvroSchemaType.HASH) {
   event.getHeaders().put(AVRO_SCHEMA_HEADER_HASH, schemaHashString);
  } else {
   event.getHeaders().put(AVRO_SCHEMA_HEADER_LITERAL, schema.toString());
  }
  return event;
 }
 return null;
}
origin: apache/incubator-gobblin

private GenericRecord getRecordFromFile(String path)
  throws IOException {
 DatumReader<GenericRecord> reader = new GenericDatumReader<>();
 DataFileReader<GenericRecord> dataFileReader = new DataFileReader<GenericRecord>(new File(path), reader);
 if (dataFileReader.hasNext()) {
  return dataFileReader.next();
 }
 return null;
}
origin: apache/incubator-gobblin

private GenericRecord getRecordFromFile(String path) throws IOException {
 DatumReader<GenericRecord> reader = new GenericDatumReader<>();
 DataFileReader<GenericRecord> dataFileReader = new DataFileReader<GenericRecord>(new File(path), reader);
 while (dataFileReader.hasNext()) {
  return dataFileReader.next();
 }
 return null;
}
origin: apache/flink

  @Override
  public void reopen(FileInputSplit split, Tuple2<Long, Long> state) throws IOException {
    Preconditions.checkNotNull(split, "reopen() cannot be called on a null split.");
    Preconditions.checkNotNull(state, "reopen() cannot be called with a null initial state.");

    try {
      this.open(split);
    } finally {
      if (state.f0 != -1) {
        lastSync = state.f0;
        recordsReadSinceLastSync = state.f1;
      }
    }

    if (lastSync != -1) {
      // open and read until the record we were before
      // the checkpoint and discard the values
      dataFileReader.seek(lastSync);
      for (int i = 0; i < recordsReadSinceLastSync; i++) {
        dataFileReader.next(null);
      }
    }
  }
}
origin: apache/incubator-gobblin

private void updateRecordFromTestResource(String resourceName, String avroFileName)
  throws IOException {
 if (avroFileName == null) {
  avroFileName = resourceName + ".avro";
 }
 recordSchema = new Schema.Parser().parse(
   getClass().getClassLoader().getResourceAsStream(resourceName + ".avsc")
 );
 DatumReader<GenericRecord> reader = new GenericDatumReader<>(recordSchema);
 DataFileReader<GenericRecord> dataFileReader = new DataFileReader<GenericRecord>(
   new File(getClass().getClassLoader().getResource(avroFileName).getPath()), reader);
 Assert.assertTrue(dataFileReader.hasNext());
 record = dataFileReader.next(record);
 accessor = new AvroGenericRecordAccessor(record);
}
origin: h2oai/h2o-3

static <T> T runOnPreview(byte[] bits, AvroPreviewProcessor<T> processor) throws IOException {
 DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>();
 SeekableByteArrayInput sbai = new SeekableByteArrayInput(bits);
 DataFileReader<GenericRecord> dataFileReader = null;
 try {
  dataFileReader = new DataFileReader<>(sbai, datumReader);
  int headerLen = (int) dataFileReader.previousSync();
  byte[] header = Arrays.copyOf(bits, headerLen);
  if (dataFileReader.hasNext()) {
   GenericRecord gr = dataFileReader.next();
   return processor.process(header, gr, dataFileReader.getBlockCount(), dataFileReader.getBlockSize());
  } else {
   throw new RuntimeException("Empty Avro file - cannot run preview! ");
  }
 } finally {
  try { if (dataFileReader!=null) dataFileReader.close(); } catch (IOException safeToIgnore) {}
 }
}
origin: apache/nifi

private Map<String, GenericRecord> getGenericRecordMap(byte[] data, Schema schema, String key) throws IOException {
  // create a reader for the merged contet
  DatumReader<GenericRecord> datumReader = new GenericDatumReader<>(schema);
  SeekableByteArrayInput input = new SeekableByteArrayInput(data);
  DataFileReader<GenericRecord> dataFileReader = new DataFileReader<>(input, datumReader);
  // read all the records into a map to verify all the records are there
  Map<String,GenericRecord> records = new HashMap<>();
  while (dataFileReader.hasNext()) {
    GenericRecord user = dataFileReader.next();
    records.put(user.get(key).toString(), user);
  }
  return records;
}
origin: apache/storm

  private void fileIsGoodAvro(Path path) throws IOException {
    DatumReader<GenericRecord> datumReader = new GenericDatumReader<>();
    try (FSDataInputStream in = fs.open(path, 0); FileOutputStream out = new FileOutputStream("target/FOO.avro")) {
      byte[] buffer = new byte[100];
      int bytesRead;
      while ((bytesRead = in.read(buffer)) > 0) {
        out.write(buffer, 0, bytesRead);
      }
    }

    java.io.File file = new File("target/FOO.avro");

    try (DataFileReader<GenericRecord> dataFileReader = new DataFileReader<>(file, datumReader)) {
      GenericRecord user = null;
      while (dataFileReader.hasNext()) {
        user = dataFileReader.next(user);
      }
    }

    file.delete();
  }
}
origin: apache/avro

private void checkFileContains(File repairedFile, String... lines) throws IOException {
 DataFileReader r = new DataFileReader<>(repairedFile,
   new GenericDatumReader<>(SCHEMA));
 for (String line : lines) {
  assertEquals(line, r.next().toString());
 }
 assertFalse(r.hasNext());
}
origin: apache/flink

@Test
public void testGenericRecord() throws IOException {
  final Path outputPath = new Path(File.createTempFile("avro-output-file", "generic.avro").getAbsolutePath());
  final AvroOutputFormat<GenericRecord> outputFormat = new AvroOutputFormat<>(outputPath, GenericRecord.class);
  Schema schema = new Schema.Parser().parse("{\"type\":\"record\", \"name\":\"user\", \"fields\": [{\"name\":\"user_name\", \"type\":\"string\"}, {\"name\":\"favorite_number\", \"type\":\"int\"}, {\"name\":\"favorite_color\", \"type\":\"string\"}]}");
  outputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
  outputFormat.setSchema(schema);
  output(outputFormat, schema);
  GenericDatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
  DataFileReader<GenericRecord> dataFileReader = new DataFileReader<>(new File(outputPath.getPath()), reader);
  while (dataFileReader.hasNext()) {
    GenericRecord record = dataFileReader.next();
    assertEquals(record.get("user_name").toString(), "testUser");
    assertEquals(record.get("favorite_number"), 1);
    assertEquals(record.get("favorite_color").toString(), "blue");
  }
  //cleanup
  FileSystem fs = FileSystem.getLocalFileSystem();
  fs.delete(outputPath, false);
}
origin: apache/avro

public void testGenericRead() throws IOException {
 DataFileReader<Object> reader =
  new DataFileReader<>(makeFile(), new GenericDatumReader<>());
 try {
  Object datum = null;
  if (VALIDATE) {
   for (Object expected : new RandomData(SCHEMA, COUNT, SEED)) {
    datum = reader.next(datum);
    assertEquals(expected, datum);
   }
  } else {
   for (int i = 0; i < COUNT; i++) {
    datum = reader.next(datum);
   }
  }
 } finally {
  reader.close();
 }
}
origin: apache/flink

  User user = dataFileReader1.next();
  result1.add(user.getName() + "|" + user.getFavoriteNumber() + "|" + user.getFavoriteColor());
DataFileReader<ReflectiveUser> dataFileReader2 = new DataFileReader<>(avroOutput, userDatumReader2);
while (dataFileReader2.hasNext()) {
  ReflectiveUser user = dataFileReader2.next();
  result2.add(user.getName() + "|" + user.getFavoriteNumber() + "|" + user.getFavoriteColor());
origin: h2oai/h2o-3

if (sbai.chunkCnt == 0) { // Find data in first chunk
 while (dataFileReader.hasNext() && dataFileReader.previousSync() == sync) {
  gr = dataFileReader.next(gr);
origin: apache/avro

/**
 * Test that non-string map-keys are readable through ReflectDatumReader
 * This method should form the original map and should not return any
 * array of {key, value} as done by {@link #testGenericDatumRead()}
 */
private <T> List<T> testReflectDatumRead
 (String testType, byte[] bytes, T ... entityObjs) throws IOException {
 ReflectDatumReader<T> datumReader = new ReflectDatumReader<>();
 SeekableByteArrayInput avroInputStream = new SeekableByteArrayInput(bytes);
 DataFileReader<T> fileReader = new DataFileReader<>(avroInputStream, datumReader);
 Schema schema = fileReader.getSchema();
 T record = null;
 List<T> records = new ArrayList<>();
 while (fileReader.hasNext()) {
  records.add (fileReader.next(record));
 }
 return records;
}
org.apache.avro.fileDataFileReadernext

Popular methods of DataFileReader

  • <init>
    Construct using a DataFileStream.Header. Does not call #sync(long) or #seek(long).
  • hasNext
  • close
  • openReader
    Construct a reader for a file at the current position of the input, without reading the header.
  • getSchema
  • sync
    Move to the next synchronization point after a position. To process a range of file entires, call th
  • previousSync
    Return the last synchronization point before our current position.
  • seek
    Move to a specific, known synchronization point, one returned from DataFileWriter#sync() while writi
  • pastSync
    Return true if past the next synchronization point after a position.
  • getMeta
  • getMetaString
  • getHeader
  • getMetaString,
  • getHeader,
  • getBlockCount,
  • initialize,
  • iterator,
  • blockFinished,
  • getBlockSize,
  • getMetaKeys,
  • nextBlock

Popular in Java

  • Making http requests using okhttp
  • getResourceAsStream (ClassLoader)
  • getSupportFragmentManager (FragmentActivity)
  • setRequestProperty (URLConnection)
  • HttpServer (com.sun.net.httpserver)
    This class implements a simple HTTP server. A HttpServer is bound to an IP address and port number a
  • Font (java.awt)
    The Font class represents fonts, which are used to render text in a visible way. A font provides the
  • EOFException (java.io)
    Thrown when a program encounters the end of a file or stream during an input operation.
  • FileNotFoundException (java.io)
    Thrown when a file specified by a program cannot be found.
  • Hashtable (java.util)
    A plug-in replacement for JDK1.5 java.util.Hashtable. This version is based on org.cliffc.high_scale
  • ImageIO (javax.imageio)
  • Top Sublime Text plugins
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now