Tabnine Logo
com.netflix.iceberg.avro
Code IndexAdd Tabnine to your IDE (free)

How to use com.netflix.iceberg.avro

Best Java code snippets using com.netflix.iceberg.avro (Showing top 20 results out of 315)

origin: Netflix/iceberg

@Override
public Iterator<D> iterator() {
 FileReader<D> reader = initMetadata(newFileReader());
 if (start != null) {
  reader = new AvroRangeIterator<>(reader, start, end);
 }
 if (reuseContainers) {
  return new AvroReuseIterator<>(reader);
 }
 addCloseable(reader);
 return reader;
}
origin: Netflix/iceberg

@Override
@SuppressWarnings("unchecked")
public void setSchema(Schema schema) {
 this.writer = (ValueWriter<T>) visit(schema, new WriteBuilder());
}
origin: Netflix/iceberg

 @Override
 public void write(S row, Encoder encoder) throws IOException {
  for (int i = 0; i < writers.length; i += 1) {
   writers[i].write(get(row, i), encoder);
  }
 }
}
origin: Netflix/iceberg

private CloseableIterable<InternalRow> newAvroIterable(InputFile location,
                         FileScanTask task,
                         Schema readSchema) {
 return Avro.read(location)
   .reuseContainers()
   .project(readSchema)
   .split(task.start(), task.length())
   .createReaderFunc(SparkAvroReader::new)
   .build();
}
origin: Netflix/iceberg

 public <D> FileAppender<D> build() throws IOException {
  Preconditions.checkNotNull(schema, "Schema is required");
  Preconditions.checkNotNull(name, "Table name is required and cannot be null");
  // add the Iceberg schema to keyValueMetadata
  meta("iceberg.schema", SchemaParser.toJson(schema));
  return new AvroFileAppender<>(
    AvroSchemaUtil.convert(schema, name), file, createWriterFunc, codec(), metadata);
 }
}
origin: Netflix/iceberg

@Override
public ValueReader<?> array(Schema array, ValueReader<?> elementReader) {
 if (array.getLogicalType() instanceof LogicalMap) {
  ValueReaders.StructReader<?> keyValueReader = (ValueReaders.StructReader) elementReader;
  ValueReader<?> keyReader = keyValueReader.reader(0);
  ValueReader<?> valueReader = keyValueReader.reader(1);
  return ValueReaders.arrayMap(keyReader, valueReader);
 }
 return ValueReaders.array(elementReader);
}
origin: Netflix/iceberg

@Override
public ValueWriter<?> array(Schema array, ValueWriter<?> elementWriter) {
 if (array.getLogicalType() instanceof LogicalMap) {
  ValueWriters.StructWriter<?> keyValueWriter = (ValueWriters.StructWriter<?>) elementWriter;
  return ValueWriters.arrayMap(keyValueWriter.writer(0), keyValueWriter.writer(1));
 }
 return ValueWriters.array(elementWriter);
}
origin: Netflix/iceberg

@SuppressWarnings("unchecked")
private void initReader() {
 this.reader = (ValueReader<T>) AvroSchemaVisitor.visit(readSchema, new ReadBuilder(loader));
}
origin: Netflix/iceberg

public static Type convert(Schema schema) {
 return AvroSchemaVisitor.visit(schema, new SchemaToType(schema));
}
origin: Netflix/iceberg

@Override
public ValueReader<?> map(Schema map, ValueReader<?> valueReader) {
 return ValueReaders.map(ValueReaders.strings(), valueReader);
}
origin: Netflix/iceberg

@Override
public void setSchema(Schema fileSchema) {
 this.fileSchema = fileSchema;
 Set<Integer> projectedIds = getProjectedIds(expectedSchema);
 Schema prunedSchema = AvroSchemaUtil.pruneColumns(fileSchema, projectedIds);
 this.readSchema = AvroSchemaUtil.buildAvroProjection(prunedSchema, expectedSchema, renames);
 this.wrapped = newDatumReader();
}
origin: Netflix/iceberg

@Override
public T read(T reuse, Decoder decoder) throws IOException {
 ResolvingDecoder resolver = resolve(decoder);
 T value = reader.read(resolver, reuse);
 resolver.drain();
 return value;
}
origin: Netflix/iceberg

@Override
public ValueReader<?> union(Schema union, List<ValueReader<?>> options) {
 return ValueReaders.union(options);
}
origin: Netflix/iceberg

 public <D> AvroIterable<D> build() {
  Preconditions.checkNotNull(schema, "Schema is required");
  return new AvroIterable<>(file,
    new ProjectionDatumReader<>(createReaderFunc, schema, renames),
    start, length, reuseContainers);
 }
}
origin: Netflix/iceberg

private int getId(Schema.Field field) {
 if (field.getObjectProp(AvroSchemaUtil.FIELD_ID_PROP) != null) {
  return AvroSchemaUtil.getFieldId(field);
 } else {
  return allocateId();
 }
}
origin: Netflix/iceberg

public static Schema buildAvroProjection(Schema schema, com.netflix.iceberg.Schema expected,
                     Map<String, String> renames) {
 return AvroCustomOrderSchemaVisitor.visit(schema, new BuildAvroProjection(expected, renames));
}
origin: Netflix/iceberg

 @Override
 public F get() {
  return visitor.field(field, new VisitFuture<>(field.schema(), visitor));
 }
}
origin: Netflix/iceberg

AvroFileAppender(Schema schema, OutputFile file,
         Function<Schema, DatumWriter<?>> createWriterFunc,
         CodecFactory codec, Map<String, String> metadata) throws IOException {
 this.writer = newAvroWriter(schema, file, createWriterFunc, codec, metadata);
}
origin: Netflix/iceberg

 @Override
 public T get() {
  return visit(schema, visitor);
 }
}
origin: Netflix/iceberg

public static <K, V> ValueWriter<Map<K, V>> map(ValueWriter<K> keyWriter,
                        ValueWriter<V> valueWriter) {
 return new MapWriter<>(keyWriter, valueWriter);
}
com.netflix.iceberg.avro

Most used classes

  • AvroSchemaUtil
  • Avro$ReadBuilder
  • Avro
  • Avro$WriteBuilder
  • AvroSchemaVisitor
  • ValueReaders,
  • ValueWriter,
  • ValueWriters,
  • ProjectionDatumReader,
  • TestReadProjection,
  • ValueReaders$StructReader,
  • ValueWriters$StructWriter,
  • Avro$CodecName,
  • AvroCustomOrderSchemaVisitor$VisitFieldFuture,
  • AvroCustomOrderSchemaVisitor$VisitFuture,
  • AvroCustomOrderSchemaVisitor,
  • AvroDataTest,
  • AvroFileAppender,
  • AvroIO$AvroInputStreamAdapter
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now