OptionWriter(int definitionLevel, ParquetValueWriter<T> writer) { this.definitionLevel = definitionLevel; this.writer = writer; this.children = writer.columns(); }
@Override public void setColumnStore(ColumnWriteStore columnStore) { writer.setColumnStore(columnStore); } }
@Override public void write(int repetitionLevel, T value) { if (value != null) { writer.write(repetitionLevel, value); } else { for (TripleWriter<?> column : children) { column.writeNull(repetitionLevel, definitionLevel - 1); } } }
@Override public void write(int repetitionLevel, S value) { for (int i = 0; i < writers.length; i += 1) { Object fieldValue = get(value, i); writers[i].write(repetitionLevel, fieldValue); } }
@Override public void write(int parentRepetition, M value) { Iterator<Map.Entry<K, V>> pairs = pairs(value); if (!pairs.hasNext()) { // write the empty map to each column for (TripleWriter<?> column : children) { column.writeNull(parentRepetition, definitionLevel - 1); } } else { boolean first = true; while (pairs.hasNext()) { Map.Entry<K, V> pair = pairs.next(); int rl = repetitionLevel; if (first) { rl = parentRepetition; first = false; } keyWriter.write(rl, pair.getKey()); valueWriter.write(rl, pair.getValue()); } } }
@Override public void setColumnStore(ColumnWriteStore columnStore) { writer.setColumnStore(columnStore); }
protected RepeatedWriter(int definitionLevel, int repetitionLevel, ParquetValueWriter<E> writer) { this.definitionLevel = definitionLevel; this.repetitionLevel = repetitionLevel; this.writer = writer; this.children = writer.columns(); }
@Override public void add(T value) { recordCount += 1; model.write(0, value); writeStore.endRecord(); checkSize(); }
@Override public void setColumnStore(ColumnWriteStore columnStore) { keyWriter.setColumnStore(columnStore); valueWriter.setColumnStore(columnStore); }
protected RepeatedKeyValueWriter(int definitionLevel, int repetitionLevel, ParquetValueWriter<K> keyWriter, ParquetValueWriter<V> valueWriter) { this.definitionLevel = definitionLevel; this.repetitionLevel = repetitionLevel; this.keyWriter = keyWriter; this.valueWriter= valueWriter; this.children = ImmutableList.<TripleWriter<?>>builder() .addAll(keyWriter.columns()) .addAll(valueWriter.columns()) .build(); }
@Override public void write(int parentRepetition, L value) { Iterator<E> elements = elements(value); if (!elements.hasNext()) { // write the empty list to each column // TODO: make sure this definition level is correct for (TripleWriter<?> column : children) { column.writeNull(parentRepetition, definitionLevel - 1); } } else { boolean first = true; while (elements.hasNext()) { E element = elements.next(); int rl = repetitionLevel; if (first) { rl = parentRepetition; first = false; } writer.write(rl, element); } } }
@Override public void setColumnStore(ColumnWriteStore columnStore) { for (ParquetValueWriter<?> writer : writers) { writer.setColumnStore(columnStore); } }
@SuppressWarnings("unchecked") protected StructWriter(List<ParquetValueWriter<?>> writers) { this.writers = (ParquetValueWriter<Object>[]) Array.newInstance( ParquetValueWriter.class, writers.size()); ImmutableList.Builder<TripleWriter<?>> columnsBuilder = ImmutableList.builder(); for (int i = 0; i < writers.size(); i += 1) { ParquetValueWriter<?> writer = writers.get(i); this.writers[i] = (ParquetValueWriter<Object>) writer; columnsBuilder.addAll(writer.columns()); } this.children = columnsBuilder.build(); }
private void startRowGroup() { try { this.nextRowGroupSize = min(writer.getNextRowGroupSize(), targetRowGroupSize); } catch (IOException e) { throw new RuntimeIOException(e); } this.nextCheckRecordCount = min(max(recordCount / 2, 100), 10000); this.recordCount = 0; PageWriteStore pageStore = pageStoreCtor.newInstance( compressor, parquetSchema, props.getAllocator()); this.flushPageStoreToWriter = flushToWriter.bind(pageStore); this.writeStore = props.newColumnWriteStore(parquetSchema, pageStore); model.setColumnStore(writeStore); }