private ColumnMetaData addColumn(String path, ValueType type, ColumnMetaData parent, boolean isArray) { ColumnMetaData column = new ColumnMetaData(path, type); if (parent != null) column.setParent(parent); column.isArray(isArray); columns.add(column); arrayWidths.add(1); // placeholder return column; }
public void writeValue(Object value) throws IOException { buffer.writeValue(value, meta.getType()); if (meta.hasIndexValues() && rowCount == 0) firstValues.add(buffer.toByteArray()); }
private void readColumnMetaData(InputBuffer in) throws IOException { for (int i = 0; i < columnCount; i++) { ColumnMetaData meta = ColumnMetaData.read(in, this); meta.setDefaults(this.metaData); ColumnDescriptor column = new ColumnDescriptor(file, meta); columns[i] = column; meta.setNumber(i); columnsByName.put(meta.getName(), column); } }
private String shortName(ColumnMetaData column) { String name = column.getName(); ColumnMetaData parent = column.getParent(); if (parent != null && name.startsWith(parent.getName())) name = name.substring(parent.getName().length()); if (!Character.isLetterOrDigit(name.charAt(0))) name = name.substring(1); return name; }
@Test public void testSimpleUnion() throws Exception { String s = "[\"int\",\"string\"]"; check(Schema.parse(s), new ColumnMetaData("int", ValueType.INT).isArray(true), new ColumnMetaData("string", ValueType.STRING).isArray(true)); }
@Test public void testNamedRecord() throws Exception { String s = "{\"type\":\"record\",\"name\":\"S\",\"fields\":[" +"{\"name\":\"R1\",\"type\":"+SIMPLE_RECORD+"}," +"{\"name\":\"R2\",\"type\":\"R\"}" +"]}"; check(Schema.parse(s), new ColumnMetaData("R1#x", ValueType.INT), new ColumnMetaData("R1#y", ValueType.STRING), new ColumnMetaData("R2#x", ValueType.INT), new ColumnMetaData("R2#y", ValueType.STRING)); }
static ColumnMetaData read(InputBuffer in, ColumnFileReader file) throws IOException { ColumnMetaData result = new ColumnMetaData(); MetaData.read(in, result); result.name = result.getString(NAME_KEY); result.type = ValueType.forName(result.getString(TYPE_KEY)); result.values = result.getBoolean(VALUES_KEY); result.isArray = result.getBoolean(ARRAY_KEY); String parentName = result.getString(PARENT_KEY); if (parentName != null) result.setParent(file.getColumnMetaData(parentName)); return result; }
public ArrayColumnOutputBuffer(ColumnFileWriter writer, ColumnMetaData meta) throws IOException { super(writer, meta); assert getMeta().isArray() || getMeta().getParent() != null; assert !getMeta().hasIndexValues(); }
/** Set this column's parent. A parent must be a preceding array column. */ public ColumnMetaData setParent(ColumnMetaData parent) { if (!parent.isArray()) throw new TrevniRuntimeException("Parent is not an array: "+parent); if (values) throw new TrevniRuntimeException("Array column cannot have index: "+this); this.parent = parent; parent.children.add(this); return setReserved(PARENT_KEY, parent.getName()); }
private void valueToJson(ColumnMetaData column) throws IOException { generator.writeFieldName(shortNames[column.getNumber()]); ColumnValues in = values[column.getNumber()]; if (!column.isArray()) { primitiveToJson(column, in.nextValue()); } else { generator.writeStartArray(); int length = in.nextLength(); for (int i = 0; i < length; i++) { Object value = in.nextValue(); List<ColumnMetaData> children = column.getChildren(); if (children.size() == 0) { primitiveToJson(column, value); } else { generator.writeStartObject(); if (value != null) { generator.writeFieldName("value$"); primitiveToJson(column, value); } for (ColumnMetaData child : children) valueToJson(child); generator.writeEndObject(); } } generator.writeEndArray(); } }
new ColumnMetaData("test", ValueType.STRING) .hasIndexValues(true));
/** Construct given metadata for each column in the file. */ public ColumnFileWriter(ColumnFileMetaData fileMeta, ColumnMetaData... columnMeta) throws IOException { checkColumns(columnMeta); this.metaData = fileMeta; this.columnCount = columnMeta.length; this.columns = new ColumnOutputBuffer[columnCount]; for (int i = 0; i < columnCount; i++) { ColumnMetaData c = columnMeta[i]; c.setDefaults(metaData); columns[i] = c.isArray() ? new ArrayColumnOutputBuffer(this, c) : new ColumnOutputBuffer(this, c); size += OutputBuffer.BLOCK_SIZE; // over-estimate } }
/** Seek to the named value. */ public void seek(T v) throws IOException { if (!column.metaData.hasIndexValues()) throw new TrevniRuntimeException ("Column does not have value index: " +column.metaData.getName()); if (previous == null // not in current block? || previous.compareTo(v) > 0 || (block != column.blockCount()-1 && column.firstValues[block+1].compareTo(v) <= 0)) startBlock(column.findBlock(v)); // seek to block start while (hasNext()) { // scan block long savedPosition = values.tell(); T savedPrevious = previous; if (next().compareTo(v) >= 0) { values.seek(savedPosition); previous = savedPrevious; row--; return; } } }
private void primitiveToJson(ColumnMetaData column, Object value) throws IOException { switch (column.getType()) { case NULL: generator.writeNull(); break; case BOOLEAN: generator.writeBoolean((Boolean)value); break; case INT: generator.writeNumber((Integer)value); break; case LONG: generator.writeNumber((Long)value); break; case FIXED32: generator.writeNumber((Integer)value); break; case FIXED64: generator.writeNumber((Long)value); break; case FLOAT: generator.writeNumber((Float)value); break; case DOUBLE: generator.writeNumber((Double)value); break; case STRING: generator.writeString((String)value); break; case BYTES: generator.writeBinary((byte[])value); break; default: throw new RuntimeException("Unknown value type: "+column.getType()); } }
void initialize() throws IOException { // compute a mapping from column name to number for file Map<String,Integer> fileColumnNumbers = new HashMap<>(); int i = 0; for (ColumnMetaData c : new AvroColumnator(fileSchema).getColumns()) fileColumnNumbers.put(c.getName(), i++); // create iterator for each column in readSchema AvroColumnator readColumnator = new AvroColumnator(readSchema); this.arrayWidths = readColumnator.getArrayWidths(); ColumnMetaData[] readColumns = readColumnator.getColumns(); this.values = new ColumnValues[readColumns.length]; int j = 0; for (ColumnMetaData c : readColumns) { Integer n = fileColumnNumbers.get(c.getName()); if (n != null) values[j++] = reader.getValues(n); } findDefaults(readSchema, fileSchema); }
public ColumnOutputBuffer(ColumnFileWriter writer, ColumnMetaData meta) throws IOException { this.writer = writer; this.meta = meta; this.codec = Codec.get(meta); this.checksum = Checksum.get(meta); this.buffer = new OutputBuffer(); this.blockDescriptors = new ArrayList<>(); this.blockData = new ArrayList<>(); if (meta.hasIndexValues()) this.firstValues = new ArrayList<>(); }
/** Return root columns' metadata. Roots are columns that have no parent. */ public List<ColumnMetaData> getRoots() { List<ColumnMetaData> result = new ArrayList<>(); for (int i = 0; i < columnCount; i++) if (columns[i].metaData.getParent() == null) result.add(columns[i].metaData); return result; }
@Test public void testSimpleOptional() throws Exception { String s = "[\"null\",\"string\"]"; check(Schema.parse(s), new ColumnMetaData("string", ValueType.STRING).isArray(true)); }