private ColumnMetaData addColumn(String path, ValueType type, ColumnMetaData parent, boolean isArray) { ColumnMetaData column = new ColumnMetaData(path, type); if (parent != null) column.setParent(parent); column.isArray(isArray); columns.add(column); arrayWidths.add(1); // placeholder return column; }
/** Expert: Returns the next length in an array column. */ public int nextLength() throws IOException { if (!column.metaData.isArray()) throw new TrevniRuntimeException ("Column is not array: " +column.metaData.getName()); assert arrayLength == 0; return arrayLength = values.readLength(); }
/** Set this column's parent. A parent must be a preceding array column. */ public ColumnMetaData setParent(ColumnMetaData parent) { if (!parent.isArray()) throw new TrevniRuntimeException("Parent is not an array: "+parent); if (values) throw new TrevniRuntimeException("Array column cannot have index: "+this); this.parent = parent; parent.children.add(this); return setReserved(PARENT_KEY, parent.getName()); }
/** Construct given metadata for each column in the file. */ public ColumnFileWriter(ColumnFileMetaData fileMeta, ColumnMetaData... columnMeta) throws IOException { checkColumns(columnMeta); this.metaData = fileMeta; this.columnCount = columnMeta.length; this.columns = new ColumnOutputBuffer[columnCount]; for (int i = 0; i < columnCount; i++) { ColumnMetaData c = columnMeta[i]; c.setDefaults(metaData); columns[i] = c.isArray() ? new ArrayColumnOutputBuffer(this, c) : new ColumnOutputBuffer(this, c); size += OutputBuffer.BLOCK_SIZE; // over-estimate } }
private void valueToJson(ColumnMetaData column) throws IOException { generator.writeFieldName(shortNames[column.getNumber()]); ColumnValues in = values[column.getNumber()]; if (!column.isArray()) { primitiveToJson(column, in.nextValue()); } else { generator.writeStartArray(); int length = in.nextLength(); for (int i = 0; i < length; i++) { Object value = in.nextValue(); List<ColumnMetaData> children = column.getChildren(); if (children.size() == 0) { primitiveToJson(column, value); } else { generator.writeStartObject(); if (value != null) { generator.writeFieldName("value$"); primitiveToJson(column, value); } for (ColumnMetaData child : children) valueToJson(child); generator.writeEndObject(); } } generator.writeEndArray(); } }
public ArrayColumnOutputBuffer(ColumnFileWriter writer, ColumnMetaData meta) throws IOException { super(writer, meta); assert getMeta().isArray() || getMeta().getParent() != null; assert !getMeta().hasIndexValues(); }
@Override public T next() { if (column.metaData.isArray() || column.metaData.getParent() != null) throw new TrevniRuntimeException ("Column is array: " +column.metaData.getName()); try { startRow(); return nextValue(); } catch (IOException e) { throw new TrevniRuntimeException(e); } }
@Test public void testSimpleOptional() throws Exception { String s = "[\"null\",\"string\"]"; check(Schema.parse(s), new ColumnMetaData("string", ValueType.STRING).isArray(true)); }
@Test public void testSimpleArray() throws Exception { String s = "{\"type\":\"array\",\"items\":\"long\"}"; check(Schema.parse(s), new ColumnMetaData("[]", ValueType.LONG).isArray(true)); }
@Test public void testArrayInUnion() throws Exception { String s = "{\"type\":\"record\",\"name\":\"S\",\"fields\":[" +"{\"name\":\"a\",\"type\":[\"int\","+RECORD_ARRAY+"]}]}"; ColumnMetaData q = new ColumnMetaData("a/array",ValueType.NULL) .isArray(true); ColumnMetaData r = new ColumnMetaData("a/array[]", ValueType.NULL) .setParent(q) .isArray(true); check(Schema.parse(s), new ColumnMetaData("a/int", ValueType.INT).isArray(true), q, r, new ColumnMetaData("a/array[]#x", ValueType.INT).setParent(r), new ColumnMetaData("a/array[]#y", ValueType.STRING).setParent(r)); }
@Test public void testUnion() throws Exception { ColumnMetaData p = new ColumnMetaData("R", ValueType.NULL).isArray(true); check(Schema.parse(UNION), new ColumnMetaData("int", ValueType.INT).isArray(true), p, new ColumnMetaData("R#x", ValueType.INT).setParent(p), new ColumnMetaData("R#y", ValueType.STRING).setParent(p)); }
@Test public void testUnionInArray() throws Exception { String s = "{\"type\":\"record\",\"name\":\"S\",\"fields\":[" +"{\"name\":\"a\",\"type\":{\"type\":\"array\",\"items\":"+UNION+"}}" +"]}"; ColumnMetaData p = new ColumnMetaData("a[]",ValueType.NULL).isArray(true); ColumnMetaData r = new ColumnMetaData("a[]/R", ValueType.NULL) .setParent(p) .isArray(true); check(Schema.parse(s), p, new ColumnMetaData("a[]/int", ValueType.INT) .setParent(p) .isArray(true), r, new ColumnMetaData("a[]/R#x", ValueType.INT).setParent(r), new ColumnMetaData("a[]/R#y", ValueType.STRING).setParent(r)); }
@Test public void testNestedUnion() throws Exception { String s = "{\"type\":\"record\",\"name\":\"S\",\"fields\":[" +"{\"name\":\"x\",\"type\":\"int\"}," +"{\"name\":\"u\",\"type\":"+UNION+"}," +"{\"name\":\"y\",\"type\":\"string\"}" +"]}"; ColumnMetaData p = new ColumnMetaData("u/R", ValueType.NULL).isArray(true); check(Schema.parse(s), new ColumnMetaData("x", ValueType.INT), new ColumnMetaData("u/int", ValueType.INT).isArray(true), p, new ColumnMetaData("u/R#x", ValueType.INT).setParent(p), new ColumnMetaData("u/R#y", ValueType.STRING).setParent(p), new ColumnMetaData("y", ValueType.STRING)); }
@Test public void testSimpleMap() throws Exception { String s = "{\"type\":\"map\",\"values\":\"long\"}"; ColumnMetaData p = new ColumnMetaData(">", ValueType.NULL).isArray(true); check(Schema.parse(s), p, new ColumnMetaData(">key", ValueType.STRING).setParent(p), new ColumnMetaData(">value", ValueType.LONG).setParent(p)); }
@Test public void testArray() throws Exception { ColumnMetaData p = new ColumnMetaData("[]", ValueType.NULL).isArray(true); check(Schema.parse(RECORD_ARRAY), p, new ColumnMetaData("[]#x", ValueType.INT).setParent(p), new ColumnMetaData("[]#y", ValueType.STRING).setParent(p)); }
@Test public void testMap() throws Exception { String s = "{\"type\":\"map\",\"values\":"+SIMPLE_RECORD+"}"; ColumnMetaData p = new ColumnMetaData(">", ValueType.NULL).isArray(true); check(Schema.parse(s), p, new ColumnMetaData(">key", ValueType.STRING).setParent(p), new ColumnMetaData(">value#x", ValueType.INT).setParent(p), new ColumnMetaData(">value#y", ValueType.STRING).setParent(p)); }
@Test public void testNestedArray() throws Exception { String s = "{\"type\":\"record\",\"name\":\"S\",\"fields\":[" +"{\"name\":\"x\",\"type\":\"int\"}," +"{\"name\":\"A\",\"type\":"+RECORD_ARRAY+"}," +"{\"name\":\"y\",\"type\":\"string\"}" +"]}"; ColumnMetaData p = new ColumnMetaData("A[]", ValueType.NULL).isArray(true); check(Schema.parse(s), new ColumnMetaData("x", ValueType.INT), p, new ColumnMetaData("A[]#x", ValueType.INT).setParent(p), new ColumnMetaData("A[]#y", ValueType.STRING).setParent(p), new ColumnMetaData("y", ValueType.STRING)); }
public ArrayColumnOutputBuffer(ColumnFileWriter writer, ColumnMetaData meta) throws IOException { super(writer, meta); assert getMeta().isArray() || getMeta().getParent() != null; assert !getMeta().hasIndexValues(); }