public AvroColumnWriter(Schema s, ColumnFileMetaData meta, GenericData model) throws IOException { this.schema = s; AvroColumnator columnator = new AvroColumnator(s); meta.set(SCHEMA_KEY, s.toString()); // save schema in file this.writer = new ColumnFileWriter(meta, columnator.getColumns()); this.arrayWidths = columnator.getArrayWidths(); this.model = model; }
new ColumnFileWriter(createFileMeta(), new ColumnMetaData("test", ValueType.LONG)); Random random = TestUtil.createRandom();
new ColumnFileWriter(createFileMeta(), new ColumnMetaData("test", ValueType.STRING) .hasIndexValues(true));
@Test public void testEmptyFile() throws Exception { FILE.delete(); ColumnFileWriter out = new ColumnFileWriter(createFileMeta()); out.writeTo(FILE); ColumnFileReader in = new ColumnFileReader(FILE); Assert.assertEquals(0, in.getRowCount()); Assert.assertEquals(0, in.getColumnCount()); in.close(); }
@Test public void testEmptyColumn() throws Exception { FILE.delete(); ColumnFileWriter out = new ColumnFileWriter(createFileMeta(), new ColumnMetaData("test", ValueType.INT)); out.writeTo(FILE); ColumnFileReader in = new ColumnFileReader(FILE); Assert.assertEquals(0, in.getRowCount()); Assert.assertEquals(1, in.getColumnCount()); ColumnValues<Integer> values = in.getValues("test"); for (int i : values) throw new Exception("no value should be found"); in.close(); }
@Test public void testLongs() throws Exception { FILE.delete(); ColumnFileWriter out = new ColumnFileWriter(createFileMeta(), new ColumnMetaData("test", ValueType.LONG)); Random random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) out.writeRow(random.nextLong()); out.writeTo(FILE); random = TestUtil.createRandom(); ColumnFileReader in = new ColumnFileReader(FILE); Assert.assertEquals(COUNT, in.getRowCount()); Assert.assertEquals(1, in.getColumnCount()); Iterator<Long> i = in.getValues("test"); int count = 0; while (i.hasNext()) { Assert.assertEquals(random.nextLong(), (long)i.next()); count++; } Assert.assertEquals(COUNT, count); }
@Test public void testTwoColumn() throws Exception { FILE.delete(); ColumnFileWriter out = new ColumnFileWriter(createFileMeta(), new ColumnMetaData("a", ValueType.FIXED32), new ColumnMetaData("b", ValueType.STRING)); Random random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) out.writeRow(random.nextInt(), TestUtil.randomString(random)); out.writeTo(FILE); random = TestUtil.createRandom(); ColumnFileReader in = new ColumnFileReader(FILE); Assert.assertEquals(COUNT, in.getRowCount()); Assert.assertEquals(2, in.getColumnCount()); Iterator<String> i = in.getValues("a"); Iterator<String> j = in.getValues("b"); int count = 0; while (i.hasNext() && j.hasNext()) { Assert.assertEquals(random.nextInt(), i.next()); Assert.assertEquals(TestUtil.randomString(random), j.next()); count++; } Assert.assertEquals(COUNT, count); }
@Test public void testStrings() throws Exception { FILE.delete(); ColumnFileWriter out = new ColumnFileWriter(createFileMeta(), new ColumnMetaData("test", ValueType.STRING)); Random random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) out.writeRow(TestUtil.randomString(random)); out.writeTo(FILE); random = TestUtil.createRandom(); ColumnFileReader in = new ColumnFileReader(FILE); Assert.assertEquals(COUNT, in.getRowCount()); Assert.assertEquals(1, in.getColumnCount()); Iterator<String> i = in.getValues("test"); int count = 0; while (i.hasNext()) { Assert.assertEquals(TestUtil.randomString(random), i.next()); count++; } Assert.assertEquals(COUNT, count); }
@Test public void testInts() throws Exception { FILE.delete(); ColumnFileWriter out = new ColumnFileWriter(createFileMeta(), new ColumnMetaData("test", ValueType.INT)); Random random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) out.writeRow(TestUtil.randomLength(random)); out.writeTo(FILE); random = TestUtil.createRandom(); ColumnFileReader in = new ColumnFileReader(FILE); Assert.assertEquals(COUNT, in.getRowCount()); Assert.assertEquals(1, in.getColumnCount()); Iterator<Integer> i = in.getValues("test"); int count = 0; while (i.hasNext()) { Assert.assertEquals(TestUtil.randomLength(random), (int)i.next()); count++; } Assert.assertEquals(COUNT, count); }
public AvroColumnWriter(Schema s, ColumnFileMetaData meta, GenericData model) throws IOException { this.schema = s; AvroColumnator columnator = new AvroColumnator(s); meta.set(SCHEMA_KEY, s.toString()); // save schema in file this.writer = new ColumnFileWriter(meta, columnator.getColumns()); this.arrayWidths = columnator.getArrayWidths(); this.model = model; }
public void init() throws IOException { fs = path.getFileSystem(conf); if (!fs.exists(path.getParent())) { throw new FileNotFoundException(path.toString()); } fos = fs.create(path); ColumnMetaData [] trevniMetas = new ColumnMetaData[schema.size()]; int i = 0; for (Column column : schema.getColumns()) { trevniMetas[i++] = new ColumnMetaData(column.getSimpleName(), getType(column.getDataType().getType())); } writer = new ColumnFileWriter(createFileMeta(), trevniMetas); if (enabledStats) { this.stats = new TableStatistics(this.schema); } super.init(); }
new ColumnFileWriter(createFileMeta(), new ColumnMetaData("test", ValueType.LONG)); Random random = TestUtil.createRandom();
new ColumnFileWriter(createFileMeta(), new ColumnMetaData("test", ValueType.STRING) .hasIndexValues(true));
@Test public void testEmptyFile() throws Exception { FILE.delete(); ColumnFileWriter out = new ColumnFileWriter(createFileMeta()); out.writeTo(FILE); ColumnFileReader in = new ColumnFileReader(FILE); Assert.assertEquals(0, in.getRowCount()); Assert.assertEquals(0, in.getColumnCount()); in.close(); }
@Test public void testEmptyColumn() throws Exception { FILE.delete(); ColumnFileWriter out = new ColumnFileWriter(createFileMeta(), new ColumnMetaData("test", ValueType.INT)); out.writeTo(FILE); ColumnFileReader in = new ColumnFileReader(FILE); Assert.assertEquals(0, in.getRowCount()); Assert.assertEquals(1, in.getColumnCount()); ColumnValues<Integer> values = in.getValues("test"); for (int i : values) throw new Exception("no value should be found"); in.close(); }
@Test public void testLongs() throws Exception { FILE.delete(); ColumnFileWriter out = new ColumnFileWriter(createFileMeta(), new ColumnMetaData("test", ValueType.LONG)); Random random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) out.writeRow(random.nextLong()); out.writeTo(FILE); random = TestUtil.createRandom(); ColumnFileReader in = new ColumnFileReader(FILE); Assert.assertEquals(COUNT, in.getRowCount()); Assert.assertEquals(1, in.getColumnCount()); Iterator<Long> i = in.getValues("test"); int count = 0; while (i.hasNext()) { Assert.assertEquals(random.nextLong(), (long)i.next()); count++; } Assert.assertEquals(COUNT, count); }
@Test public void testTwoColumn() throws Exception { FILE.delete(); ColumnFileWriter out = new ColumnFileWriter(createFileMeta(), new ColumnMetaData("a", ValueType.FIXED32), new ColumnMetaData("b", ValueType.STRING)); Random random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) out.writeRow(random.nextInt(), TestUtil.randomString(random)); out.writeTo(FILE); random = TestUtil.createRandom(); ColumnFileReader in = new ColumnFileReader(FILE); Assert.assertEquals(COUNT, in.getRowCount()); Assert.assertEquals(2, in.getColumnCount()); Iterator<String> i = in.getValues("a"); Iterator<String> j = in.getValues("b"); int count = 0; while (i.hasNext() && j.hasNext()) { Assert.assertEquals(random.nextInt(), i.next()); Assert.assertEquals(TestUtil.randomString(random), j.next()); count++; } Assert.assertEquals(COUNT, count); }
@Test public void testInts() throws Exception { FILE.delete(); ColumnFileWriter out = new ColumnFileWriter(createFileMeta(), new ColumnMetaData("test", ValueType.INT)); Random random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) out.writeRow(TestUtil.randomLength(random)); out.writeTo(FILE); random = TestUtil.createRandom(); ColumnFileReader in = new ColumnFileReader(FILE); Assert.assertEquals(COUNT, in.getRowCount()); Assert.assertEquals(1, in.getColumnCount()); Iterator<Integer> i = in.getValues("test"); int count = 0; while (i.hasNext()) { Assert.assertEquals(TestUtil.randomLength(random), (int)i.next()); count++; } Assert.assertEquals(COUNT, count); }
@Test public void testStrings() throws Exception { FILE.delete(); ColumnFileWriter out = new ColumnFileWriter(createFileMeta(), new ColumnMetaData("test", ValueType.STRING)); Random random = TestUtil.createRandom(); for (int i = 0; i < COUNT; i++) out.writeRow(TestUtil.randomString(random)); out.writeTo(FILE); random = TestUtil.createRandom(); ColumnFileReader in = new ColumnFileReader(FILE); Assert.assertEquals(COUNT, in.getRowCount()); Assert.assertEquals(1, in.getColumnCount()); Iterator<String> i = in.getValues("test"); int count = 0; while (i.hasNext()) { Assert.assertEquals(TestUtil.randomString(random), i.next()); count++; } Assert.assertEquals(COUNT, count); }