/** * An example on how to use this utility class. */ public static void main(String[] args) { Schema schema = new Schema.SchemaBuilder().setSchemaName("testSchema") .addSingleValueDimension("dimension", FieldSpec.DataType.DOUBLE).addMetric("metric", FieldSpec.DataType.INT) .addTime("time", TimeUnit.DAYS, FieldSpec.DataType.INT).build(); System.out.println(postSchema("localhost", 8100, schema)); Schema fetchedSchema = getSchema("localhost", 8100, "testSchema"); Preconditions.checkNotNull(fetchedSchema); System.out.println(fetchedSchema); System.out.println(equalsIgnoreVersion(schema, fetchedSchema)); System.out.println(deleteSchema("localhost", 8100, "testSchema")); } }
@BeforeClass public void setUp() { Schema schema = new Schema.SchemaBuilder().setSchemaName("testSchema") .addSingleValueDimension(DIMENSION_1, FieldSpec.DataType.INT) .addSingleValueDimension(DIMENSION_2, FieldSpec.DataType.STRING).addMetric(METRIC, FieldSpec.DataType.LONG) .build(); _mutableSegmentImpl = MutableSegmentImplTestUtils .createMutableSegmentImpl(schema, new HashSet<>(Arrays.asList(DIMENSION_1, METRIC)), Collections.singleton(DIMENSION_1), true); }
projectedColumns.add("column2"); Schema schema = new Schema.SchemaBuilder().addSingleValueDimension("column3", DataType.STRING) .addSingleValueDimension("column2", DataType.STRING).build(); final SegmentGeneratorConfig config = new SegmentGeneratorConfig(schema);
@Test public void testSchemaBuilder() { String defaultString = "default"; Schema schema = new Schema.SchemaBuilder().addSingleValueDimension("svDimension", FieldSpec.DataType.INT) .addSingleValueDimension("svDimensionWithDefault", FieldSpec.DataType.INT, 10) .addMultiValueDimension("mvDimension", FieldSpec.DataType.STRING) .addMultiValueDimension("mvDimensionWithDefault", FieldSpec.DataType.STRING, defaultString)
.addMetric("column3", FieldSpec.DataType.INT).addSingleValueDimension("column5", FieldSpec.DataType.STRING) .addSingleValueDimension("column6", FieldSpec.DataType.INT) .addSingleValueDimension("column7", FieldSpec.DataType.INT) .addSingleValueDimension("column9", FieldSpec.DataType.INT) .addSingleValueDimension("column11", FieldSpec.DataType.STRING) .addSingleValueDimension("column12", FieldSpec.DataType.STRING).addMetric("column17", FieldSpec.DataType.INT) .addMetric("column18", FieldSpec.DataType.INT).addTime("daysSinceEpoch", 1, TimeUnit.DAYS, DataType.INT) .build(); .addMetric("column3", FieldSpec.DataType.INT).addSingleValueDimension("column5", FieldSpec.DataType.STRING) .addSingleValueDimension("column6", FieldSpec.DataType.INT) .addSingleValueDimension("column7", FieldSpec.DataType.INT) .addSingleValueDimension("column9", FieldSpec.DataType.INT) .addSingleValueDimension("column11", FieldSpec.DataType.STRING) .addSingleValueDimension("column12", FieldSpec.DataType.STRING).addMetric("column17", FieldSpec.DataType.INT) .addMetric("column18", FieldSpec.DataType.INT).addTime("daysSinceEpoch", 1, TimeUnit.DAYS, DataType.INT) .build();
_aggregatedValueType = _valueAggregator.getAggregatedValueType(); Schema.SchemaBuilder schemaBuilder = new Schema.SchemaBuilder().addSingleValueDimension(DIMENSION_D1, DataType.INT) .addSingleValueDimension(DIMENSION_D2, DataType.INT); DataType rawValueType = getRawValueType();
.addMetric("column3", FieldSpec.DataType.INT).addSingleValueDimension("column5", FieldSpec.DataType.STRING) .addSingleValueDimension("column6", FieldSpec.DataType.INT) .addSingleValueDimension("column7", FieldSpec.DataType.INT) .addSingleValueDimension("column9", FieldSpec.DataType.INT) .addSingleValueDimension("column11", FieldSpec.DataType.STRING) .addSingleValueDimension("column12", FieldSpec.DataType.STRING) .addMetric("column17", FieldSpec.DataType.INT).addMetric("column18", FieldSpec.DataType.INT) .addTime("daysSinceEpoch", TimeUnit.DAYS, FieldSpec.DataType.INT); if (hasPreGeneratedHllColumns) { schemaBuilder.addSingleValueDimension("column17_HLL", FieldSpec.DataType.STRING) .addSingleValueDimension("column18_HLL", FieldSpec.DataType.STRING);
@Test public void TestReadAvro() throws Exception { final String filePath = TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(AVRO_DATA)); final String jsonPath = TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(JSON_DATA)); Schema schema = new Schema.SchemaBuilder().addSingleValueDimension("column3", DataType.STRING) .addSingleValueDimension("column2", DataType.STRING).build(); final SegmentGeneratorConfig config = new SegmentGeneratorConfig(schema); config.setFormat(FileFormat.AVRO); config.setInputFilePath(filePath); config.setSegmentVersion(SegmentVersion.v1); AvroRecordReader avroDataPublisher = (AvroRecordReader) RecordReaderFactory.getRecordReader(config); int cnt = 0; for (String line : FileUtils.readLines(new File(jsonPath))) { JsonNode jsonNode = JsonUtils.stringToJsonNode(line); if (avroDataPublisher.hasNext()) { GenericRow recordRow = avroDataPublisher.next(); for (String column : recordRow.getFieldNames()) { String valueFromJson = jsonNode.get(column).asText(); String valueFromAvro = recordRow.getValue(column).toString(); if (cnt > 1) { Assert.assertEquals(valueFromJson, valueFromAvro); } } } cnt++; } Assert.assertEquals(cnt, 10001); }
@BeforeTest public void buildSegment() throws Exception { FileUtils.deleteQuietly(INDEX_DIR); // Get resource file path. URL resource = getClass().getClassLoader().getResource(AVRO_DATA); Assert.assertNotNull(resource); String filePath = resource.getFile(); // Build the segment schema. Schema schema = new Schema.SchemaBuilder().setSchemaName("testTable").addMetric("column1", FieldSpec.DataType.INT) .addMetric("column3", FieldSpec.DataType.INT).addSingleValueDimension("column5", FieldSpec.DataType.STRING) .addSingleValueDimension("column6", FieldSpec.DataType.INT) .addSingleValueDimension("column7", FieldSpec.DataType.INT) .addSingleValueDimension("column9", FieldSpec.DataType.INT) .addSingleValueDimension("column11", FieldSpec.DataType.STRING) .addSingleValueDimension("column12", FieldSpec.DataType.STRING).addMetric("column17", FieldSpec.DataType.INT) .addMetric("column18", FieldSpec.DataType.INT).addTime("daysSinceEpoch", TimeUnit.DAYS, FieldSpec.DataType.INT) .build(); // Create the segment generator config. SegmentGeneratorConfig segmentGeneratorConfig = new SegmentGeneratorConfig(schema); segmentGeneratorConfig.setInputFilePath(filePath); segmentGeneratorConfig.setTableName("testTable"); segmentGeneratorConfig.setOutDir(INDEX_DIR.getAbsolutePath()); segmentGeneratorConfig .setInvertedIndexCreationColumns(Arrays.asList("column6", "column7", "column11", "column17", "column18")); // Build the index segment. SegmentIndexCreationDriver driver = new SegmentIndexCreationDriverImpl(); driver.init(segmentGeneratorConfig); driver.build(); }
@BeforeTest public void buildSegment() throws Exception { FileUtils.deleteQuietly(INDEX_DIR); // Get resource file path. URL resource = getClass().getClassLoader().getResource(AVRO_DATA); Assert.assertNotNull(resource); String filePath = resource.getFile(); // Build the segment schema. Schema schema = new Schema.SchemaBuilder().setSchemaName("testTable").addMetric("column1", FieldSpec.DataType.INT) .addMetric("column2", FieldSpec.DataType.INT).addSingleValueDimension("column3", FieldSpec.DataType.STRING) .addSingleValueDimension("column5", FieldSpec.DataType.STRING) .addMultiValueDimension("column6", FieldSpec.DataType.INT) .addMultiValueDimension("column7", FieldSpec.DataType.INT) .addSingleValueDimension("column8", FieldSpec.DataType.INT).addMetric("column9", FieldSpec.DataType.INT) .addMetric("column10", FieldSpec.DataType.INT).addTime("daysSinceEpoch", TimeUnit.DAYS, FieldSpec.DataType.INT) .build(); // Create the segment generator config. SegmentGeneratorConfig segmentGeneratorConfig = new SegmentGeneratorConfig(schema); segmentGeneratorConfig.setInputFilePath(filePath); segmentGeneratorConfig.setTableName("testTable"); segmentGeneratorConfig.setOutDir(INDEX_DIR.getAbsolutePath()); segmentGeneratorConfig.setInvertedIndexCreationColumns(Arrays.asList("column3", "column7", "column8", "column9")); // Build the index segment. SegmentIndexCreationDriver driver = new SegmentIndexCreationDriverImpl(); driver.init(segmentGeneratorConfig); driver.build(); }
private Schema getSchema() { return new Schema.SchemaBuilder().setSchemaName("ThriftSampleData") .addSingleValueDimension("id", FieldSpec.DataType.INT) .addSingleValueDimension("name", FieldSpec.DataType.STRING) .addSingleValueDimension("created_at", FieldSpec.DataType.LONG) .addSingleValueDimension("active", FieldSpec.DataType.BOOLEAN) .addMultiValueDimension("groups", FieldSpec.DataType.INT) .addMultiValueDimension("map_values", FieldSpec.DataType.STRING) .addMultiValueDimension("set_values", FieldSpec.DataType.STRING).build(); }
public static void main(String[] args) throws IOException { SchemaBuilder schemaBuilder = new SchemaBuilder(); schemaBuilder.addSingleValueDimension("name", DataType.STRING); schemaBuilder.addSingleValueDimension("age", DataType.INT); schemaBuilder.addMetric("percent", DataType.FLOAT); schemaBuilder.addTime("days", TimeUnit.DAYS, DataType.LONG); Schema schema = schemaBuilder.build(); System.out.println(JsonUtils.objectToPrettyString(schema)); } }