try { if (columnDescriptor.exists()) { descriptorBuilder.columnMapping(columnDescriptor); } else { descriptorBuilder.columnMapping(Resources.getResource( columnDescriptorFile).openStream());
@Test public void testMappingSourceMustBeSchemaField() { Assert.assertNotNull(new DatasetDescriptor.Builder() .schema(USER_SCHEMA) .columnMapping(new ColumnMapping.Builder() .column("id", "meta", "id") .build()) .build()); TestHelpers.assertThrows("Should reject mapping source not in schema", ValidationException.class, new Runnable() { @Override public void run() { new DatasetDescriptor.Builder() .schema(USER_SCHEMA) .columnMapping(new ColumnMapping.Builder() .column("created_at", "meta", "created_at") .build()) .build(); } }); }
.columnMapping(new ColumnMapping.Builder() .keyAsColumn("custom_attributes", "attrs") .build()) .columnMapping(new ColumnMapping.Builder() .keyAsColumn("preferences", "prefs") .build())
@Test public void testKeyMappingSourceMustBeIdentityPartitioned() { // and it works when the field is present Assert.assertNotNull(new DatasetDescriptor.Builder() .schema(USER_SCHEMA) .partitionStrategy(new PartitionStrategy.Builder() .hash("id", 16) .identity("id") .build()) .columnMapping(new ColumnMapping.Builder() .key("id") .build()) .build()); TestHelpers.assertThrows("Should reject mapping source not id partitioned", ValidationException.class, new Runnable() { @Override public void run() { new DatasetDescriptor.Builder() .schema(USER_SCHEMA) .partitionStrategy(new PartitionStrategy.Builder() .hash("id", 16) .build()) .columnMapping(new ColumnMapping.Builder() .key("id") .build()) .build(); } } ); }
/** * Configure the dataset's column mappings from a URI. * * @param uri * A URI to a column mapping JSON file * @return This builder for method chaining * @throws ValidationException * If the literal is not valid JSON-encoded column mappings * @throws java.io.IOException * If accessing the URI results in an IOException * * @since 0.14.0 */ public Builder columnMappingUri(URI uri) throws IOException { InputStream in = null; boolean threw = true; try { in = open(uri); columnMapping(in); threw = false; } finally { Closeables.close(in, threw); } return this; }
@Override public void run() { new DatasetDescriptor.Builder() .schema(USER_SCHEMA) .partitionStrategy(new PartitionStrategy.Builder() .hash("id", 16) .build()) .columnMapping(new ColumnMapping.Builder() .key("id") .build()) .build(); } }
@Override public AvroEntitySchema parseEntitySchema(String rawSchema, ColumnMapping columnMapping) { // use DatasetDescriptor.Builder because it checks consistency DatasetDescriptor descriptor = new DatasetDescriptor.Builder() .schemaLiteral(rawSchema) .columnMapping(columnMapping) .build(); return new AvroEntitySchema( descriptor.getSchema(), rawSchema, descriptor.getColumnMapping()); } }
@Override public void run() { new DatasetDescriptor.Builder() .schema(USER_SCHEMA) .columnMapping(new ColumnMapping.Builder() .keyAsColumn("email", "kac") .build()) .build(); } }
@Override public void run() { new DatasetDescriptor.Builder() .schema(USER_SCHEMA) .columnMapping(new ColumnMapping.Builder() .version("posts") .build()) .build(); } }
@Override public void run() { new DatasetDescriptor.Builder() .schema(USER_SCHEMA) .columnMapping(new ColumnMapping.Builder() .version("preferences") .build()) .build(); } });
@Override public void run() { new DatasetDescriptor.Builder() .schema(USER_SCHEMA) .columnMapping(new ColumnMapping.Builder() .version("custom_attributes") .build()) .build(); } });
@Override public void run() { new DatasetDescriptor.Builder() .schema(USER_SCHEMA) .columnMapping(new ColumnMapping.Builder() .version("name") .build()) .build(); } });
@Override public void run() { new DatasetDescriptor.Builder() .schema(USER_SCHEMA) .columnMapping(new ColumnMapping.Builder() .counter("email", "meta", "email") .build()) .build(); } });
@Override public void run() { new DatasetDescriptor.Builder() .schema(USER_SCHEMA) .columnMapping(new ColumnMapping.Builder() .keyAsColumn("posts", "kac") .build()) .build(); } }
@Override public void run() { new DatasetDescriptor.Builder() .schema(USER_SCHEMA) .columnMapping(new ColumnMapping.Builder() .counter("custom_attributes", "meta", "attrs") .build()) .build(); } });
@Override public AvroEntitySchema parseEntitySchema(String rawSchema, ColumnMapping columnMapping) { // use DatasetDescriptor.Builder because it checks consistency DatasetDescriptor descriptor = new DatasetDescriptor.Builder() .schemaLiteral(rawSchema) .columnMapping(columnMapping) .build(); return new AvroEntitySchema( descriptor.getSchema(), rawSchema, descriptor.getColumnMapping()); } }
@Override public void run() { new DatasetDescriptor.Builder() .schema(USER_SCHEMA) .columnMapping(new ColumnMapping.Builder() .counter("posts", "meta", "post_ids") .build()) .build(); } }
@Override public void run() { new DatasetDescriptor.Builder() .schema(USER_SCHEMA) .columnMapping(new ColumnMapping.Builder() .counter("preferences", "meta", "prefs") .build()) .build(); } }
@Override public void run() { new DatasetDescriptor.Builder() .schema(USER_SCHEMA) .columnMapping(new ColumnMapping.Builder() .keyAsColumn("id", "kac") .build()) .build(); } });
@Override public void run() { new DatasetDescriptor.Builder() .schema(USER_SCHEMA) .columnMapping(new ColumnMapping.Builder() .column("created_at", "meta", "created_at") .build()) .build(); } });