private static void printInfo(Logger console, Dataset<?> dataset) { DatasetDescriptor desc = dataset.getDescriptor(); String schema = ColumnMappingParser.removeEmbeddedMapping( PartitionStrategyParser.removeEmbeddedStrategy(desc.getSchema())) .toString(true); Collection<String> properties = desc.listProperties(); console.info("\nDataset \"{}\":", dataset.getName()); console.info("\tURI: \"{}\"", dataset.getUri()); console.info("\tSchema: {}", indent(schema)); if (desc.isPartitioned()) { console.info("\tPartition strategy: {}", indent(desc.getPartitionStrategy().toString(true))); } else { console.info("\tNot partitioned"); } if (desc.isColumnMapped()) { console.info("\tColumn mapping: {}", indent(desc.getColumnMapping().toString(true))); } if (!properties.isEmpty()) { StringBuilder sb = new StringBuilder(); for (String prop : properties) { sb.append("\n\t\t").append(prop).append("=") .append(desc.getProperty(prop)); } console.info("\tProperties:{}", sb.toString()); } }
@Override public AvroEntitySchema parseEntitySchema(String rawSchema) { DatasetDescriptor descriptor = new DatasetDescriptor.Builder() .schemaLiteral(rawSchema) .build(); return new AvroEntitySchema( descriptor.getSchema(), rawSchema, descriptor.getColumnMapping()); }
@Override public AvroEntitySchema parseEntitySchema(String rawSchema) { DatasetDescriptor descriptor = new DatasetDescriptor.Builder() .schemaLiteral(rawSchema) .build(); return new AvroEntitySchema( descriptor.getSchema(), rawSchema, descriptor.getColumnMapping()); }
@Override public AvroEntitySchema parseEntitySchema(String rawSchema, ColumnMapping columnMapping) { // use DatasetDescriptor.Builder because it checks consistency DatasetDescriptor descriptor = new DatasetDescriptor.Builder() .schemaLiteral(rawSchema) .columnMapping(columnMapping) .build(); return new AvroEntitySchema( descriptor.getSchema(), rawSchema, descriptor.getColumnMapping()); } }
@Override public AvroEntitySchema parseEntitySchema(String rawSchema, ColumnMapping columnMapping) { // use DatasetDescriptor.Builder because it checks consistency DatasetDescriptor descriptor = new DatasetDescriptor.Builder() .schemaLiteral(rawSchema) .columnMapping(columnMapping) .build(); return new AvroEntitySchema( descriptor.getSchema(), rawSchema, descriptor.getColumnMapping()); } }
private static Schema getEmbeddedSchema(DatasetDescriptor descriptor) { // the SchemaManager stores schemas, so this embeds the column mapping and // partition strategy in the schema. the result is parsed by // AvroKeyEntitySchemaParser Schema schema = descriptor.getSchema(); if (descriptor.isColumnMapped()) { schema = ColumnMappingParser .embedColumnMapping(schema, descriptor.getColumnMapping()); } if (descriptor.isPartitioned()) { schema = PartitionStrategyParser .embedPartitionStrategy(schema, descriptor.getPartitionStrategy()); } return schema; }
private static Schema getEmbeddedSchema(DatasetDescriptor descriptor) { // the SchemaManager stores schemas, so this embeds the column mapping and // partition strategy in the schema. the result is parsed by // AvroKeyEntitySchemaParser Schema schema = descriptor.getSchema(); if (descriptor.isColumnMapped()) { schema = ColumnMappingParser .embedColumnMapping(schema, descriptor.getColumnMapping()); } if (descriptor.isPartitioned()) { schema = PartitionStrategyParser .embedPartitionStrategy(schema, descriptor.getPartitionStrategy()); } return schema; }
.build(); String mapping = descriptor.getColumnMapping().toString(!minimize);
String entitySchemaString = schema.toString(true); AvroEntitySchema entitySchema = new AvroEntitySchema( schema, entitySchemaString, descriptor.getColumnMapping());
String entitySchemaString = schema.toString(true); AvroEntitySchema entitySchema = new AvroEntitySchema( schema, entitySchemaString, descriptor.getColumnMapping());
@Override public DatasetDescriptor update(String namespace, String name, DatasetDescriptor descriptor) { Preconditions.checkArgument(DEFAULT_NAMESPACE.equals(namespace), "Non-default namespaces are not supported"); Preconditions.checkNotNull(name, "Dataset name cannot be null"); Preconditions.checkNotNull(descriptor, "Descriptor cannot be null"); Compatibility.checkAndWarn( namespace, HBaseMetadataProvider.getTableName(name), descriptor.getSchema()); Preconditions.checkArgument(descriptor.isColumnMapped(), "Cannot update dataset %s: missing column mapping", name); String tableName = getTableName(name); String entityName = getEntityName(name); schemaManager.refreshManagedSchemaCache(tableName, entityName); Schema newSchema = getEmbeddedSchema(descriptor); String schemaString = newSchema.toString(true); EntitySchema entitySchema = new AvroEntitySchema( newSchema, schemaString, descriptor.getColumnMapping()); if (!schemaManager.hasSchemaVersion(tableName, entityName, entitySchema)) { schemaManager.migrateSchema(tableName, entityName, schemaString); } else { LOG.info("Schema hasn't changed, not migrating: (" + name + ")"); } return getDatasetDescriptor(newSchema, descriptor.getLocation()); }
@Override public DatasetDescriptor update(String namespace, String name, DatasetDescriptor descriptor) { Preconditions.checkArgument(DEFAULT_NAMESPACE.equals(namespace), "Non-default namespaces are not supported"); Preconditions.checkNotNull(name, "Dataset name cannot be null"); Preconditions.checkNotNull(descriptor, "Descriptor cannot be null"); Compatibility.checkAndWarn( namespace, HBaseMetadataProvider.getTableName(name), descriptor.getSchema()); Preconditions.checkArgument(descriptor.isColumnMapped(), "Cannot update dataset %s: missing column mapping", name); String tableName = getTableName(name); String entityName = getEntityName(name); schemaManager.refreshManagedSchemaCache(tableName, entityName); Schema newSchema = getEmbeddedSchema(descriptor); String schemaString = newSchema.toString(true); EntitySchema entitySchema = new AvroEntitySchema( newSchema, schemaString, descriptor.getColumnMapping()); if (!schemaManager.hasSchemaVersion(tableName, entityName, entitySchema)) { schemaManager.migrateSchema(tableName, entityName, schemaString); } else { LOG.info("Schema hasn't changed, not migrating: (" + name + ")"); } return getDatasetDescriptor(newSchema, descriptor.getLocation()); }
@Test public void testBasic() { DatasetDescriptor desc = provider.create("default", tableName + ".TestEntity", new DatasetDescriptor.Builder().schemaLiteral(testEntity).build()); ColumnMapping columnMapping = desc.getColumnMapping(); PartitionStrategy partStrat = desc.getPartitionStrategy(); assertEquals(9, columnMapping.getFieldMappings().size()); assertEquals(2, Accessor.getDefault().getFieldPartitioners(partStrat).size()); }
.column("real_name", "u", "name") .build(); Assert.assertEquals(expected, descriptor.getColumnMapping());
for (String columnFamily : descriptor.getColumnMapping().getRequiredColumnFamilies()) { try { hbaseAdmin.disableTable(tableName);
for (String columnFamily : descriptor.getColumnMapping().getRequiredColumnFamilies()) { try { hbaseAdmin.disableTable(tableName);
.column("real_name", "u", "name") .build(); Assert.assertEquals(expected, descriptor.getColumnMapping());
.column("real_name", "u", "name") .build(); Assert.assertEquals(expected, descriptor.getColumnMapping()); .column("real_name", "u", "name") .build(); Assert.assertEquals(expected, descriptor.getColumnMapping());
.column("real_name", "u", "name") .build(); Assert.assertEquals(expected, descriptor.getColumnMapping()); .column("age", "u", "age") .build(); Assert.assertEquals(expected, descriptor.getColumnMapping());