@Override public void prepareRun(BatchSinkContext context) throws DatasetManagementException { Map<String, String> properties = getProperties(); if (!context.datasetExists(properties.get(Properties.BatchReadableWritable.NAME))) { context.createDataset(properties.get(Properties.BatchReadableWritable.NAME), properties.get(Properties.BatchReadableWritable.TYPE), DatasetProperties.builder().addAll(properties).build()); } context.addOutput(Output.ofDataset(properties.get(Properties.BatchReadableWritable.NAME))); } }
private DatasetProperties addIndexColumns(DatasetProperties properties, String... indexColumns) { return DatasetProperties .builder() .addAll(properties.getProperties()) .add(IndexedTable.INDEX_COLUMNS_CONF_KEY, Joiner.on(",").join(indexColumns)) .build(); }
private DatasetProperties addIndexColumns(DatasetProperties properties, String... indexColumns) { return DatasetProperties .builder() .addAll(properties.getProperties()) .add(IndexedTable.INDEX_COLUMNS_CONF_KEY, Joiner.on(",").join(indexColumns)) .build(); }
private DatasetProperties noConflictDetection(DatasetProperties properties) { // Use ConflictDetection.NONE as we only need a flag whether a program uses a dataset/stream. // Having conflict detection will lead to failures when programs try to register accesses at the same time. return TableProperties.builder() .setConflictDetection(ConflictDetection.NONE) .addAll(properties.getProperties()) .build(); }
private static MetadataDataset getMetadataDataset(DatasetContext context, DatasetFramework dsFramework, MetadataScope scope) { try { return DatasetsUtil.getOrCreateDataset(context, dsFramework, getMetadataDatasetInstance(scope), MetadataDataset.class.getName(), DatasetProperties.builder().add(MetadataDatasetDefinition.SCOPE_KEY, scope.name()).build()); } catch (DatasetManagementException | IOException e) { throw Throwables.propagate(e); } }
@Override public DatasetSpecification configure(String instanceName, DatasetProperties properties) { // add the partition key to the properties. properties = PartitionedFileSetProperties .builder() .setPartitioning(TimePartitionedFileSetDataset.PARTITIONING) .addAll(properties.getProperties()) .build(); return super.configure(instanceName, properties); }
private DatasetProperties computeFactTableProperties(DatasetProperties props) { // Configuring tables that hold data of specific resolution Map<String, Aggregation> aggregations = getAggregations(props.getProperties()); // Adding pre-splitting for fact tables byte[][] splits = FactTable.getSplits(aggregations.size()); // and combine them return DatasetProperties.builder() .addAll(props.getProperties()) .add(HBaseTableAdmin.PROPERTY_SPLITS, GSON.toJson(splits)) .build(); }
@Override public void configure() { setName(NAME); addDatasetModule("other", RecordDatasetModule.class); createDataset(DATASET_NAME, RecordDataset.class, DatasetProperties.builder().add("recordClassName", getRecordClass().getName()).build()); addService("NoOpService", new NoOpHandler()); } }
@Override public void configure() { setName(NAME); addDatasetModule("record", RecordDatasetModule.class); createDataset(DATASET_NAME, RecordDataset.class.getName(), DatasetProperties.builder().add("recordClassName", getRecordClass().getName()).build()); addService("NoOpService", new NoOpHandler()); }
/** * Creates properties for {@link TimeseriesTable} data set instance. * @param timeIntervalToStorePerRow time interval to store per row. See {@link TimeseriesTable} for details. * @return {@link co.cask.cdap.api.dataset.DatasetProperties} for the data set */ public static DatasetProperties timeseriesTableProperties(int timeIntervalToStorePerRow, DatasetProperties props) { return DatasetProperties.builder() .add(TimeseriesTable.ATTR_TIME_INTERVAL_TO_STORE_PER_ROW, timeIntervalToStorePerRow) .addAll(props.getProperties()) .build(); } }
@Override public DatasetSpecification reconfigure(String instanceName, DatasetProperties properties, DatasetSpecification currentSpec) throws IncompatibleUpdateException { // add the partition key to the properties. properties = PartitionedFileSetProperties .builder() .setPartitioning(TimePartitionedFileSetDataset.PARTITIONING) .addAll(properties.getProperties()) .build(); return super.reconfigure(instanceName, properties, currentSpec); }
@BeforeClass public static void beforeClass() throws Exception { dsFrameworkUtil.createInstance("indexedTable", tabInstance, DatasetProperties.builder() .add(IndexedTable.INDEX_COLUMNS_CONF_KEY, idxColString) .build()); table = dsFrameworkUtil.getInstance(tabInstance); }
private DatasetProperties noConflictDetection(DatasetProperties properties) { // Use ConflictDetection.NONE as we only need a flag whether a program uses a dataset/stream. // Having conflict detection will lead to failures when programs try to register accesses at the same time. return TableProperties.builder() .setConflictDetection(ConflictDetection.NONE) .addAll(properties.getProperties()) .build(); }
@Override public DatasetSpecification reconfigure(String instanceName, DatasetProperties properties, DatasetSpecification currentSpec) throws IncompatibleUpdateException { // add the partition key to the properties. properties = PartitionedFileSetProperties .builder() .setPartitioning(TimePartitionedFileSetDataset.PARTITIONING) .addAll(properties.getProperties()) .build(); return super.reconfigure(instanceName, properties, currentSpec); }
private DatasetProperties computeFactTableProperties(DatasetProperties props) { // Configuring tables that hold data of specific resolution Map<String, Aggregation> aggregations = getAggregations(props.getProperties()); // Adding pre-splitting for fact tables byte[][] splits = FactTable.getSplits(aggregations.size()); // and combine them return DatasetProperties.builder() .addAll(props.getProperties()) .add(HBaseTableAdmin.PROPERTY_SPLITS, GSON.toJson(splits)) .build(); }
@Override public void configure() { setName("FooApp"); setDescription("Foo App"); createDataset("dataset2", KeyValueTable.class); createDataset("dataset3", IndexedTable.class, DatasetProperties.builder().add(IndexedTable.INDEX_COLUMNS_CONF_KEY, "foo").build()); addMapReduce(new FooMapReduceJob("mrJob2")); addMapReduce(new FooMapReduceJob("mrJob3")); } }
@Override public void configure() { DatasetProperties props = DatasetProperties.builder() .add("dataset.cube.resolutions", "1,60") .add("dataset.cube.aggregation.agg1.dimensions", "user,action") .add("dataset.cube.aggregation.agg1.requiredDimensions", "user,action").build(); createDataset(CUBE_NAME, Cube.class, props); addService(SERVICE_NAME, new CubeHandler()); }
@Override public DatasetSpecification configure(String instanceName, DatasetProperties properties) { // add the partition key to the properties. properties = PartitionedFileSetProperties .builder() .setPartitioning(TimePartitionedFileSetDataset.PARTITIONING) .addAll(properties.getProperties()) .build(); return super.configure(instanceName, properties); }
public static MetadataDataset getMetadataDataset(DatasetContext context, DatasetFramework dsFramework, MetadataScope scope) { try { return DatasetsUtil.getOrCreateDataset( context, dsFramework, getMetadataDatasetInstance(scope), MetadataDataset.class.getName(), DatasetProperties.builder().add(MetadataDatasetDefinition.SCOPE_KEY, scope.name()).build()); } catch (DatasetManagementException | IOException e) { throw Throwables.propagate(e); } }
/** * Helper method to create a DatasetProperties from a map of key/values. */ public static DatasetProperties of(Map<String, String> props) { return builder().addAll(props).build(); }