/** * Helper method to create a DatasetProperties from a map of key/values. */ public static DatasetProperties of(Map<String, String> props) { return builder().addAll(props).build(); }
/** * Adds datasets and types to the given {@link DatasetFramework}. Used by the upgrade tool to upgrade Datasets * * @param datasetFramework framework to add types and datasets to */ public static void setupDatasets(DatasetFramework datasetFramework) throws IOException, DatasetManagementException { datasetFramework.addInstance(JobQueueDataset.class.getName(), Schedulers.JOB_QUEUE_DATASET_ID, DatasetProperties.builder(). add(Constants.Scheduler.JOB_QUEUE_NUM_PARTITIONS, DEFAULT_NUM_PARTITIONS) .build()); } }
private DatasetProperties addIndexColumns(DatasetProperties properties, String... indexColumns) { return DatasetProperties .builder() .addAll(properties.getProperties()) .add(IndexedTable.INDEX_COLUMNS_CONF_KEY, Joiner.on(",").join(indexColumns)) .build(); }
private DatasetProperties addIndexColumns(DatasetProperties properties, String... indexColumns) { return DatasetProperties .builder() .addAll(properties.getProperties()) .add(IndexedTable.INDEX_COLUMNS_CONF_KEY, Joiner.on(",").join(indexColumns)) .build(); }
public static MetadataDataset getMetadataDataset(DatasetContext context, DatasetFramework dsFramework, MetadataScope scope) { try { return DatasetsUtil.getOrCreateDataset( context, dsFramework, getMetadataDatasetInstance(scope), MetadataDataset.class.getName(), DatasetProperties.builder().add(MetadataDatasetDefinition.SCOPE_KEY, scope.name()).build()); } catch (DatasetManagementException | IOException e) { throw Throwables.propagate(e); } }
@Override public void prepareRun(BatchSinkContext context) throws DatasetManagementException { Map<String, String> properties = getProperties(); if (!context.datasetExists(properties.get(Properties.BatchReadableWritable.NAME))) { context.createDataset(properties.get(Properties.BatchReadableWritable.NAME), properties.get(Properties.BatchReadableWritable.TYPE), DatasetProperties.builder().addAll(properties).build()); } context.addOutput(Output.ofDataset(properties.get(Properties.BatchReadableWritable.NAME))); } }
private static MetadataDataset getMetadataDataset(DatasetContext context, DatasetFramework dsFramework, MetadataScope scope) { try { return DatasetsUtil.getOrCreateDataset(context, dsFramework, getMetadataDatasetInstance(scope), MetadataDataset.class.getName(), DatasetProperties.builder().add(MetadataDatasetDefinition.SCOPE_KEY, scope.name()).build()); } catch (DatasetManagementException | IOException e) { throw Throwables.propagate(e); } }
private MetadataDataset getMetadataDataset(DatasetContext context, DatasetId datasetId) throws IOException, DatasetManagementException { MetadataScope scope = datasetId.getDataset().contains("business") ? MetadataScope.USER : MetadataScope.SYSTEM; return DatasetsUtil.getOrCreateDataset(context, dsFramework, datasetId, MetadataDataset.class.getName(), DatasetProperties.builder().add(MetadataDatasetDefinition.SCOPE_KEY, scope.name()).build()); } }
@Override public void prepareRun(BatchSourceContext context) throws DatasetManagementException { Map<String, String> properties = getProperties(); // if macros were provided at runtime, dataset needs to be created now if (!context.datasetExists(properties.get(Properties.BatchReadableWritable.NAME))) { context.createDataset(properties.get(Properties.BatchReadableWritable.NAME), properties.get(Properties.BatchReadableWritable.TYPE), DatasetProperties.builder().addAll(properties).build()); } context.setInput(Input.ofDataset(properties.get(Properties.BatchReadableWritable.NAME))); } }
private DatasetProperties computeFactTableProperties(DatasetProperties props) { // Configuring tables that hold data of specific resolution Map<String, Aggregation> aggregations = getAggregations(props.getProperties()); // Adding pre-splitting for fact tables byte[][] splits = FactTable.getSplits(aggregations.size()); // and combine them return DatasetProperties.builder() .addAll(props.getProperties()) .add(HBaseTableAdmin.PROPERTY_SPLITS, GSON.toJson(splits)) .build(); }
private DatasetProperties computeFactTableProperties(DatasetProperties props) { // Configuring tables that hold data of specific resolution Map<String, Aggregation> aggregations = getAggregations(props.getProperties()); // Adding pre-splitting for fact tables byte[][] splits = FactTable.getSplits(aggregations.size()); // and combine them return DatasetProperties.builder() .addAll(props.getProperties()) .add(HBaseTableAdmin.PROPERTY_SPLITS, GSON.toJson(splits)) .build(); }
/** * Creates properties for {@link TimeseriesTable} data set instance. * @param timeIntervalToStorePerRow time interval to store per row. See {@link TimeseriesTable} for details. * @return {@link co.cask.cdap.api.dataset.DatasetProperties} for the data set */ public static DatasetProperties timeseriesTableProperties(int timeIntervalToStorePerRow, DatasetProperties props) { return DatasetProperties.builder() .add(TimeseriesTable.ATTR_TIME_INTERVAL_TO_STORE_PER_ROW, timeIntervalToStorePerRow) .addAll(props.getProperties()) .build(); } }
private static MetadataDataset getDataset(DatasetId instance, MetadataScope scope) throws Exception { return DatasetsUtil.getOrCreateDataset(dsFrameworkUtil.getFramework(), instance, MetadataDataset.class.getName(), DatasetProperties.builder() .add(MetadataDatasetDefinition.SCOPE_KEY, scope.name()) .build(), null); } }
@Override public void configure() { setName(NAME); addDatasetModule("record", RecordDatasetModule.class); createDataset(DATASET_NAME, RecordDataset.class.getName(), DatasetProperties.builder().add("recordClassName", getRecordClass().getName()).build()); addService("NoOpService", new NoOpHandler()); }
@BeforeClass public static void beforeClass() throws Exception { dsFrameworkUtil.createInstance("indexedTable", tabInstance, DatasetProperties.builder() .add(IndexedTable.INDEX_COLUMNS_CONF_KEY, idxColString) .build()); table = dsFrameworkUtil.getInstance(tabInstance); }
@Override public void configure() { setName(NAME); addDatasetModule("other", RecordDatasetModule.class); createDataset(DATASET_NAME, RecordDataset.class, DatasetProperties.builder().add("recordClassName", getRecordClass().getName()).build()); addService("NoOpService", new NoOpHandler()); } }
@Override public void configure() { setName(NAME); setDescription("SampleWorkflow description"); createLocalDataset(TABLE_NAME, KeyValueTable.class, DatasetProperties.builder().add("foo", "bar").build()); createLocalDataset(FILE_NAME, FileSet.class, DatasetProperties.builder().add("anotherFoo", "anotherBar").build()); addAction(new DummyAction(FIRST_ACTION)); addAction(new DummyAction(SECOND_ACTION)); addMapReduce(WORD_COUNT_MR); } }
@Override public void configure() { DatasetProperties props = DatasetProperties.builder() .add("dataset.cube.resolutions", "1,60") .add("dataset.cube.aggregation.agg1.dimensions", "user,action") .add("dataset.cube.aggregation.agg1.requiredDimensions", "user,action").build(); createDataset(CUBE_NAME, Cube.class, props); addService(SERVICE_NAME, new CubeHandler()); }
@Test public void testPutAndGet() throws Exception { dsFrameworkUtil.createInstance("table", users, DatasetProperties.builder().build()); try { final Table usersTable = dsFrameworkUtil.getInstance(users); final byte[] rowKey = Bytes.toBytes(123); final Schema schema = new ReflectionSchemaGenerator().generate(User.class); assertGetAndPut(usersTable, rowKey, SAMUEL, schema); } finally { dsFrameworkUtil.deleteInstance(users); } }
@Override public void configure() { setName("FooApp"); setDescription("Foo App"); createDataset("dataset2", KeyValueTable.class); createDataset("dataset3", IndexedTable.class, DatasetProperties.builder().add(IndexedTable.INDEX_COLUMNS_CONF_KEY, "foo").build()); addMapReduce(new FooMapReduceJob("mrJob2")); addMapReduce(new FooMapReduceJob("mrJob3")); } }