public Collection<DatasetSpecification> getByTypes(NamespaceId namespaceId, Set<String> typeNames) { List<DatasetSpecification> filtered = Lists.newArrayList(); for (DatasetSpecification spec : getAll(namespaceId)) { if (typeNames.contains(spec.getType())) { filtered.add(spec); } } return filtered; }
public Collection<DatasetSpecification> getByTypes(NamespaceId namespaceId, Set<String> typeNames) { List<DatasetSpecification> filtered = Lists.newArrayList(); for (DatasetSpecification spec : getAll(namespaceId)) { if (typeNames.contains(spec.getType())) { filtered.add(spec); } } return filtered; }
@Override public void deleteAllModules(NamespaceId namespaceId) throws ModuleConflictException { writeLock.lock(); try { // check if there are any datasets that use types from the namespace from which we want to remove all modules Set<String> typesInNamespace = nonDefaultTypes.get(namespaceId); for (DatasetSpecification spec : instances.row(namespaceId).values()) { if (typesInNamespace.contains(spec.getType())) { throw new ModuleConflictException( String.format("Cannot delete all modules in namespace '%s', some datasets use them", namespaceId)); } } moduleClasses.row(namespaceId).clear(); nonDefaultTypes.removeAll(namespaceId); registries.put(namespaceId, registryFactory.create()); } finally { writeLock.unlock(); } }
@Override public void deleteAllModules(NamespaceId namespaceId) throws ModuleConflictException { writeLock.lock(); try { // check if there are any datasets that use types from the namespace from which we want to remove all modules Set<String> typesInNamespace = nonDefaultTypes.get(namespaceId); for (DatasetSpecification spec : instances.row(namespaceId).values()) { if (typesInNamespace.contains(spec.getType())) { throw new ModuleConflictException( String.format("Cannot delete all modules in namespace '%s', some datasets use them", namespaceId)); } } moduleClasses.row(namespaceId).clear(); nonDefaultTypes.removeAll(namespaceId); registries.put(namespaceId, registryFactory.create()); } finally { writeLock.unlock(); } }
@Override public Collection<DatasetSpecificationSummary> getInstances(NamespaceId namespaceId, Map<String, String> properties) { readLock.lock(); try { // don't expect this to be called a lot. // might be better to maintain this collection separately and just return it, but seems like its not worth it. Collection<DatasetSpecification> specs = instances.row(namespaceId).values(); ImmutableList.Builder<DatasetSpecificationSummary> specSummaries = ImmutableList.builder(); for (DatasetSpecification spec : specs) { if (properties.isEmpty() || Maps.difference(properties, spec.getProperties()).entriesOnlyOnLeft().isEmpty()) { specSummaries.add(new DatasetSpecificationSummary(spec.getName(), spec.getType(), spec.getProperties())); } } return specSummaries.build(); } finally { readLock.unlock(); } }
@Override public Collection<DatasetSpecificationSummary> getInstances(NamespaceId namespaceId, Map<String, String> properties) { readLock.lock(); try { // don't expect this to be called a lot. // might be better to maintain this collection separately and just return it, but seems like its not worth it. Collection<DatasetSpecification> specs = instances.row(namespaceId).values(); ImmutableList.Builder<DatasetSpecificationSummary> specSummaries = ImmutableList.builder(); for (DatasetSpecification spec : specs) { if (properties.isEmpty() || Maps.difference(properties, spec.getProperties()).entriesOnlyOnLeft().isEmpty()) { specSummaries.add(new DatasetSpecificationSummary(spec.getName(), spec.getType(), spec.getProperties())); } } return specSummaries.build(); } finally { readLock.unlock(); } }
static Collection<DatasetSpecificationSummary> spec2Summary(Collection<DatasetSpecification> specs) { List<DatasetSpecificationSummary> datasetSummaries = Lists.newArrayList(); for (DatasetSpecification spec : specs) { // TODO: (CDAP-3097) handle system datasets specially within a namespace instead of filtering them out // by the handler. This filter is only in the list endpoint because the other endpoints are used by // HBaseQueueAdmin through DatasetFramework. spec = DatasetsUtil.fixOriginalProperties(spec); datasetSummaries.add(new DatasetSpecificationSummary(spec.getName(), spec.getType(), spec.getDescription(), spec.getOriginalProperties())); } return datasetSummaries; }
static Collection<DatasetSpecificationSummary> spec2Summary(Collection<DatasetSpecification> specs) { List<DatasetSpecificationSummary> datasetSummaries = Lists.newArrayList(); for (DatasetSpecification spec : specs) { // TODO: (CDAP-3097) handle system datasets specially within a namespace instead of filtering them out // by the handler. This filter is only in the list endpoint because the other endpoints are used by // HBaseQueueAdmin through DatasetFramework. if (QueueConstants.STATE_STORE_NAME.equals(spec.getName())) { continue; } spec = DatasetsUtil.fixOriginalProperties(spec); datasetSummaries.add(new DatasetSpecificationSummary(spec.getName(), spec.getType(), spec.getDescription(), spec.getOriginalProperties())); } return datasetSummaries; }
@Nullable @Override public <T extends DatasetAdmin> T getAdmin(DatasetId datasetInstanceId, @Nullable ClassLoader classLoader, DatasetClassLoaderProvider classLoaderProvider) throws IOException { readLock.lock(); try { DatasetSpecification spec = instances.get(datasetInstanceId.getParent(), datasetInstanceId); if (spec == null) { return null; } LinkedHashSet<String> availableModuleClasses = getAvailableModuleClasses(datasetInstanceId.getParent()); DatasetDefinition impl = createRegistry(availableModuleClasses, classLoader).get(spec.getType()); return (T) impl.getAdmin(DatasetContext.from(datasetInstanceId.getNamespace()), spec, classLoader); } finally { readLock.unlock(); } }
@Override public void deleteAllInstances(NamespaceId namespaceId) throws DatasetManagementException, IOException { writeLock.lock(); try { for (DatasetSpecification spec : instances.row(namespaceId).values()) { DatasetDefinition def = getDefinitionForType(namespaceId, spec.getType()); if (def == null) { throw new DatasetManagementException( String.format("Dataset type '%s' is neither registered in the '%s' namespace nor in the system namespace", spec.getType(), namespaceId)); } def.getAdmin(DatasetContext.from(namespaceId.getEntityName()), spec, null).drop(); publishAudit(namespaceId.dataset(spec.getName()), AuditType.DELETE); } instances.row(namespaceId).clear(); } finally { writeLock.unlock(); } }
@Override public void deleteAllInstances(NamespaceId namespaceId) throws DatasetManagementException, IOException { writeLock.lock(); try { for (DatasetSpecification spec : instances.row(namespaceId).values()) { DatasetDefinition def = getDefinitionForType(namespaceId, spec.getType()); if (def == null) { throw new DatasetManagementException( String.format("Dataset type '%s' is neither registered in the '%s' namespace nor in the system namespace", spec.getType(), namespaceId)); } def.getAdmin(DatasetContext.from(namespaceId.getEntityName()), spec, null).drop(); publishAudit(namespaceId.dataset(spec.getName()), AuditType.DELETE); } instances.row(namespaceId).clear(); } finally { writeLock.unlock(); } }
@Nullable @Override public <T extends DatasetAdmin> T getAdmin(DatasetId datasetInstanceId, @Nullable ClassLoader classLoader, DatasetClassLoaderProvider classLoaderProvider) throws IOException { readLock.lock(); try { DatasetSpecification spec = instances.get(datasetInstanceId.getParent(), datasetInstanceId); if (spec == null) { return null; } LinkedHashSet<String> availableModuleClasses = getAvailableModuleClasses(datasetInstanceId.getParent()); DatasetDefinition impl = createRegistry(availableModuleClasses, classLoader).get(spec.getType()); return (T) impl.getAdmin(DatasetContext.from(datasetInstanceId.getNamespace()), spec, classLoader); } finally { readLock.unlock(); } }
@Override public void truncateInstance(DatasetId instanceId) throws DatasetManagementException, IOException { writeLock.lock(); try { DatasetSpecification spec = instances.get(instanceId.getParent(), instanceId); if (spec == null) { throw new InstanceNotFoundException(instanceId.getEntityName()); } DatasetDefinition def = getDefinitionForType(instanceId.getParent(), spec.getType()); if (def == null) { throw new DatasetManagementException( String.format("Dataset type '%s' is neither registered in the '%s' namespace nor in the system namespace", spec.getType(), instanceId.getParent())); } def.getAdmin(DatasetContext.from(instanceId.getNamespace()), spec, null).truncate(); publishAudit(instanceId, AuditType.TRUNCATE); } finally { writeLock.unlock(); } }
@Override public void truncateInstance(DatasetId instanceId) throws DatasetManagementException, IOException { writeLock.lock(); try { DatasetSpecification spec = instances.get(instanceId.getParent(), instanceId); if (spec == null) { throw new InstanceNotFoundException(instanceId.getEntityName()); } DatasetDefinition def = getDefinitionForType(instanceId.getParent(), spec.getType()); if (def == null) { throw new DatasetManagementException( String.format("Dataset type '%s' is neither registered in the '%s' namespace nor in the system namespace", spec.getType(), instanceId.getParent())); } def.getAdmin(DatasetContext.from(instanceId.getNamespace()), spec, null).truncate(); publishAudit(instanceId, AuditType.TRUNCATE); } finally { writeLock.unlock(); } }
@Override public void deleteInstance(DatasetId instanceId) throws DatasetManagementException, IOException { writeLock.lock(); try { DatasetSpecification spec = instances.remove(instanceId.getParent(), instanceId); if (spec == null) { throw new InstanceNotFoundException(instanceId.getEntityName()); } DatasetDefinition def = getDefinitionForType(instanceId.getParent(), spec.getType()); if (def == null) { throw new DatasetManagementException( String.format("Dataset type '%s' is neither registered in the '%s' namespace nor in the system namespace", spec.getType(), instanceId.getParent())); } def.getAdmin(DatasetContext.from(instanceId.getNamespace()), spec, null).drop(); publishAudit(instanceId, AuditType.DELETE); } finally { writeLock.unlock(); } }
@Override public void deleteInstance(DatasetId instanceId) throws DatasetManagementException, IOException { writeLock.lock(); try { DatasetSpecification spec = instances.remove(instanceId.getParent(), instanceId); if (spec == null) { throw new InstanceNotFoundException(instanceId.getEntityName()); } DatasetDefinition def = getDefinitionForType(instanceId.getParent(), spec.getType()); if (def == null) { throw new DatasetManagementException( String.format("Dataset type '%s' is neither registered in the '%s' namespace nor in the system namespace", spec.getType(), instanceId.getParent())); } def.getAdmin(DatasetContext.from(instanceId.getNamespace()), spec, null).drop(); publishAudit(instanceId, AuditType.DELETE); } finally { writeLock.unlock(); } }
private DatasetSpecificationSummary spec2Summary(DatasetSpecification spec) { return new DatasetSpecificationSummary(spec.getName(), spec.getType(), spec.getDescription(), spec.getOriginalProperties()); }
@Override public ObjectMappedTableDataset<?> getDataset(DatasetContext datasetContext, DatasetSpecification spec, Map<String, String> arguments, ClassLoader classLoader) throws IOException { String keyName = ObjectMappedTableProperties.getRowKeyExploreName(spec.getProperties()); DatasetSpecification tableSpec = spec.getSpecification(TABLE_NAME); // if the table spec did not have schema, this is an ObjectMappedTable from CDAP 2.8. // add the schema and row key as arguments so that explore will work // TODO: remove after CDAP-2122 is done if (!tableSpec.getProperties().containsKey(Table.PROPERTY_SCHEMA)) { tableSpec = DatasetSpecification.builder(tableSpec.getName(), tableSpec.getType()) .properties(tableSpec.getProperties()) .property(Table.PROPERTY_SCHEMA, spec.getProperty(Table.PROPERTY_SCHEMA)) .property(Table.PROPERTY_SCHEMA_ROW_FIELD, keyName) .datasets(tableSpec.getSpecifications().values()) .build(); } // reconstruct the table schema here because of backwards compatibility DatasetDefinition<Table, DatasetAdmin> tableDef = getDelegate(TABLE_NAME); Table table = tableDef.getDataset(datasetContext, tableSpec, arguments, classLoader); Map<String, String> properties = spec.getProperties(); TypeRepresentation typeRep = GSON.fromJson( ObjectMappedTableProperties.getObjectTypeRepresentation(properties), TypeRepresentation.class); Schema objSchema = ObjectMappedTableProperties.getObjectSchema(properties); return new ObjectMappedTableDataset(spec.getName(), table, typeRep, objSchema, classLoader); }
@Test public void testNamespacing() { DatasetSpecification innerSpec = DatasetSpecification.builder("inner", "table") .build(); DatasetSpecification outerSpec = DatasetSpecification.builder("kv", "kvtable") .datasets(innerSpec) .build(); Assert.assertEquals("kv", outerSpec.getName()); Assert.assertEquals("kvtable", outerSpec.getType()); DatasetSpecification actualInner = outerSpec.getSpecification("inner"); Assert.assertEquals("kv.inner", actualInner.getName()); Assert.assertEquals("table", actualInner.getType()); Assert.assertTrue(actualInner.getSpecifications().isEmpty()); }
@Test public void testSimpleSpec() { String name = "name"; String type = "type"; String propKey = "prop1"; String propVal = "val1"; DatasetSpecification spec = DatasetSpecification.builder(name, type) .property(propKey, propVal) .build(); Assert.assertEquals(name, spec.getName()); Assert.assertEquals(type, spec.getType()); Assert.assertEquals(propVal, spec.getProperty(propKey)); Assert.assertEquals(1, spec.getProperties().size()); Assert.assertTrue(spec.getSpecifications().isEmpty()); }