@Override public String apply(DatasetSpecificationSummary input) { return input.getName(); } });
private DatasetSpecificationSummary getSummaryForInstance(String instanceName, List<DatasetSpecificationSummary> summaries) { for (DatasetSpecificationSummary summary : summaries) { if (instanceName.equals(summary.getName())) { return summary; } } return null; }
@Override public void run() { try { LOG.info("Upgrading dataset in system namespace: {}, spec: {}", spec.getName(), spec.toString()); DatasetAdmin admin = dsFramework.getAdmin(datasetId, null); // we know admin is not null, since we are looping over existing datasets //noinspection ConstantConditions admin.upgrade(); LOG.info("Upgraded dataset: {}", spec.getName()); } catch (Exception e) { throw new RuntimeException(e); } } };
@Override public List<?> makeRow(DatasetSpecificationSummary object) { return Lists.newArrayList(object.getName(), object.getType(), object.getDescription()); } }).build();
@Override public List<?> makeRow(DatasetSpecificationSummary object) { return Lists.newArrayList(object.getName(), object.getType(), object.getDescription()); } }).build();
private void upgradeSystemDatasets(ExecutorService executor) throws Exception { Map<String, Future<?>> futures = new HashMap<>(); for (final DatasetSpecificationSummary spec : dsFramework.getInstances(NamespaceId.SYSTEM)) { final DatasetId datasetId = NamespaceId.SYSTEM.dataset(spec.getName()); Runnable runnable = new Runnable() { @Override
private Set<DatasetId> summaryToDatasetIdSet(Collection<DatasetSpecificationSummary> datasetSpecs) { Collection<DatasetId> datasetIds = Collections2.transform(datasetSpecs, input -> NamespaceId.DEFAULT.dataset(input.getName())); return ImmutableSet.copyOf(datasetIds); }
private void report(Map<TableId, LevelDBTableService.TableStats> datasetStat) throws DatasetManagementException { for (Map.Entry<TableId, LevelDBTableService.TableStats> statEntry : datasetStat.entrySet()) { String namespace = statEntry.getKey().getNamespace(); // emit metrics for only user datasets, tables in system namespace are ignored if (NamespaceId.SYSTEM.getNamespace().equals(namespace)) { continue; } String tableName = statEntry.getKey().getTableName(); Collection<DatasetSpecificationSummary> instances = dsFramework.getInstances(new NamespaceId(namespace)); for (DatasetSpecificationSummary spec : instances) { DatasetSpecification specification = dsFramework.getDatasetSpec(new DatasetId(namespace, spec.getName())); if (specification.isParent(tableName)) { MetricsContext collector = metricsService.getContext(ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, namespace, Constants.Metrics.Tag.DATASET, spec.getName())); int sizeInMb = (int) (statEntry.getValue().getDiskSizeBytes() / BYTES_IN_MB); collector.gauge("dataset.size.mb", sizeInMb); break; } } } }
private void report(Map<TableId, LevelDBTableService.TableStats> datasetStat) throws DatasetManagementException { for (Map.Entry<TableId, LevelDBTableService.TableStats> statEntry : datasetStat.entrySet()) { String namespace = statEntry.getKey().getNamespace(); // emit metrics for only user datasets, tables in system namespace are ignored if (NamespaceId.SYSTEM.getNamespace().equals(namespace)) { continue; } String tableName = statEntry.getKey().getTableName(); Collection<DatasetSpecificationSummary> instances = dsFramework.getInstances(new NamespaceId(namespace)); for (DatasetSpecificationSummary spec : instances) { DatasetSpecification specification = dsFramework.getDatasetSpec(new DatasetId(namespace, spec.getName())); if (specification.isParent(tableName)) { MetricsContext collector = metricsService.getContext(ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, namespace, Constants.Metrics.Tag.DATASET, spec.getName())); int sizeInMb = (int) (statEntry.getValue().getDiskSizeBytes() / BYTES_IN_MB); collector.gauge("dataset.size.mb", sizeInMb); break; } } } }
private boolean isUserDataset(DatasetSpecificationSummary specification) { final DefaultDatasetNamespace dsNamespace = new DefaultDatasetNamespace(CConfiguration.create()); return !dsNamespace.contains(specification.getName(), NamespaceId.SYSTEM.getNamespace()); }
private void report(Map<TableId, HBaseTableUtil.TableStats> tableStats) throws IOException { Map<String, String> reverseNamespaceMap = hBaseTableUtil.getHBaseToCDAPNamespaceMap(); for (Map.Entry<TableId, HBaseTableUtil.TableStats> statEntry : tableStats.entrySet()) { String hbaseNamespace = statEntry.getKey().getNamespace(); String cdapNamespace = reverseNamespaceMap.get(hbaseNamespace); // emit metrics for only user datasets, namespaces in system and // tableNames that doesn't start with user are ignored if (NamespaceId.SYSTEM.getNamespace().equals(cdapNamespace)) { continue; } String tableName = statEntry.getKey().getTableName(); try { Collection<DatasetSpecificationSummary> instances = dsFramework.getInstances(new NamespaceId(cdapNamespace)); for (DatasetSpecificationSummary spec : instances) { DatasetSpecification specification = dsFramework.getDatasetSpec(new DatasetId(cdapNamespace, spec.getName())); if (specification.isParent(tableName)) { MetricsContext collector = metricsService.getContext(ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, cdapNamespace, Constants.Metrics.Tag.DATASET, spec.getName())); collector.gauge("dataset.size.mb", statEntry.getValue().getTotalSizeMB()); break; } } } catch (DatasetManagementException | ServiceUnavailableException e) { // No op } } } }
private void report(Map<TableId, HBaseTableUtil.TableStats> tableStats) throws IOException { Map<String, String> reverseNamespaceMap = hBaseTableUtil.getHBaseToCDAPNamespaceMap(); for (Map.Entry<TableId, HBaseTableUtil.TableStats> statEntry : tableStats.entrySet()) { String hbaseNamespace = statEntry.getKey().getNamespace(); String cdapNamespace = reverseNamespaceMap.get(hbaseNamespace); // emit metrics for only user datasets, namespaces in system and // tableNames that doesn't start with user are ignored if (NamespaceId.SYSTEM.getNamespace().equals(cdapNamespace)) { continue; } String tableName = statEntry.getKey().getTableName(); try { Collection<DatasetSpecificationSummary> instances = dsFramework.getInstances(new NamespaceId(cdapNamespace)); for (DatasetSpecificationSummary spec : instances) { DatasetSpecification specification = dsFramework.getDatasetSpec(new DatasetId(cdapNamespace, spec.getName())); if (specification.isParent(tableName)) { MetricsContext collector = metricsService.getContext(ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, cdapNamespace, Constants.Metrics.Tag.DATASET, spec.getName())); collector.gauge("dataset.size.mb", statEntry.getValue().getTotalSizeMB()); break; } } } catch (DatasetManagementException | ServiceUnavailableException e) { // No op } } } }
@Override public void run() { try { List<NamespaceMeta> list = namespaceAdmin.list(); for (NamespaceMeta namespaceMeta : list) { Collection<DatasetSpecificationSummary> specs = datasetFramework.getInstances(namespaceMeta.getNamespaceId(), PROPERTIES); if (specs.isEmpty()) { // avoid fetching run records continue; } Set<String> activeRuns = getActiveRuns(namespaceMeta.getNamespaceId()); for (DatasetSpecificationSummary spec : specs) { deleteLocalDataset(namespaceMeta.getName(), spec.getName(), activeRuns, spec.getProperties()); } } } catch (Throwable t) { LOG.warn("Failed to delete the local datasets.", t); } }
@Override public void run() { try { List<NamespaceMeta> list = namespaceAdmin.list(); for (NamespaceMeta namespaceMeta : list) { Collection<DatasetSpecificationSummary> specs = datasetFramework.getInstances(namespaceMeta.getNamespaceId(), PROPERTIES); if (specs.isEmpty()) { // avoid fetching run records continue; } Set<String> activeRuns = getActiveRuns(namespaceMeta.getNamespaceId()); for (DatasetSpecificationSummary spec : specs) { deleteLocalDataset(namespaceMeta.getName(), spec.getName(), activeRuns, spec.getProperties()); } } } catch (Throwable t) { LOG.warn("Failed to delete the local datasets.", t); } }
private void deleteInstances() throws IOException { // NOTE: intentionally, there's no endpoint to delete all datasets instances currently to prevent bad things // happen by accident for (DatasetSpecificationSummary spec : getInstances().getResponseObject()) { deleteInstance(spec.getName()); } }
@After public void after() throws Exception { // cleanup user data (only user datasets) for (DatasetSpecificationSummary spec : dsFramework.getInstances(DefaultId.NAMESPACE)) { dsFramework.deleteInstance(DefaultId.NAMESPACE.dataset(spec.getName())); } }
@After public void after() throws Throwable { // stop all running programs for (ProgramController controller : runningPrograms) { stopProgram(controller); } // cleanup user data (only user datasets) for (DatasetSpecificationSummary spec : dsFramework.getInstances(DefaultId.NAMESPACE)) { dsFramework.deleteInstance(DefaultId.NAMESPACE.dataset(spec.getName())); } }
private void doClear(NamespaceId namespace, boolean deleteNamespace) throws Exception { // stop all programs in the namespace getProgramClient().stopAll(namespace); if (deleteNamespace) { getNamespaceClient().delete(namespace); return; } // delete all apps in the namespace for (ApplicationRecord app : getApplicationClient().list(namespace)) { getApplicationClient().delete(namespace.app(app.getName(), app.getAppVersion())); } // delete all dataset instances for (DatasetSpecificationSummary datasetSpecSummary : getDatasetClient().list(namespace)) { getDatasetClient().delete(namespace.dataset(datasetSpecSummary.getName())); } ArtifactClient artifactClient = new ArtifactClient(getClientConfig(), getRestClient()); for (ArtifactSummary artifactSummary : artifactClient.list(namespace, ArtifactScope.USER)) { artifactClient.delete(namespace.artifact(artifactSummary.getName(), artifactSummary.getVersion())); } assertIsClear(namespace); }
ObjectResponse<List<DatasetSpecificationSummary>> instances = getInstances(); Assert.assertEquals(1, instances.getResponseObject().size()); Assert.assertEquals("myTable2", instances.getResponseObject().get(0).getName()); Assert.assertEquals(HttpStatus.SC_OK, createInstance("myTable1", "table", DatasetProperties.EMPTY).getResponseCode());