/** * Returns true if the datasetName is the name of a non-composite dataset represented by this spec. * That is, it is represented by one of the leaf-nodes of this dataset spec. * @param datasetName the name of a dataset * @return <code>true</code> if the datasetName is represented by the dataset spec; * <code>false</code> otherwise */ public boolean isParent(String datasetName) { return isParent(datasetName, this); }
private boolean isParent(String datasetName, DatasetSpecification specification) { if (datasetName == null) { return false; } if (specification.getSpecifications().size() == 0 && specification.getName().equals(datasetName)) { return true; } if (datasetName.startsWith(specification.getName())) { return specification.getSpecifications().values().stream().anyMatch(spec -> isParent(datasetName, spec)); } return false; }
private void report(Map<TableId, LevelDBTableService.TableStats> datasetStat) throws DatasetManagementException { for (Map.Entry<TableId, LevelDBTableService.TableStats> statEntry : datasetStat.entrySet()) { String namespace = statEntry.getKey().getNamespace(); // emit metrics for only user datasets, tables in system namespace are ignored if (NamespaceId.SYSTEM.getNamespace().equals(namespace)) { continue; } String tableName = statEntry.getKey().getTableName(); Collection<DatasetSpecificationSummary> instances = dsFramework.getInstances(new NamespaceId(namespace)); for (DatasetSpecificationSummary spec : instances) { DatasetSpecification specification = dsFramework.getDatasetSpec(new DatasetId(namespace, spec.getName())); if (specification.isParent(tableName)) { MetricsContext collector = metricsService.getContext(ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, namespace, Constants.Metrics.Tag.DATASET, spec.getName())); int sizeInMb = (int) (statEntry.getValue().getDiskSizeBytes() / BYTES_IN_MB); collector.gauge("dataset.size.mb", sizeInMb); break; } } } }
private void report(Map<TableId, LevelDBTableService.TableStats> datasetStat) throws DatasetManagementException { for (Map.Entry<TableId, LevelDBTableService.TableStats> statEntry : datasetStat.entrySet()) { String namespace = statEntry.getKey().getNamespace(); // emit metrics for only user datasets, tables in system namespace are ignored if (NamespaceId.SYSTEM.getNamespace().equals(namespace)) { continue; } String tableName = statEntry.getKey().getTableName(); Collection<DatasetSpecificationSummary> instances = dsFramework.getInstances(new NamespaceId(namespace)); for (DatasetSpecificationSummary spec : instances) { DatasetSpecification specification = dsFramework.getDatasetSpec(new DatasetId(namespace, spec.getName())); if (specification.isParent(tableName)) { MetricsContext collector = metricsService.getContext(ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, namespace, Constants.Metrics.Tag.DATASET, spec.getName())); int sizeInMb = (int) (statEntry.getValue().getDiskSizeBytes() / BYTES_IN_MB); collector.gauge("dataset.size.mb", sizeInMb); break; } } } }
private void report(Map<TableId, HBaseTableUtil.TableStats> tableStats) throws IOException { Map<String, String> reverseNamespaceMap = hBaseTableUtil.getHBaseToCDAPNamespaceMap(); for (Map.Entry<TableId, HBaseTableUtil.TableStats> statEntry : tableStats.entrySet()) { String hbaseNamespace = statEntry.getKey().getNamespace(); String cdapNamespace = reverseNamespaceMap.get(hbaseNamespace); // emit metrics for only user datasets, namespaces in system and // tableNames that doesn't start with user are ignored if (NamespaceId.SYSTEM.getNamespace().equals(cdapNamespace)) { continue; } String tableName = statEntry.getKey().getTableName(); try { Collection<DatasetSpecificationSummary> instances = dsFramework.getInstances(new NamespaceId(cdapNamespace)); for (DatasetSpecificationSummary spec : instances) { DatasetSpecification specification = dsFramework.getDatasetSpec(new DatasetId(cdapNamespace, spec.getName())); if (specification.isParent(tableName)) { MetricsContext collector = metricsService.getContext(ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, cdapNamespace, Constants.Metrics.Tag.DATASET, spec.getName())); collector.gauge("dataset.size.mb", statEntry.getValue().getTotalSizeMB()); break; } } } catch (DatasetManagementException | ServiceUnavailableException e) { // No op } } } }
private void report(Map<TableId, HBaseTableUtil.TableStats> tableStats) throws IOException { Map<String, String> reverseNamespaceMap = hBaseTableUtil.getHBaseToCDAPNamespaceMap(); for (Map.Entry<TableId, HBaseTableUtil.TableStats> statEntry : tableStats.entrySet()) { String hbaseNamespace = statEntry.getKey().getNamespace(); String cdapNamespace = reverseNamespaceMap.get(hbaseNamespace); // emit metrics for only user datasets, namespaces in system and // tableNames that doesn't start with user are ignored if (NamespaceId.SYSTEM.getNamespace().equals(cdapNamespace)) { continue; } String tableName = statEntry.getKey().getTableName(); try { Collection<DatasetSpecificationSummary> instances = dsFramework.getInstances(new NamespaceId(cdapNamespace)); for (DatasetSpecificationSummary spec : instances) { DatasetSpecification specification = dsFramework.getDatasetSpec(new DatasetId(cdapNamespace, spec.getName())); if (specification.isParent(tableName)) { MetricsContext collector = metricsService.getContext(ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, cdapNamespace, Constants.Metrics.Tag.DATASET, spec.getName())); collector.gauge("dataset.size.mb", statEntry.getValue().getTotalSizeMB()); break; } } } catch (DatasetManagementException | ServiceUnavailableException e) { // No op } } } }
@Test public void testParent() { DatasetSpecification firstSpec = DatasetSpecification.builder("inner", "table") .build(); DatasetSpecification secondSpec = DatasetSpecification.builder("kv", "kvtable") .datasets(firstSpec) .build(); DatasetSpecification objSpec1 = DatasetSpecification.builder("obj", "objects") .datasets(secondSpec) .build(); DatasetSpecification objSpec2 = DatasetSpecification.builder("objt", "objects") .datasets(secondSpec) .build(); DatasetSpecification fourthSpec = DatasetSpecification.builder("history", "store") .datasets(objSpec1, objSpec2) .build(); Assert.assertTrue(fourthSpec.isParent("history.obj.kv.inner")); Assert.assertTrue(objSpec2.isParent("objt.kv.inner")); Assert.assertTrue(fourthSpec.isParent("history.objt.kv.inner")); Assert.assertFalse(objSpec1.isParent("objt.kv.inner")); Assert.assertFalse(fourthSpec.isParent("history.obj.kv.outer")); Assert.assertFalse(fourthSpec.isParent("history.obj.kv.inner.outer")); Assert.assertFalse(fourthSpec.isParent("history.obj.kv.inner.inner")); Assert.assertFalse(fourthSpec.isParent("obj.kv.inner")); } }