/** * Creates the metrics tables and kafka-meta table using the factory {@link DefaultMetricDatasetFactory} * <p> * It is primarily used by upgrade and data-migration tool. * * @param factory : metrics dataset factory */ public static void setupDatasets(DefaultMetricDatasetFactory factory) { // adding all fact tables factory.getOrCreateFactTable(Constants.Metrics.SECOND_RESOLUTION); factory.getOrCreateFactTable(Constants.Metrics.MINUTE_RESOLUTION); factory.getOrCreateFactTable(Constants.Metrics.HOUR_RESOLUTION); factory.getOrCreateFactTable(Integer.MAX_VALUE); // adding kafka consumer meta factory.createConsumerMeta(); }
@Inject public DefaultMetricDatasetFactory(final CConfiguration cConf, DatasetFramework dsFramework) { this.cConf = cConf; this.dsFramework = dsFramework; this.entityTable = Suppliers.memoize(() -> { String tableName = cConf.get(Constants.Metrics.ENTITY_TABLE_NAME, Constants.Metrics.DEFAULT_ENTITY_TABLE_NAME); return new EntityTable(getOrCreateMetricsTable(tableName, DatasetProperties.EMPTY)); }); }
@Override public FactTable getOrCreateFactTable(int resolution) { String v3TableName = cConf.get(Constants.Metrics.METRICS_TABLE_PREFIX, Constants.Metrics.DEFAULT_METRIC_V3_TABLE_PREFIX) + ".ts." + resolution; TableProperties.Builder props = TableProperties.builder(); // don't add TTL for MAX_RESOLUTION table. CDAP-1626 if (resolution != Integer.MAX_VALUE) { int ttl = cConf.getInt(Constants.Metrics.RETENTION_SECONDS + resolution + Constants.Metrics.RETENTION_SECONDS_SUFFIX); if (ttl > 0) { props.setTTL(ttl); } } // for efficient counters props.setReadlessIncrementSupport(true); // configuring pre-splits props.add(HBaseTableAdmin.PROPERTY_SPLITS, GSON.toJson(FactTable.getSplits(DefaultMetricStore.AGGREGATIONS.size()))); // Disable auto split props.add(HBaseTableAdmin.SPLIT_POLICY, cConf.get(Constants.Metrics.METRICS_TABLE_HBASE_SPLIT_POLICY)); MetricsTable table = getOrCreateResolutionMetricsTable(v3TableName, props, resolution); return new FactTable(table, entityTable.get(), resolution, getRollTime(resolution)); }
DefaultMetricDatasetFactory factory = new DefaultMetricDatasetFactory(cConf, datasetFramework); DefaultMetricDatasetFactory.setupDatasets(cConf, factory);
protected MetricsTable getOrCreateResolutionMetricsTable(String tableName, TableProperties.Builder props, int resolution) { try { props.add(HBaseTableAdmin.PROPERTY_SPLITS, GSON.toJson(getMetricsTableSplits(cConf.getInt(Constants.Metrics.METRICS_HBASE_TABLE_SPLITS)))); props.add(Constants.Metrics.METRICS_HBASE_TABLE_SPLITS, cConf.getInt(Constants.Metrics.METRICS_HBASE_TABLE_SPLITS)); DatasetId tableId = NamespaceId.SYSTEM.dataset(tableName); return DatasetsUtil.getOrCreateDataset(dsFramework, tableId, MetricsTable.class.getName(), props.build(), null); } catch (Exception e) { throw Throwables.propagate(e); } }
protected MetricsTable getOrCreateResolutionMetricsTable(String v3TableName, TableProperties.Builder props, int resolution) { try { String v2TableName = cConf.get(Constants.Metrics.METRICS_TABLE_PREFIX, Constants.Metrics.DEFAULT_METRIC_TABLE_PREFIX + ".ts." + resolution); // metrics tables are in the system namespace DatasetId v2TableId = NamespaceId.SYSTEM.dataset(v2TableName); MetricsTable v2Table = dsFramework.getDataset(v2TableId, ImmutableMap.of(), null); props.add(HBaseTableAdmin.PROPERTY_SPLITS, GSON.toJson(getV3MetricsTableSplits(cConf.getInt(Constants.Metrics.METRICS_HBASE_TABLE_SPLITS)))); props.add(Constants.Metrics.METRICS_HBASE_TABLE_SPLITS, cConf.getInt(Constants.Metrics.METRICS_HBASE_TABLE_SPLITS)); DatasetId v3TableId = NamespaceId.SYSTEM.dataset(v3TableName); MetricsTable v3Table = DatasetsUtil.getOrCreateDataset(dsFramework, v3TableId, MetricsTable.class.getName(), props.build(), null); if (v2Table != null) { // the cluster is upgraded, so use Combined Metrics Table return new CombinedHBaseMetricsTable(v2Table, v3Table, resolution, cConf, dsFramework); } return v3Table; } catch (Exception e) { throw Throwables.propagate(e); } }
@Override public FactTable getOrCreateFactTable(int resolution) { String tableName = cConf.get(Constants.Metrics.METRICS_TABLE_PREFIX, Constants.Metrics.DEFAULT_METRIC_TABLE_PREFIX) + ".ts." + resolution; TableProperties.Builder props = TableProperties.builder(); // don't add TTL for MAX_RESOLUTION table. CDAP-1626 if (resolution != Integer.MAX_VALUE) { int ttl = resolution < 60 ? cConf.getInt(Constants.Metrics.MINIMUM_RESOLUTION_RETENTION_SECONDS) : cConf.getInt(Constants.Metrics.RETENTION_SECONDS + resolution + Constants.Metrics.RETENTION_SECONDS_SUFFIX); if (ttl > 0) { props.setTTL(ttl); } } // for efficient counters props.setReadlessIncrementSupport(true); // configuring pre-splits props.add(HBaseTableAdmin.PROPERTY_SPLITS, GSON.toJson(FactTable.getSplits(DefaultMetricStore.AGGREGATIONS.size()))); // Disable auto split props.add(HBaseTableAdmin.SPLIT_POLICY, cConf.get(Constants.Metrics.METRICS_TABLE_HBASE_SPLIT_POLICY)); MetricsTable table = getOrCreateResolutionMetricsTable(tableName, props, resolution); return new FactTable(table, entityTable.get(), resolution, getRollTime(resolution)); }
/** * Creates the metrics tables and metrics meta table using the factory {@link DefaultMetricDatasetFactory} * <p> * It is primarily used by upgrade and data-migration tool. * * @param cConf configuration * @param factory metrics dataset factory */ public static void setupDatasets(CConfiguration cConf, DefaultMetricDatasetFactory factory) { // adding all fact tables int minimumResolution = cConf.getInt(Constants.Metrics.METRICS_MINIMUM_RESOLUTION_SECONDS); if (minimumResolution < Constants.Metrics.MINUTE_RESOLUTION) { factory.getOrCreateFactTable(minimumResolution); } factory.getOrCreateFactTable(Constants.Metrics.MINUTE_RESOLUTION); factory.getOrCreateFactTable(Constants.Metrics.HOUR_RESOLUTION); factory.getOrCreateFactTable(Integer.MAX_VALUE); // adding consumer meta factory.createConsumerMeta(); }
@Override public MetricsConsumerMetaTable createConsumerMeta() { String tableName = cConf.get(Constants.Metrics.KAFKA_META_TABLE); MetricsTable table = getOrCreateMetricsTable(tableName, DatasetProperties.EMPTY); return new MetricsConsumerMetaTable(table); }
@Override public EntityTable get() { String tableName = cConf.get(Constants.Metrics.ENTITY_TABLE_NAME, Constants.Metrics.DEFAULT_ENTITY_TABLE_NAME); return new EntityTable(getOrCreateMetricsTable(tableName, DatasetProperties.EMPTY)); } });
@Override public MetricsConsumerMetaTable createConsumerMeta() { String tableName = cConf.get(Constants.Metrics.METRICS_META_TABLE); MetricsTable table = getOrCreateMetricsTable(tableName, DatasetProperties.EMPTY); return new MetricsConsumerMetaTable(table); }