public HiveProducerRecord parseToHiveProducerRecord(String tableName, Map<String, String> partitionKVs, Map<String, Object> rawValue) throws Exception { Pair<String, String> tableNameSplits = ActiveReservoirReporter.getTableNameSplits(tableName); List<FieldSchema> fields = tableFieldSchemaCache.get(tableNameSplits).getSecond(); List<Object> columnValues = Lists.newArrayListWithExpectedSize(fields.size()); for (FieldSchema fieldSchema : fields) { columnValues.add(rawValue.get(fieldSchema.getName().toUpperCase(Locale.ROOT))); } return new HiveProducerRecord(tableNameSplits.getFirst(), tableNameSplits.getSecond(), partitionKVs, columnValues); }
private void init() { if (KylinConfig.getInstanceFromEnv().isKylinMetricsMonitorEnabled()) { logger.info("Kylin metrics monitor is enabled."); int nameIdx = 0; for (ActiveReservoir activeReservoir : sourceReporterBindProps.keySet()) { String registerName = MetricsSystem.name(MetricsManager.class, "-" + nameIdx + "-" + activeReservoir.toString()); activeReservoirPointers.add(registerName); List<Pair<Class<? extends ActiveReservoirReporter>, Properties>> reportProps = sourceReporterBindProps .get(activeReservoir); for (Pair<Class<? extends ActiveReservoirReporter>, Properties> subEntry : reportProps) { try { Method method = subEntry.getFirst().getMethod(METHOD_FOR_REGISTRY, ActiveReservoir.class); ((ReporterBuilder) method.invoke(null, activeReservoir)).setConfig(subEntry.getSecond()).build() .start(); } catch (Exception e) { logger.warn("Cannot initialize ActiveReservoirReporter: Builder class - " + subEntry.getFirst() + ", Properties - " + subEntry.getSecond()); } } Metrics.register(registerName, activeReservoir); } Preconditions.checkArgument(activeReservoirPointers.size() == sourceReporterBindProps.keySet().size(), "Duplicate register names exist!!!"); } else { logger.info("Kylin metrics monitor is not enabled"); } }
public static TableDesc generateKylinTable(KylinConfig kylinConfig, SinkTool sinkTool, String subject, List<Pair<String, String>> columns) { TableDesc kylinTable = new TableDesc(); Pair<String, String> tableNameSplits = ActiveReservoirReporter .getTableNameSplits(sinkTool.getTableNameForMetrics(subject)); kylinTable.setUuid(RandomUtil.randomUUID().toString()); kylinTable.setDatabase(tableNameSplits.getFirst()); kylinTable.setName(tableNameSplits.getSecond()); kylinTable.setTableType(null); kylinTable.setLastModified(0L); kylinTable.setSourceType(sinkTool.getSourceType()); ColumnDesc[] columnDescs = new ColumnDesc[columns.size()]; for (int i = 0; i < columns.size(); i++) { columnDescs[i] = new ColumnDesc(); Pair<String, String> entry = columns.get(i); columnDescs[i].setId(Integer.toString(i + 1)); columnDescs[i].setName(entry.getFirst()); columnDescs[i].setDatatype(entry.getSecond()); } kylinTable.setColumns(columnDescs); kylinTable.init(kylinConfig, MetricsManager.SYSTEM_PROJECT); return kylinTable; } }
private void init() { if (KylinConfig.getInstanceFromEnv().isKylinMetricsMonitorEnabled()) { logger.info("Kylin metrics monitor is enabled."); int nameIdx = 0; for (ActiveReservoir activeReservoir : sourceReporterBindProps.keySet()) { String registerName = MetricsSystem.name(MetricsManager.class, "-" + nameIdx + "-" + activeReservoir.toString()); activeReservoirPointers.add(registerName); List<Pair<Class<? extends ActiveReservoirReporter>, Properties>> reportProps = sourceReporterBindProps .get(activeReservoir); for (Pair<Class<? extends ActiveReservoirReporter>, Properties> subEntry : reportProps) { try { Method method = subEntry.getFirst().getMethod(METHOD_FOR_REGISTRY, ActiveReservoir.class); ((ReporterBuilder) method.invoke(null, activeReservoir)).setConfig(subEntry.getSecond()).build() .start(); } catch (Exception e) { logger.warn("Cannot initialize ActiveReservoirReporter: Builder class - " + subEntry.getFirst() + ", Properties - " + subEntry.getSecond()); } } Metrics.register(registerName, activeReservoir); } Preconditions.checkArgument(activeReservoirPointers.size() == sourceReporterBindProps.keySet().size(), "Duplicate register names exist!!!"); } else { logger.info("Kylin metrics monitor is not enabled"); } }
public static TableDesc generateKylinTable(KylinConfig kylinConfig, SinkTool sinkTool, String subject, List<Pair<String, String>> columns) { TableDesc kylinTable = new TableDesc(); Pair<String, String> tableNameSplits = ActiveReservoirReporter .getTableNameSplits(sinkTool.getTableNameForMetrics(subject)); kylinTable.setUuid(RandomUtil.randomUUID().toString()); kylinTable.setDatabase(tableNameSplits.getFirst()); kylinTable.setName(tableNameSplits.getSecond()); kylinTable.setTableType(null); kylinTable.setLastModified(0L); kylinTable.setSourceType(sinkTool.getSourceType()); ColumnDesc[] columnDescs = new ColumnDesc[columns.size()]; for (int i = 0; i < columns.size(); i++) { columnDescs[i] = new ColumnDesc(); Pair<String, String> entry = columns.get(i); columnDescs[i].setId(Integer.toString(i + 1)); columnDescs[i].setName(entry.getFirst()); columnDescs[i].setDatatype(entry.getSecond()); } kylinTable.setColumns(columnDescs); kylinTable.init(kylinConfig, MetricsManager.SYSTEM_PROJECT); return kylinTable; } }