@BeforeClass public static void setup() throws Exception { instanceZKMetadata = getInstanceZKMetadata(); realtimeSegmentZKMetadata = getRealtimeSegmentZKMetadata(); tableDataManagerConfig = getTableDataManagerConfig(); Map<String, String> streamConfigs = new HashMap<>(); streamConfigs.put("streamType", "kafka"); streamConfigs.put("stream.kafka.consumer.type", "highLevel"); streamConfigs.put("stream.kafka.topic.name", "kafkaTopic"); streamConfigs .put("stream.kafka.decoder.class.name", "org.apache.pinot.core.realtime.impl.kafka.KafkaAvroMessageDecoder"); streamConfigs.put("stream.kafka.hlc.zk.connect.string", "localhost:1111/zkConnect"); streamConfigs.put("stream.kafka.decoder.prop.schema.registry.rest.url", "http://localhost:2222/schemaRegistry"); tableConfig = new TableConfig.Builder(CommonConstants.Helix.TableType.REALTIME).setTableName("mirror") .setStreamConfigs(streamConfigs).build(); }
.put(StreamConfigProperties.constructStreamProperty(streamType, StreamConfigProperties.STREAM_DECODER_CLASS), decoderClass); tableConfigBuilder.setStreamConfigs(streamConfigs);
private void setupRealtimeTable() throws IOException { // Set up the realtime table. Map<String, String> streamConfigs = new HashMap<>(); streamConfigs.put("streamType", "kafka"); streamConfigs.put("stream.kafka.consumer.type", "highLevel"); streamConfigs.put("stream.kafka.topic.name", "kafkaTopic"); streamConfigs .put("stream.kafka.decoder.class.name", "org.apache.pinot.core.realtime.impl.kafka.KafkaAvroMessageDecoder"); streamConfigs.put("stream.kafka.hlc.zk.connect.string", "localhost:1111/zkConnect"); streamConfigs.put("stream.kafka.decoder.prop.schema.registry.rest.url", "http://localhost:2222/schemaRegistry"); TableConfig realtimeTimeConfig = new TableConfig.Builder(CommonConstants.Helix.TableType.REALTIME).setTableName(RAW_DINING_TABLE_NAME) .setTimeColumnName("timeColumn").setTimeType("DAYS"). setStreamConfigs(streamConfigs).build(); Schema schema = new Schema(); schema.setSchemaName(RAW_DINING_TABLE_NAME); _pinotResourceManager.addOrUpdateSchema(schema); // Fake an PinotLLCRealtimeSegmentManager instance: required for a realtime table creation. PinotLLCRealtimeSegmentManager .create(_pinotResourceManager, new ControllerConf(), new ControllerMetrics(new MetricsRegistry())); _pinotResourceManager.addTable(realtimeTimeConfig); _helixBrokerStarter.getHelixExternalViewBasedRouting() .markDataResourceOnline(realtimeTimeConfig, null, new ArrayList<InstanceConfig>()); }
.setServerTenant(serverTenant).setLoadMode(loadMode).setSortedColumn(sortedColumn) .setInvertedIndexColumns(invertedIndexColumns).setBloomFilterColumns(bloomFilterColumns) .setNoDictionaryColumns(noDictionaryColumns).setStreamConfigs(streamConfigs).setTaskConfig(taskConfig).build();
_realtimeBuilder.setTableName(REALTIME_TABLE_NAME).setTimeColumnName("timeColumn").setTimeType("DAYS") .setRetentionTimeUnit("DAYS").setRetentionTimeValue("5").setSchemaName(REALTIME_TABLE_NAME) .setStreamConfigs(streamConfigs);