public static Configuration getDefaultBrokerConf() { Configuration brokerConf = new PropertiesConfiguration(); // config based routing brokerConf.addProperty("pinot.broker.transport.routingMode", "HELIX"); brokerConf.addProperty("pinot.broker.routing.table.builder.default.offline.class", "balanced"); brokerConf.addProperty("pinot.broker.routing.table.builder.default.offline.numOfRoutingTables", "10"); brokerConf.addProperty("pinot.broker.routing.table.builder.tables", ""); //client properties brokerConf.addProperty("pinot.broker.client.queryPort", "8099"); // [PINOT-2435] setting to 0 so it doesn't disconnect from zk brokerConf.addProperty("pinot.broker.helix.flappingTimeWindowMs", "0"); return brokerConf; }
/** * This method is for reference purpose only. */ public static HelixServerStarter startDefault() throws Exception { Configuration configuration = new PropertiesConfiguration(); int port = 8003; configuration.addProperty(CommonConstants.Helix.KEY_OF_SERVER_NETTY_PORT, port); configuration.addProperty("pinot.server.instance.dataDir", "/tmp/PinotServer/test" + port + "/index"); configuration.addProperty("pinot.server.instance.segmentTarDir", "/tmp/PinotServer/test" + port + "/segmentTar"); return new HelixServerStarter("quickstart", "localhost:2191", configuration); }
public static Configuration loadDefaultServerConf() { Configuration serverConf = new PropertiesConfiguration(); serverConf.addProperty(CommonConstants.Server.CONFIG_OF_INSTANCE_DATA_DIR, CommonConstants.Server.DEFAULT_INSTANCE_DATA_DIR); serverConf.addProperty(CommonConstants.Server.CONFIG_OF_INSTANCE_SEGMENT_TAR_DIR, CommonConstants.Server.DEFAULT_INSTANCE_SEGMENT_TAR_DIR); .addProperty(CommonConstants.Server.CONFIG_OF_INSTANCE_READ_MODE, CommonConstants.Server.DEFAULT_READ_MODE); serverConf.addProperty(CommonConstants.Server.CONFIG_OF_INSTANCE_DATA_MANAGER_CLASS, CommonConstants.Server.DEFAULT_DATA_MANAGER_CLASS); serverConf.addProperty(CommonConstants.Server.CONFIG_OF_QUERY_EXECUTOR_CLASS, CommonConstants.Server.DEFAULT_QUERY_EXECUTOR_CLASS); serverConf.addProperty(CommonConstants.Server.CONFIG_OF_QUERY_EXECUTOR_PRUNER_CLASS, "DataSchemaSegmentPruner,ColumnValueSegmentPruner,ValidSegmentPruner,PartitionSegmentPruner"); serverConf.addProperty("pinot.server.query.executor.pruner.DataSchemaSegmentPruner.id", "0"); serverConf.addProperty("pinot.server.query.executor.pruner.ColumnValueSegmentPruner.id", "1"); serverConf.addProperty("pinot.server.query.executor.pruner.ValidSegmentPruner.id", "2"); serverConf.addProperty("pinot.server.query.executor.pruner.PartitionSegmentPruner.id", "3"); serverConf.addProperty(CommonConstants.Helix.CONFIG_OF_HELIX_FLAPPING_TIMEWINDOW_MS, CommonConstants.Helix.DEFAULT_HELIX_FLAPPING_TIMEWINDOW_MS); serverConf.addProperty(CommonConstants.Server.CONFIG_OF_REQUEST_HANDLER_FACTORY_CLASS, CommonConstants.Server.DEFAULT_REQUEST_HANDLER_FACTORY_CLASS); .addProperty(CommonConstants.Server.CONFIG_OF_NETTY_PORT, CommonConstants.Helix.DEFAULT_SERVER_NETTY_PORT);
public static HelixBrokerStarter startDefault() throws Exception { Configuration configuration = new PropertiesConfiguration(); int port = 5001; configuration.addProperty(CommonConstants.Helix.KEY_OF_BROKER_QUERY_PORT, port); configuration.addProperty("pinot.broker.timeoutMs", 500 * 1000L); final HelixBrokerStarter pinotHelixBrokerStarter = new HelixBrokerStarter(null, "quickstart", "localhost:2122", configuration); return pinotHelixBrokerStarter; }
public static TableDataManagerConfig getDefaultHelixTableDataManagerConfig( @Nonnull InstanceDataManagerConfig instanceDataManagerConfig, @Nonnull String tableNameWithType) { Configuration defaultConfig = new PropertiesConfiguration(); defaultConfig.addProperty(TABLE_DATA_MANAGER_NAME, tableNameWithType); defaultConfig.addProperty(TABLE_DATA_MANAGER_DATA_DIRECTORY, instanceDataManagerConfig.getInstanceDataDir() + "/" + tableNameWithType); defaultConfig.addProperty(TABLE_DATA_MANAGER_CONSUMER_DIRECTORY, instanceDataManagerConfig.getConsumerDir()); TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableNameWithType); Preconditions.checkNotNull(tableType); defaultConfig.addProperty(TABLE_DATA_MANAGER_TYPE, tableType.name()); return new TableDataManagerConfig(defaultConfig); }
configuration.addProperty(CommonConstants.Helix.KEY_OF_SERVER_NETTY_HOST, _serverHost); configuration.addProperty(CommonConstants.Helix.KEY_OF_SERVER_NETTY_PORT, _serverPort); configuration.addProperty(CommonConstants.Server.CONFIG_OF_ADMIN_API_PORT, _serverAdminPort); configuration.addProperty("pinot.server.instance.dataDir", _dataDir + _serverPort + "/index"); configuration.addProperty("pinot.server.instance.segmentTarDir", _segmentDir + _serverPort + "/segmentTar");
private void startServer() throws Exception { if (!_conf.shouldStartServer()) { LOGGER.info("Skipping start server step. Assumes server is already started."); return; } Configuration serverConfiguration = new PropertiesConfiguration(); serverConfiguration.addProperty(CommonConstants.Server.CONFIG_OF_INSTANCE_DATA_DIR, _serverInstanceDataDir); serverConfiguration .addProperty(CommonConstants.Server.CONFIG_OF_INSTANCE_SEGMENT_TAR_DIR, _serverInstanceSegmentTarDir); if (_segmentFormatVersion != null) { serverConfiguration.setProperty(CommonConstants.Server.CONFIG_OF_SEGMENT_FORMAT_VERSION, _segmentFormatVersion); } serverConfiguration.setProperty(CommonConstants.Helix.Instance.INSTANCE_ID_KEY, _serverInstanceName); LOGGER.info("Starting server instance: {}", _serverInstanceName); new HelixServerStarter(_clusterName, _zkAddress, serverConfiguration); }
private static TableDataManagerConfig getTableDataManagerConfig() throws ConfigurationException { String tableName = "testTable_R"; Configuration defaultConfig = new PropertiesConfiguration(); defaultConfig.addProperty(TABLE_DATA_MANAGER_NAME, tableName); String dataDir = "/tmp/" + tableName; defaultConfig.addProperty(TABLE_DATA_MANAGER_DATA_DIRECTORY, dataDir); defaultConfig.addProperty(READ_MODE, ReadMode.heap.toString()); defaultConfig.addProperty(TABLE_DATA_MANAGER_NUM_QUERY_EXECUTOR_THREADS, 20); TableDataManagerConfig tableDataManagerConfig = new TableDataManagerConfig(defaultConfig); defaultConfig.addProperty(TABLE_DATA_MANAGER_TYPE, "realtime"); return tableDataManagerConfig; }
@Override public boolean execute() throws Exception { try { Configuration configuration = readConfigFromFile(_configFileName); if (configuration == null) { if (_configFileName != null) { LOGGER.error("Error: Unable to find file {}.", _configFileName); return false; } configuration = new PropertiesConfiguration(); configuration.addProperty(CommonConstants.Helix.KEY_OF_BROKER_QUERY_PORT, _brokerPort); configuration.setProperty("pinot.broker.routing.table.builder.class", "random"); } LOGGER.info("Executing command: " + toString()); final HelixBrokerStarter pinotHelixBrokerStarter = new HelixBrokerStarter(_brokerHost, _clusterName, _zkAddress, configuration); String pidFile = ".pinotAdminBroker-" + String.valueOf(System.currentTimeMillis()) + ".pid"; savePID(System.getProperty("java.io.tmpdir") + File.separator + pidFile); return true; } catch (Exception e) { LOGGER.error("Caught exception while starting broker, exiting", e); System.exit(-1); return false; } } }
@Test public void testCustomizedSegmentFetcherFactory() { Configuration config = new PropertiesConfiguration(); config.addProperty("class.file", LocalPinotFS.class.getName()); config.addProperty("class.test", TestPinotFS.class.getName()); PinotFSFactory.init(config); PinotFS testPinotFS = PinotFSFactory.create("test"); Assert.assertTrue(testPinotFS instanceof TestPinotFS); Assert.assertEquals(((TestPinotFS) testPinotFS).getInitCalled(), 1); Assert.assertTrue(PinotFSFactory.create("file") instanceof LocalPinotFS); }
rebalanceUserConfig.addProperty(RebalanceUserConfigConstants.DRYRUN, dryRun); rebalanceUserConfig.addProperty(RebalanceUserConfigConstants.INCLUDE_CONSUMING, includeConsuming); rebalanceUserConfig.addProperty(RebalanceUserConfigConstants.DOWNTIME, downtime);
@Test public void testCustomizedSegmentFetcherFactory() throws Exception { Configuration config = new PropertiesConfiguration(); config.addProperty("something", "abc"); config.addProperty("protocols", Arrays.asList("http", "https", "test")); config.addProperty("http.other", "some config"); config.addProperty("https.class", NoOpFetcher.class.getName()); config.addProperty("test.class", TestSegmentFetcher.class.getName()); _segmentFetcherFactory.init(config); Assert.assertTrue(_segmentFetcherFactory.containsProtocol("http")); Assert.assertTrue(_segmentFetcherFactory.containsProtocol("https")); Assert.assertTrue(_segmentFetcherFactory.containsProtocol("test")); SegmentFetcher testSegmentFetcher = _segmentFetcherFactory.getSegmentFetcherBasedOnURI("test://something"); Assert.assertTrue(testSegmentFetcher instanceof TestSegmentFetcher); Assert.assertEquals(((TestSegmentFetcher) testSegmentFetcher).getInitCalled(), 1); }
public static Configuration getDefaultServerConfiguration() { Configuration configuration = DefaultHelixStarterServerConfig.loadDefaultServerConf(); configuration.setProperty(Helix.KEY_OF_SERVER_NETTY_HOST, LOCAL_HOST); configuration.setProperty(Server.CONFIG_OF_SEGMENT_FORMAT_VERSION, "v3"); configuration.addProperty(Server.CONFIG_OF_ENABLE_DEFAULT_COLUMNS, true); configuration.setProperty(Server.CONFIG_OF_ENABLE_SHUTDOWN_DELAY, false); return configuration; }
_helixAdmin = _pinotResourceManager.getHelixAdmin(); _pinotHelixBrokerProperties.addProperty(CommonConstants.Helix.KEY_OF_BROKER_QUERY_PORT, 8943); _pinotHelixBrokerProperties .addProperty(CommonConstants.Broker.CONFIG_OF_BROKER_REFRESH_TIMEBOUNDARY_INFO_SLEEP_INTERVAL, 100L); _helixBrokerStarter = new HelixBrokerStarter(HELIX_CLUSTER_NAME, ZkStarter.DEFAULT_ZK_STR, _pinotHelixBrokerProperties);
rebalanceUserConfig.addProperty(RebalanceUserConfigConstants.INCLUDE_CONSUMING, false); partitionAssignment = rebalanceSegmentsStrategy.rebalancePartitionAssignment(idealState, tableConfig, rebalanceUserConfig);
.getInt(CommonConstants.Helix.KEY_OF_SERVER_NETTY_PORT, CommonConstants.Helix.DEFAULT_SERVER_NETTY_PORT); _instanceId = CommonConstants.Helix.PREFIX_OF_SERVER_INSTANCE + host + "_" + port; _helixServerConfig.addProperty(CommonConstants.Server.CONFIG_OF_INSTANCE_ID, _instanceId);
.getInt(CommonConstants.Helix.KEY_OF_BROKER_QUERY_PORT, CommonConstants.Helix.DEFAULT_BROKER_QUERY_PORT)); _pinotHelixProperties.addProperty(CommonConstants.Broker.CONFIG_OF_BROKER_ID, brokerId); setupHelixSystemProperties();
rebalanceUserConfig.addProperty(RebalanceUserConfigConstants.DRYRUN, true); rebalanceUserConfig.addProperty(RebalanceUserConfigConstants.INCLUDE_CONSUMING, false); rebalanceUserConfig.setProperty(RebalanceUserConfigConstants.DOWNTIME, true);
rebalanceUserConfig.addProperty(RebalanceUserConfigConstants.DRYRUN, true); rebalanceUserConfig.addProperty(RebalanceUserConfigConstants.INCLUDE_CONSUMING, true); consumingInstances = getConsumingInstanceList(nConsumingInstances); setPartitionAssignment(newPartitionAssignment, targetNumReplicas, consumingInstances); rebalanceUserConfig.addProperty(RebalanceUserConfigConstants.INCLUDE_CONSUMING, false); testRebalanceRealtime(rebalancedIdealState, tableConfig, rebalanceUserConfig, newPartitionAssignment, targetNumReplicas, nCompletedSegments, nConsumingSegments, completedInstances, consumingInstances);
public static void setProperty(String key, Object value) { // ensure have instance DynamicPropertyFactory.getInstance(); ConcurrentCompositeConfiguration config = (ConcurrentCompositeConfiguration) DynamicPropertyFactory .getBackingConfigurationSource(); config.getConfiguration(0).addProperty(key, value); }