private boolean updateIndexConfig(String tableName, TableConfig tableConfig) throws Exception { String request = ControllerRequestURLBuilder.baseUrl("http://" + _controllerAddress).forTableUpdateIndexingConfigs(tableName); HttpURLConnection httpURLConnection = (HttpURLConnection) new URL(request).openConnection(); httpURLConnection.setDoOutput(true); httpURLConnection.setRequestMethod("PUT"); BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(httpURLConnection.getOutputStream(), "UTF-8")); writer.write(tableConfig.toJSONConfigString()); writer.flush(); BufferedReader reader = new BufferedReader(new InputStreamReader(httpURLConnection.getInputStream(), "UTF-8")); return reader.readLine().equals("done"); }
job.getConfiguration().set(JobConfigConstants.TABLE_CONFIG, tableConfig.toJSONConfigString());
@Test public void rebalanceOfflineTable() { String tableName = "testOfflineTable"; _offlineBuilder.setTableName(tableName); // create the table try { TableConfig offlineTableConfig = _offlineBuilder.build(); sendPostRequest(_createTableUrl, offlineTableConfig.toJSONConfigString()); } catch (Exception e) { Assert.fail("Failed to create offline table " + tableName + "Error: " + e.getMessage()); } // rebalance should not throw exception try { sendPostRequest(_controllerRequestURLBuilder.forTableRebalance(tableName, "offline"), null); } catch (Exception e) { Assert.fail("Failed to rebalance existing offline table " + tableName); } // rebalance should throw exception because realtime table does not exist try { sendPostRequest(_controllerRequestURLBuilder.forTableRebalance(tableName, "realtime"), null); } catch (Exception e) { if (!(e instanceof FileNotFoundException)) { Assert.fail("Did not fail to create non existent realtime table " + tableName); } else { return; } } Assert.fail("Did not fail to create non existent realtime table " + tableName); }
protected void updateOfflineTable(String tableName, String timeColumnName, String timeType, String brokerTenant, String serverTenant, String loadMode, SegmentVersion segmentVersion, List<String> invertedIndexColumns, List<String> bloomFilterColumns, TableTaskConfig taskConfig) throws Exception { TableConfig tableConfig = getOfflineTableConfig(tableName, timeColumnName, timeType, brokerTenant, serverTenant, loadMode, segmentVersion, invertedIndexColumns, bloomFilterColumns, taskConfig); if (!isUsingNewConfigFormat()) { sendPutRequest(_controllerRequestURLBuilder.forUpdateTableConfig(tableName), tableConfig.toJSONConfigString()); } else { _offlineTableConfig = tableConfig; } }
offlineTableConfig.setTableName("bad__table__name"); try { sendPostRequest(_createTableUrl, offlineTableConfig.toJSONConfigString()); Assert.fail("Creation of an OFFLINE table with two underscores in the table name does not fail"); } catch (IOException e) { String offlineTableJSONConfigString = offlineTableConfig.toJSONConfigString(); sendPostRequest(_createTableUrl, offlineTableJSONConfigString); offlineTableConfig.setTableName("invalid_replication_table"); try { sendPostRequest(_createTableUrl, offlineTableConfig.toJSONConfigString()); Assert.fail("Creation of an invalid OFFLINE table does not fail"); } catch (IOException e) { realtimeTableConfig.setTableName("bad__table__name"); try { sendPostRequest(_createTableUrl, realtimeTableConfig.toJSONConfigString()); Assert.fail("Creation of a REALTIME table with two underscores in the table name does not fail"); } catch (IOException e) { TableConfig invalidConfig = _realtimeBuilder.build(); try { sendPostRequest(_createTableUrl, realtimeTableConfig.toJSONConfigString()); Assert.fail("Creation of a REALTIME table without a valid schema does not fail"); } catch (IOException e) { addDummySchema(schemaName); TableConfig diffConfig = _realtimeBuilder.build();
protected void updateRealtimeTableConfig(String tablename, List<String> invertedIndexCols, List<String> bloomFilterCols) throws Exception { IndexingConfig config = _realtimeTableConfig.getIndexingConfig(); config.setInvertedIndexColumns(invertedIndexCols); config.setBloomFilterColumns(bloomFilterCols); sendPutRequest(_controllerRequestURLBuilder.forUpdateTableConfig(tablename), _realtimeTableConfig.toJSONConfigString()); }
protected void addOfflineTable(String tableName, String timeColumnName, String timeType, String brokerTenant, String serverTenant, String loadMode, SegmentVersion segmentVersion, List<String> invertedIndexColumns, List<String> bloomFilterColumns, TableTaskConfig taskConfig) throws Exception { TableConfig tableConfig = getOfflineTableConfig(tableName, timeColumnName, timeType, brokerTenant, serverTenant, loadMode, segmentVersion, invertedIndexColumns, bloomFilterColumns, taskConfig); if (!isUsingNewConfigFormat()) { sendPostRequest(_controllerRequestURLBuilder.forTableCreate(), tableConfig.toJSONConfigString()); } else { _offlineTableConfig = tableConfig; } }
private void testTableMinReplicationInternal(String tableName, int tableReplication) throws Exception { String tableJSONConfigString = _offlineBuilder.setTableName(tableName).setNumReplicas(tableReplication).build().toJSONConfigString(); sendPostRequest(_createTableUrl, tableJSONConfigString); // table creation should succeed TableConfig tableConfig = getTableConfig(tableName, "OFFLINE"); Assert.assertEquals(tableConfig.getValidationConfig().getReplicationNumber(), Math.max(tableReplication, MIN_NUM_REPLICAS)); addDummySchema(tableName); tableJSONConfigString = _realtimeBuilder.setTableName(tableName).setNumReplicas(tableReplication).build().toJSONConfigString(); sendPostRequest(_createTableUrl, tableJSONConfigString); tableConfig = getTableConfig(tableName, "REALTIME"); Assert.assertEquals(tableConfig.getValidationConfig().getReplicationNumber(), Math.max(tableReplication, MIN_NUM_REPLICAS)); // This test can only be done via integration test // int replicasPerPartition = Integer.valueOf(tableConfig.getValidationConfig().getReplicasPerPartition()); // Assert.assertEquals(replicasPerPartition, Math.max(tableReplication, TABLE_MIN_REPLICATION)); }
private void addTable() throws Exception { if (_tableConfigFile != null) { AddTableCommand addTableCommand = new AddTableCommand().setControllerPort(_controllerPort).setFilePath(_tableConfigFile).setExecute(true); addTableCommand.execute(); return; } if (_tableName == null) { LOGGER.error("Table info not specified in configuration, please specify either config file or table name"); return; } String controllerAddress = "http://" + _localhost + ":" + _controllerPort; String tableJSONConfigString = new TableConfig.Builder(CommonConstants.Helix.TableType.OFFLINE).setTableName(_tableName) .setTimeColumnName(_timeColumnName).setTimeType(_timeUnit).setNumReplicas(3).setBrokerTenant("broker") .setServerTenant("server").build().toJSONConfigString(); sendPostRequest(ControllerRequestURLBuilder.baseUrl(controllerAddress).forTableCreate(), tableJSONConfigString); }
String offlineTableJSONConfigString = offlineTableConfig.toJSONConfigString(); sendPostRequest(createTableUrl, offlineTableJSONConfigString);
String tableJSONConfigString = new TableConfig.Builder(CommonConstants.Helix.TableType.OFFLINE).setTableName(tableName).setNumReplicas(3) .build().toJSONConfigString(); sendPostRequest(_controllerRequestURLBuilder.forTableCreate(), tableJSONConfigString); Assert.assertEquals(
String tableName = "updateTC"; String tableJSONConfigString = _offlineBuilder.setTableName(tableName).setNumReplicas(2).build().toJSONConfigString(); sendPostRequest(_createTableUrl, tableJSONConfigString); sendPutRequest(_controllerRequestURLBuilder.forUpdateTableConfig(tableName), tableConfig.toJSONConfigString())); Assert.assertTrue(jsonResponse.has("status")); tableJSONConfigString = _realtimeBuilder.setTableName(tableName).setNumReplicas(2).build().toJSONConfigString(); sendPostRequest(_createTableUrl, tableJSONConfigString); tableConfig = getTableConfig(tableName, "REALTIME"); quota.setStorage("10G"); tableConfig.setQuotaConfig(quota); sendPutRequest(_controllerRequestURLBuilder.forUpdateTableConfig(tableName), tableConfig.toJSONConfigString()); modifiedConfig = getTableConfig(tableName, "REALTIME"); Assert.assertNotNull(modifiedConfig.getQuotaConfig()); sendPutRequest(_controllerRequestURLBuilder.forUpdateTableConfig(tableName), tableConfig.toJSONConfigString()); modifiedConfig = getTableConfig(tableName, "REALTIME"); Assert.assertNotNull(modifiedConfig.getQuotaConfig().getMaxQueriesPerSecond()); tableConfig.toJSONConfigString()); } catch (Exception e) { Assert.assertTrue(e instanceof FileNotFoundException);
sendPostRequest(_controllerRequestURLBuilder.forTableCreate(), tableConfig.toJSONConfigString());
.setNumReplicas(NUM_INSTANCES).build().toJSONConfigString(); sendPostRequest(_controllerRequestURLBuilder.forTableCreate(), tableJSONConfigString); Assert.assertEquals(