public String getGroupId(String resourceName) { return _groupIdMap.get(TableNameBuilder.REALTIME.tableNameWithType(resourceName)); }
@Nullable public static TableConfig getRealtimeTableConfig(@Nonnull ZkHelixPropertyStore<ZNRecord> propertyStore, @Nonnull String tableName) { return getTableConfig(propertyStore, TableNameBuilder.REALTIME.tableNameWithType(tableName)); }
@Nullable public static TableConfig getOfflineTableConfig(@Nonnull ZkHelixPropertyStore<ZNRecord> propertyStore, @Nonnull String tableName) { return getTableConfig(propertyStore, TableNameBuilder.OFFLINE.tableNameWithType(tableName)); }
@Nullable public static OfflineSegmentZKMetadata getOfflineSegmentZKMetadata( @Nonnull ZkHelixPropertyStore<ZNRecord> propertyStore, @Nonnull String tableName, @Nonnull String segmentName) { String offlineTableName = TableNameBuilder.OFFLINE.tableNameWithType(tableName); ZNRecord znRecord = propertyStore .get(constructPropertyStorePathForSegment(offlineTableName, segmentName), null, AccessOption.PERSISTENT); if (znRecord == null) { return null; } return new OfflineSegmentZKMetadata(znRecord); }
public void updateMetadataConfigFor(String tableName, TableType type, TableCustomConfig newConfigs) throws Exception { String tableNameWithType = TableNameBuilder.forType(type).tableNameWithType(tableName); TableConfig tableConfig = ZKMetadataProvider.getTableConfig(_propertyStore, tableNameWithType); if (tableConfig == null) { throw new RuntimeException("Table: " + tableName + " of type: " + type + " does not exist"); } tableConfig.setCustomConfig(newConfigs); setExistingTableConfig(tableConfig, tableNameWithType, type); }
public void updateSegmentsValidationAndRetentionConfigFor(String tableName, TableType type, SegmentsValidationAndRetentionConfig newConfigs) throws Exception { String tableNameWithType = TableNameBuilder.forType(type).tableNameWithType(tableName); TableConfig tableConfig = ZKMetadataProvider.getTableConfig(_propertyStore, tableNameWithType); if (tableConfig == null) { throw new RuntimeException("Table: " + tableName + " of type: " + type + " does not exist"); } tableConfig.setValidationConfig(newConfigs); setExistingTableConfig(tableConfig, tableNameWithType, type); }
private void removeNewSegments() throws Exception { for (int i = 0; i < 10; i++) { _helixResourceManager .deleteSegment(TableNameBuilder.OFFLINE.tableNameWithType(TABLE_NAME), NEW_SEGMENT_PREFIX + i); } }
@Test public void testBrokerDebugOutput() throws Exception { String tableName = getTableName(); Assert.assertNotNull(getDebugInfo("debug/timeBoundary/" + tableName)); Assert.assertNotNull(getDebugInfo("debug/timeBoundary/" + TableNameBuilder.OFFLINE.tableNameWithType(tableName))); Assert.assertNotNull(getDebugInfo("debug/timeBoundary/" + TableNameBuilder.REALTIME.tableNameWithType(tableName))); Assert.assertNotNull(getDebugInfo("debug/routingTable/" + tableName)); Assert.assertNotNull(getDebugInfo("debug/routingTable/" + TableNameBuilder.OFFLINE.tableNameWithType(tableName))); Assert.assertNotNull(getDebugInfo("debug/routingTable/" + TableNameBuilder.REALTIME.tableNameWithType(tableName))); }
public static boolean setOfflineSegmentZKMetadata(ZkHelixPropertyStore<ZNRecord> propertyStore, OfflineSegmentZKMetadata offlineSegmentZKMetadata, int expectedVersion) { // NOTE: Helix will throw ZkBadVersionException if version does not match try { return propertyStore.set(constructPropertyStorePathForSegment( TableNameBuilder.OFFLINE.tableNameWithType(offlineSegmentZKMetadata.getTableName()), offlineSegmentZKMetadata.getSegmentName()), offlineSegmentZKMetadata.toZNRecord(), expectedVersion, AccessOption.PERSISTENT); } catch (ZkBadVersionException e) { return false; } }
protected void dropRealtimeTable(String tableName) throws Exception { sendDeleteRequest( _controllerRequestURLBuilder.forTableDelete(TableNameBuilder.REALTIME.tableNameWithType(tableName))); }
public static boolean setOfflineSegmentZKMetadata(ZkHelixPropertyStore<ZNRecord> propertyStore, OfflineSegmentZKMetadata offlineSegmentZKMetadata) { return propertyStore.set(constructPropertyStorePathForSegment( TableNameBuilder.OFFLINE.tableNameWithType(offlineSegmentZKMetadata.getTableName()), offlineSegmentZKMetadata.getSegmentName()), offlineSegmentZKMetadata.toZNRecord(), AccessOption.PERSISTENT); }
public static boolean setRealtimeSegmentZKMetadata(ZkHelixPropertyStore<ZNRecord> propertyStore, RealtimeSegmentZKMetadata realtimeSegmentZKMetadata) { return propertyStore.set(constructPropertyStorePathForSegment( TableNameBuilder.REALTIME.tableNameWithType(realtimeSegmentZKMetadata.getTableName()), realtimeSegmentZKMetadata.getSegmentName()), realtimeSegmentZKMetadata.toZNRecord(), AccessOption.PERSISTENT); }
protected void dropOfflineTable(String tableName) throws Exception { sendDeleteRequest( _controllerRequestURLBuilder.forTableDelete(TableNameBuilder.OFFLINE.tableNameWithType(tableName))); }
private boolean allSegmentsPushedToIdealState(String tableName, int segmentNum) { IdealState idealState = _helixAdmin.getResourceIdealState(HELIX_CLUSTER_NAME, TableNameBuilder.OFFLINE.tableNameWithType(tableName)); return idealState != null && idealState.getPartitionSet() != null && idealState.getPartitionSet().size() == segmentNum; } }
private boolean validateNumSegments(int numSegments) { String tableNameWithType = TableNameBuilder.OFFLINE.tableNameWithType(TABLE_NAME); IdealState idealState = _helixAdmin.getResourceIdealState(getHelixClusterName(), tableNameWithType); return idealState.getRecord().getMapFields().keySet().size() == numSegments; }
/** * Helper method to create an OFFLINE broker request from the given hybrid broker request. * <p>This step will attach the time boundary to the request. */ private BrokerRequest getOfflineBrokerRequest(BrokerRequest hybridBrokerRequest) { BrokerRequest offlineRequest = hybridBrokerRequest.deepCopy(); String rawTableName = hybridBrokerRequest.getQuerySource().getTableName(); String offlineTableName = TableNameBuilder.OFFLINE.tableNameWithType(rawTableName); offlineRequest.getQuerySource().setTableName(offlineTableName); attachTimeBoundary(rawTableName, offlineRequest, true); return offlineRequest; }
/** * Helper method to create a REALTIME broker request from the given hybrid broker request. * <p>This step will attach the time boundary to the request. */ private BrokerRequest getRealtimeBrokerRequest(BrokerRequest hybridBrokerRequest) { BrokerRequest realtimeRequest = hybridBrokerRequest.deepCopy(); String rawTableName = hybridBrokerRequest.getQuerySource().getTableName(); String realtimeTableName = TableNameBuilder.REALTIME.tableNameWithType(rawTableName); realtimeRequest.getQuerySource().setTableName(realtimeTableName); attachTimeBoundary(rawTableName, realtimeRequest, false); return realtimeRequest; }
private boolean allSegmentsPushedToIdealState(String tableName, int segmentNum) { IdealState idealState = _helixAdmin.getResourceIdealState(getHelixClusterName(), TableNameBuilder.OFFLINE.tableNameWithType(tableName)); return idealState != null && idealState.getPartitionSet() != null && idealState.getPartitionSet().size() == segmentNum; }
@Test public void testSegmentFlushSize() throws Exception { String zkSegmentsPath = "/SEGMENTS/" + TableNameBuilder.REALTIME.tableNameWithType(getTableName()); List<String> segmentNames = _propertyStore.getChildNames(zkSegmentsPath, 0); for (String segmentName : segmentNames) { ZNRecord znRecord = _propertyStore.get(zkSegmentsPath + "/" + segmentName, null, 0); Assert.assertEquals(znRecord.getSimpleField(CommonConstants.Segment.FLUSH_THRESHOLD_SIZE), Integer.toString(getRealtimeSegmentFlushSize() / getNumKafkaPartitions()), "Segment: " + segmentName + " does not have the expected flush size"); } }
private void updateTableConfig(int targetNumInstancePerPartition, int targetNumReplicaGroup) throws IOException { String tableNameWithType = TableNameBuilder.OFFLINE.tableNameWithType(TABLE_NAME); TableConfig tableConfig = _helixResourceManager.getTableConfig(TABLE_NAME, CommonConstants.Helix.TableType.OFFLINE); tableConfig.getValidationConfig().getReplicaGroupStrategyConfig() .setNumInstancesPerPartition(targetNumInstancePerPartition); tableConfig.getValidationConfig().setReplication(Integer.toString(targetNumReplicaGroup)); _helixResourceManager .setExistingTableConfig(tableConfig, tableNameWithType, CommonConstants.Helix.TableType.OFFLINE); } }