@Test(dataProvider = "realtimeTagConfigTestDataProvider") public void testRealtimeTagConfig(TableConfig tableConfig, String expectedServerTenant, String expectedRealtimeConsumingTag, String expectedRealtimeCompletedTag) { RealtimeTagConfig tagConfig = new RealtimeTagConfig(tableConfig); Assert.assertEquals(tagConfig.getServerTenantName(), expectedServerTenant); Assert.assertEquals(tagConfig.getConsumingServerTag(), expectedRealtimeConsumingTag); Assert.assertEquals(tagConfig.getCompletedServerTag(), expectedRealtimeCompletedTag); }
@VisibleForTesting protected List<String> getConsumingTaggedInstances(TableConfig tableConfig) { RealtimeTagConfig realtimeTagConfig = new RealtimeTagConfig(tableConfig); String consumingServerTag = realtimeTagConfig.getConsumingServerTag(); List<String> consumingTaggedInstances = HelixHelper.getInstancesWithTag(_helixManager, consumingServerTag); if (consumingTaggedInstances.isEmpty()) { throw new IllegalStateException("No instances found with tag " + consumingServerTag); } return consumingTaggedInstances; } }
public List<String> getServerInstancesForTable(String tableName, TableType tableType) { TableConfig tableConfig = getTableConfig(tableName, tableType); Set<String> serverInstances = new HashSet<>(); if (TableType.OFFLINE.equals(tableType)) { OfflineTagConfig tagConfig = new OfflineTagConfig(tableConfig); serverInstances.addAll(HelixHelper.getInstancesWithTag(_helixZkManager, tagConfig.getOfflineServerTag())); } else if (TableType.REALTIME.equals(tableType)) { RealtimeTagConfig tagConfig = new RealtimeTagConfig(tableConfig); serverInstances.addAll(HelixHelper.getInstancesWithTag(_helixZkManager, tagConfig.getConsumingServerTag())); serverInstances.addAll(HelixHelper.getInstancesWithTag(_helixZkManager, tagConfig.getCompletedServerTag())); } return Lists.newArrayList(serverInstances); }
private void getServingInstances(TableConfig tableConfig, List<String> servingInstances, List<String> enabledServingInstances) { String tag; if (tableConfig.getTableType().equals(CommonConstants.Helix.TableType.REALTIME)) { RealtimeTagConfig realtimeTagConfig = new RealtimeTagConfig(tableConfig); tag = realtimeTagConfig.getCompletedServerTag(); } else { OfflineTagConfig offlineTagConfig = new OfflineTagConfig(tableConfig); tag = offlineTagConfig.getOfflineServerTag(); } servingInstances.addAll(getInstancesWithTag(tag)); enabledServingInstances.addAll(getEnabledInstancesWithTag(tag)); }
when(realtimeTagConfig.getConsumingServerTag()).thenReturn(serverTenantConsuming); when(realtimeTagConfig.getCompletedServerTag()).thenReturn(serverTenantCompleted);
/** * Check the given table. Perform relocation of segments if table is realtime and relocation is required * TODO: Model this to implement {@link org.apache.pinot.controller.helix.core.rebalance.RebalanceSegmentStrategy} interface * https://github.com/linkedin/pinot/issues/2609 * * @param tableNameWithType */ private void runRelocation(String tableNameWithType) { LOGGER.info("Starting relocation of segments for table: {}", tableNameWithType); TableConfig tableConfig = _pinotHelixResourceManager.getRealtimeTableConfig(tableNameWithType); final RealtimeTagConfig realtimeTagConfig = new RealtimeTagConfig(tableConfig); if (!realtimeTagConfig.isRelocateCompletedSegments()) { LOGGER.info("Skipping relocation of segments for {}", tableNameWithType); return; } Function<IdealState, IdealState> updater = new Function<IdealState, IdealState>() { @Nullable @Override public IdealState apply(@Nullable IdealState idealState) { if (!idealState.isEnabled()) { LOGGER.info("Skipping relocation of segments for {} since ideal state is disabled", tableNameWithType); return null; } relocateSegments(realtimeTagConfig, idealState); return idealState; } }; HelixHelper.updateIdealState(_pinotHelixResourceManager.getHelixZkManager(), tableNameWithType, updater, RetryPolicies.exponentialBackoffRetryPolicy(5, 1000, 2.0f)); }
completedServersQueue.add(chosenServer); LOGGER.info("Relocating segment {} from server {} to completed server {} (tag {})", segmentName, instance, chosenServer, realtimeTagConfig.getCompletedServerTag()); break;
public static IdealState buildInitialHighLevelRealtimeIdealStateFor(String realtimeTableName, TableConfig realtimeTableConfig, HelixManager helixManager, ZkHelixPropertyStore<ZNRecord> zkHelixPropertyStore, boolean enableBatchMessageMode) { RealtimeTagConfig realtimeTagConfig = new RealtimeTagConfig(realtimeTableConfig); final List<String> realtimeInstances = HelixHelper.getInstancesWithTag(helixManager, realtimeTagConfig.getConsumingServerTag()); IdealState idealState = buildEmptyRealtimeIdealStateFor(realtimeTableName, 1, enableBatchMessageMode); if (realtimeInstances.size() % Integer.parseInt(realtimeTableConfig.getValidationConfig().getReplication()) != 0) { throw new RuntimeException( "Number of instance in current tenant should be an integer multiples of the number of replications"); } setupInstanceConfigForHighLevelConsumer(realtimeTableName, realtimeInstances.size(), Integer.parseInt(realtimeTableConfig.getValidationConfig().getReplication()), realtimeTableConfig.getIndexingConfig().getStreamConfigs(), zkHelixPropertyStore, realtimeInstances); return idealState; }
final List<String> completedServers = getInstancesWithTag(helixManager, realtimeTagConfig.getCompletedServerTag()); if (completedServers.isEmpty()) { throw new IllegalStateException( "Found no realtime completed servers with tag " + realtimeTagConfig.getCompletedServerTag());