public static RealtimeSegmentType getSegmentType(String segmentName) { if (isHighLevelConsumerSegmentName(segmentName)) { HLCSegmentName segName = new HLCSegmentName(segmentName); if (segName.isOldStyleNaming()) { return RealtimeSegmentType.HLC_LONG; } else { return RealtimeSegmentType.HLC_SHORT; } } if (isLowLevelConsumerSegmentName(segmentName)) { return RealtimeSegmentType.LLC; } return RealtimeSegmentType.UNSUPPORTED; }
public String getGroupId() { throw new RuntimeException("No groupId in " + getSegmentName()); }
/** * Remove all segments from the ideal state, other than LLC ONLINE segments. We only want LLC ONLINE segments for this rebalance * We do not want to rebalance CONSUMING segments as part of this rebalance, it is done separately * * If any segments were OFFLINE instead of ONLINE, we will not rebalance them, * as that state was likely achieved because of some manual operation * @param mapFields * @param removedEntries */ private void filterSegmentsForRealtimeRebalance(Map<String, Map<String, String>> mapFields, Map<String, Map<String, String>> removedEntries) { Iterator<Map.Entry<String, Map<String, String>>> mapFieldsIterator = mapFields.entrySet().iterator(); while (mapFieldsIterator.hasNext()) { // Only keep LLC segments in ONLINE state for rebalance Map.Entry<String, Map<String, String>> entry = mapFieldsIterator.next(); final String segmentName = entry.getKey(); boolean keep = false; if (SegmentName.isLowLevelConsumerSegmentName(segmentName)) { Map<String, String> instanceStateMap = entry.getValue(); if (instanceStateMap.values() .contains(CommonConstants.Helix.StateModel.SegmentOnlineOfflineStateModel.ONLINE)) { keep = true; } } if (!keep) { removedEntries.put(segmentName, entry.getValue()); mapFieldsIterator.remove(); } } }
@Override protected Map<String, List<String>> computeSegmentToServersMapFromExternalView(ExternalView externalView, List<InstanceConfig> instanceConfigs) { Map<String, List<String>> segmentToServersMap = new HashMap<>(); RoutingTableInstancePruner instancePruner = new RoutingTableInstancePruner(instanceConfigs); for (String segmentName : externalView.getPartitionSet()) { List<String> servers = new ArrayList<>(); for (Map.Entry<String, String> entry : externalView.getStateMap(segmentName).entrySet()) { String serverName = entry.getKey(); if (entry.getValue().equals(CommonConstants.Helix.StateModel.SegmentOnlineOfflineStateModel.ONLINE) && !instancePruner.isInactive(serverName) && SegmentName.isHighLevelConsumerSegmentName(segmentName)) { servers.add(serverName); } } if (servers.size() != 0) { segmentToServersMap.put(segmentName, servers); } else { handleNoServingHost(segmentName); } } return segmentToServersMap; }
assertEquals(SegmentName.getSegmentType(oldV1Name), SegmentName.RealtimeSegmentType.HLC_LONG); assertEquals(SegmentName.getSegmentType(shortV1Name), SegmentName.RealtimeSegmentType.HLC_SHORT); assertEquals(SegmentName.getSegmentType(v2Name), SegmentName.RealtimeSegmentType.LLC); assertEquals(shortNameSegment.getSegmentType(), SegmentName.RealtimeSegmentType.HLC_SHORT); assertEquals(SegmentName.getSegmentType(llcSegment.getSegmentName()), SegmentName.RealtimeSegmentType.LLC); assertEquals(SegmentName.getSegmentType(longNameSegment.getSegmentName()), SegmentName.RealtimeSegmentType.HLC_LONG); assertEquals(SegmentName.getSegmentType(shortNameSegment.getSegmentName()), SegmentName.RealtimeSegmentType.HLC_SHORT); assertEquals(SegmentName.getSegmentType(shortNameSegment.getSegmentName() + "__"), SegmentName.RealtimeSegmentType.UNSUPPORTED); assertEquals(SegmentName.getSegmentType("__" + shortNameSegment.getSegmentName()), SegmentName.RealtimeSegmentType.UNSUPPORTED); assertEquals(SegmentName.getSegmentType("__abc__"), SegmentName.RealtimeSegmentType.UNSUPPORTED); assertEquals(SegmentName.getSegmentType("a__abc__1__3__4__54__g__gg___h"), SegmentName.RealtimeSegmentType.UNSUPPORTED);
@Transition(from = "OFFLINE", to = "CONSUMING") public void onBecomeConsumingFromOffline(Message message, NotificationContext context) { Preconditions.checkState(SegmentName.isLowLevelConsumerSegmentName(message.getPartitionName()), "Tried to go into CONSUMING state on non-low level segment"); _logger.info("SegmentOnlineOfflineStateModel.onBecomeConsumingFromOffline() : " + message); // We do the same processing as usual for going to the consuming state, which adds the segment to the table data // manager and starts Kafka consumption onBecomeOnlineFromOffline(message, context); }
@Nullable public static RealtimeSegmentZKMetadata getRealtimeSegmentZKMetadata( @Nonnull ZkHelixPropertyStore<ZNRecord> propertyStore, @Nonnull String tableName, @Nonnull String segmentName) { String realtimeTableName = TableNameBuilder.REALTIME.tableNameWithType(tableName); ZNRecord znRecord = propertyStore .get(constructPropertyStorePathForSegment(realtimeTableName, segmentName), null, AccessOption.PERSISTENT); // It is possible that the segment metadata has just been deleted due to retention. if (znRecord == null) { return null; } if (SegmentName.isHighLevelConsumerSegmentName(segmentName)) { return new RealtimeSegmentZKMetadata(znRecord); } else { return new LLCRealtimeSegmentZKMetadata(znRecord); } }
@Override public void computeOnExternalViewChange(String tableName, ExternalView externalView, List<InstanceConfig> instanceConfigs) { Set<String> segmentSet = externalView.getPartitionSet(); for (String segmentName : segmentSet) { if (SegmentName.isHighLevelConsumerSegmentName(segmentName)) { _hasHLC = true; } if (SegmentName.isLowLevelConsumerSegmentName(segmentName)) { _hasLLC = true; } } if (_hasHLC) { _realtimeHLCRoutingTableBuilder.computeOnExternalViewChange(tableName, externalView, instanceConfigs); } if (_hasLLC) { _realtimeLLCRoutingTableBuilder.computeOnExternalViewChange(tableName, externalView, instanceConfigs); } }
public void cleanupLLC(final String realtimeTableName) { // If there are any completions in the pipeline we let them commit. Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS); IdealState idealState = HelixHelper.getTableIdealState(_helixManager, realtimeTableName); final List<String> segmentsToRemove = new ArrayList<String>(); Set<String> allSegments = idealState.getPartitionSet(); int removeCount = 0; for (String segmentName : allSegments) { if (SegmentName.isLowLevelConsumerSegmentName(segmentName)) { segmentsToRemove.add(segmentName); removeCount++; } } LOGGER.info("Attempting to remove {} LLC segments of table {}", removeCount, realtimeTableName); _helixResourceManager.deleteSegments(realtimeTableName, segmentsToRemove); }
deletedSegmentDestURI.toString()); } else { if (!SegmentName.isHighLevelConsumerSegmentName(segmentId)) { LOGGER.warn("Not found local segment file for segment {}" + fileToMoveURI.toString());
public int getPartitionId() { throw new RuntimeException("No partitionId in " + getSegmentName()); }
protected RealtimeIndexOffHeapMemoryManager(ServerMetrics serverMetrics, String segmentName) { _serverMetrics = serverMetrics; _segmentName = segmentName; if (SegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); _tableName = llcSegmentName.getTableName(); } else if (SegmentName.isHighLevelConsumerSegmentName(segmentName)) { HLCSegmentName hlcSegmentName = new HLCSegmentName(segmentName); _tableName = hlcSegmentName.getTableName(); } else { // For testing only _tableName = "NoSuchTable"; } }
/** * Compute the table of a sorted list of segments grouped by Kafka partition. * * @param segmentSet is the set of segment names that need to be sorted. * @return map of Stream partition to sorted set of segment names */ public static Map<String, SortedSet<SegmentName>> sortSegmentsByStreamPartition(Set<String> segmentSet) { Map<String, SortedSet<SegmentName>> sortedSegmentsByStreamPartition = new HashMap<>(); for (String segment : segmentSet) { // Ignore segments that are not low level consumer segments if (!SegmentName.isLowLevelConsumerSegmentName(segment)) { continue; } final LLCSegmentName segmentName = new LLCSegmentName(segment); String streamPartitionId = segmentName.getPartitionRange(); SortedSet<SegmentName> segmentsForPartition = sortedSegmentsByStreamPartition.get(streamPartitionId); // Create sorted set if necessary if (segmentsForPartition == null) { segmentsForPartition = new TreeSet<>(); sortedSegmentsByStreamPartition.put(streamPartitionId, segmentsForPartition); } segmentsForPartition.add(segmentName); } return sortedSegmentsByStreamPartition; } }
@VisibleForTesting static long computeRealtimeTotalDocumentInSegments(List<RealtimeSegmentZKMetadata> realtimeSegmentZKMetadataList, boolean countHLCSegments) { long numTotalDocs = 0; String groupId = ""; for (RealtimeSegmentZKMetadata realtimeSegmentZKMetadata : realtimeSegmentZKMetadataList) { String segmentName = realtimeSegmentZKMetadata.getSegmentName(); if (SegmentName.isHighLevelConsumerSegmentName(segmentName)) { if (countHLCSegments) { HLCSegmentName hlcSegmentName = new HLCSegmentName(segmentName); String segmentGroupIdName = hlcSegmentName.getGroupId(); if (groupId.isEmpty()) { groupId = segmentGroupIdName; } // Discard all segments with different groupids as they are replicas if (groupId.equals(segmentGroupIdName) && realtimeSegmentZKMetadata.getTotalRawDocs() >= 0) { numTotalDocs += realtimeSegmentZKMetadata.getTotalRawDocs(); } } } else { // Low level segments if (!countHLCSegments) { numTotalDocs += realtimeSegmentZKMetadata.getTotalRawDocs(); } } } return numTotalDocs; }
public String getPartitionRange() { throw new RuntimeException("No partitionRange in " + getSegmentName()); }
/** * Returns the LLC realtime segments for the given table. * * @param propertyStore Helix property store * @param realtimeTableName Realtime table name * @return List of LLC realtime segment names */ public static List<String> getLLCRealtimeSegments(ZkHelixPropertyStore<ZNRecord> propertyStore, String realtimeTableName) { List<String> llcRealtimeSegments = new ArrayList<>(); String segmentsPath = constructPropertyStorePathForResource(realtimeTableName); if (propertyStore.exists(segmentsPath, AccessOption.PERSISTENT)) { List<String> segments = propertyStore.getChildNames(segmentsPath, AccessOption.PERSISTENT); for (String segment : segments) { if (SegmentName.isLowLevelConsumerSegmentName(segment)) { llcRealtimeSegments.add(segment); } } } return llcRealtimeSegments; }
if (SegmentName.isHighLevelConsumerSegmentName(segmentName)) { manager = new HLRealtimeSegmentDataManager(realtimeSegmentZKMetadata, tableConfig, instanceZKMetadata, this, _indexDir.getAbsolutePath(), indexLoadingConfig, schema, _serverMetrics);
Map<String, String> helixPartitionState = externalView.getStateMap(segmentName.getSegmentName()); boolean allInConsumingState = true; int replicasInConsumingState = 0;
if (SegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); int partitionId = llcSegmentName.getPartitionId();
if (SegmentName.isHighLevelConsumerSegmentName(partition)) { HLCSegmentName segName = new HLCSegmentName(partition); RealtimeSegmentZKMetadata realtimeSegmentZKMetadata = ZKMetadataProvider