@Override protected List<Map<String, List<String>>> computeRoutingTablesFromSegmentToServersMap( Map<String, List<String>> segmentsToServerMap) { List<Map<String, List<String>>> routingTables = new ArrayList<>(); Map<String, Map<String, List<String>>> groupIdToRouting = new HashMap<>(); for (Map.Entry<String, List<String>> entry : segmentsToServerMap.entrySet()) { String segmentName = entry.getKey(); HLCSegmentName hlcSegmentName = new HLCSegmentName(segmentName); String groupId = hlcSegmentName.getGroupId(); Map<String, List<String>> routingTableForGroupId = groupIdToRouting.computeIfAbsent(groupId, k -> new HashMap<>()); List<String> servers = entry.getValue(); for (String serverName : servers) { List<String> segmentsForServer = routingTableForGroupId.computeIfAbsent(serverName, k -> new ArrayList<>()); segmentsForServer.add(segmentName); } } routingTables.addAll(groupIdToRouting.values()); return routingTables; }
@Test public void testExceptions() { // Nulls should not work /* try { SegmentNameBuilder.Realtime.buildHighLevelConsumerSegmentName(null, null, null); fail("Exception not thrown"); } catch (Exception e) { // Expected } */ // Double underscores should not work try { HLCSegmentName segName = new HLCSegmentName("__", "a", "b"); // SegmentNameBuilder.Realtime.buildHighLevelConsumerSegmentName("__", "a", "b"); fail("Exception not thrown"); } catch (Exception e) { // Expected } }
public static RealtimeSegmentType getSegmentType(String segmentName) { if (isHighLevelConsumerSegmentName(segmentName)) { HLCSegmentName segName = new HLCSegmentName(segmentName); if (segName.isOldStyleNaming()) { return RealtimeSegmentType.HLC_LONG; } else { return RealtimeSegmentType.HLC_SHORT; } } if (isLowLevelConsumerSegmentName(segmentName)) { return RealtimeSegmentType.LLC; } return RealtimeSegmentType.UNSUPPORTED; }
@VisibleForTesting static long computeRealtimeTotalDocumentInSegments(List<RealtimeSegmentZKMetadata> realtimeSegmentZKMetadataList, boolean countHLCSegments) { long numTotalDocs = 0; String groupId = ""; for (RealtimeSegmentZKMetadata realtimeSegmentZKMetadata : realtimeSegmentZKMetadataList) { String segmentName = realtimeSegmentZKMetadata.getSegmentName(); if (SegmentName.isHighLevelConsumerSegmentName(segmentName)) { if (countHLCSegments) { HLCSegmentName hlcSegmentName = new HLCSegmentName(segmentName); String segmentGroupIdName = hlcSegmentName.getGroupId(); if (groupId.isEmpty()) { groupId = segmentGroupIdName; } // Discard all segments with different groupids as they are replicas if (groupId.equals(segmentGroupIdName) && realtimeSegmentZKMetadata.getTotalRawDocs() >= 0) { numTotalDocs += realtimeSegmentZKMetadata.getTotalRawDocs(); } } } else { // Low level segments if (!countHLCSegments) { numTotalDocs += realtimeSegmentZKMetadata.getTotalRawDocs(); } } } return numTotalDocs; }
protected RealtimeIndexOffHeapMemoryManager(ServerMetrics serverMetrics, String segmentName) { _serverMetrics = serverMetrics; _segmentName = segmentName; if (SegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); _tableName = llcSegmentName.getTableName(); } else if (SegmentName.isHighLevelConsumerSegmentName(segmentName)) { HLCSegmentName hlcSegmentName = new HLCSegmentName(segmentName); _tableName = hlcSegmentName.getTableName(); } else { // For testing only _tableName = "NoSuchTable"; } }
if (groupId != null && !groupId.isEmpty() && partitionId != null && !partitionId.isEmpty()) { listOfSegmentsToAddToInstances.add(new Pair<String, String>( new HLCSegmentName(groupId, partitionId, String.valueOf(System.currentTimeMillis())).getSegmentName(), instanceId)); } else { HLCSegmentName segName = new HLCSegmentName(partition); RealtimeSegmentZKMetadata realtimeSegmentZKMetadata = ZKMetadataProvider .getRealtimeSegmentZKMetadata(_pinotHelixResourceManager.getPropertyStore(), segName.getTableName(), String partitionId = instanceZKMetadata.getPartition(realtimeTableName); listOfSegmentsToAddToInstances.add(new Pair<String, String>( new HLCSegmentName(groupId, partitionId, String.valueOf(System.currentTimeMillis())).getSegmentName(), instanceId)); final HLCSegmentName segName = new HLCSegmentName(segmentId); String resourceName = segName.getTableName();
String segmentName1 = new HLCSegmentName(group1, "0", "1").getSegmentName(); String segmentName2 = new HLCSegmentName(group1, "0", "2").getSegmentName(); String segmentName3 = new HLCSegmentName(group1, "0", "3").getSegmentName(); String segmentName4 = new HLCSegmentName(group2, "0", "3").getSegmentName();
String instanceName = instanceNames[groupId * numInstancesPerGroup + groupOffset]; String groupName = groupPrefix + "_" + groupId; String segmentName = new HLCSegmentName(groupName, ALL_PARTITIONS, String.valueOf(segmentNum)).getSegmentName(); externalView.setState(segmentName, instanceName, "ONLINE");
.setState(new HLCSegmentName(group0, ALL_PARTITIONS, "0").getSegmentName(), "dataServer_instance_0", "ONLINE"); externalView .setState(new HLCSegmentName(group0, ALL_PARTITIONS, "1").getSegmentName(), "dataServer_instance_1", "ONLINE"); externalView .setState(new HLCSegmentName(group1, ALL_PARTITIONS, "2").getSegmentName(), "dataServer_instance_2", "ONLINE"); externalView .setState(new HLCSegmentName(group1, ALL_PARTITIONS, "3").getSegmentName(), "dataServer_instance_3", "ONLINE"); externalView .setState(new HLCSegmentName(group2, ALL_PARTITIONS, "4").getSegmentName(), "dataServer_instance_4", "ONLINE"); externalView .setState(new HLCSegmentName(group2, ALL_PARTITIONS, "5").getSegmentName(), "dataServer_instance_5", "ONLINE"); routingTable.markDataResourceOnline(generateTableConfig("testResource0_REALTIME"), externalView, generateInstanceConfigs("dataServer_instance", 0, 5)); ExternalView externalView1 = new ExternalView("testResource1_REALTIME"); externalView1.setState(new HLCSegmentName(group0, ALL_PARTITIONS, "10").getSegmentName(), "dataServer_instance_10", "ONLINE"); externalView1.setState(new HLCSegmentName(group0, ALL_PARTITIONS, "11").getSegmentName(), "dataServer_instance_11", "ONLINE"); externalView1.setState(new HLCSegmentName(group0, ALL_PARTITIONS, "12").getSegmentName(), "dataServer_instance_12", "ONLINE"); routingTable.markDataResourceOnline(generateTableConfig("testResource1_REALTIME"), externalView1, generateInstanceConfigs("dataServer_instance", 10, 12)); ExternalView externalView2 = new ExternalView("testResource2_REALTIME"); externalView2.setState(new HLCSegmentName(group0, ALL_PARTITIONS, "20").getSegmentName(), "dataServer_instance_20", "ONLINE"); externalView2.setState(new HLCSegmentName(group0, ALL_PARTITIONS, "21").getSegmentName(), "dataServer_instance_21", "ONLINE"); externalView2.setState(new HLCSegmentName(group0, ALL_PARTITIONS, "22").getSegmentName(), "dataServer_instance_22", "ONLINE");
final String partitionRange = "ALL"; final String sequenceNumber = "1234567"; HLCSegmentName shortNameSegment = new HLCSegmentName(groupId, partitionRange, sequenceNumber); HLCSegmentName longNameSegment = new HLCSegmentName(oldV1Name); final String shortV1Name = shortNameSegment.getSegmentName(); LLCSegmentName llcSegment = new LLCSegmentName("myTable", 0, 1, 1465508537069L);