/** * Return whether the given resource name represents a table resource. * * @param resourceName Resource name * @return Whether the resource name represents a table resource */ public static boolean isTableResource(@Nonnull String resourceName) { return OFFLINE.tableHasTypeSuffix(resourceName) || REALTIME.tableHasTypeSuffix(resourceName); } }
/** * Get the table type based on the given table name with type suffix. * * @param tableName Table name with or without type suffix * @return Table type for the given table name, null if cannot be determined by table name */ @Nullable public static TableType getTableTypeFromTableName(@Nonnull String tableName) { if (OFFLINE.tableHasTypeSuffix(tableName)) { return TableType.OFFLINE; } if (REALTIME.tableHasTypeSuffix(tableName)) { return TableType.REALTIME; } return null; }
/** * Extract the raw table name from the given table name with type suffix. * * @param tableName Table name with or without type suffix * @return Table name without type suffix */ @Nonnull public static String extractRawTableName(@Nonnull String tableName) { if (OFFLINE.tableHasTypeSuffix(tableName)) { return tableName.substring(0, tableName.length() - OFFLINE._typeSuffix.length()); } if (REALTIME.tableHasTypeSuffix(tableName)) { return tableName.substring(0, tableName.length() - REALTIME._typeSuffix.length()); } return tableName; }
/** * Get all Pinot offline table names * @return List of Pinot realtime table names */ @Nonnull public List<String> getAllOfflineTables() { List<String> resourceNames = getAllResources(); Iterator<String> iterator = resourceNames.iterator(); while (iterator.hasNext()) { if (!TableNameBuilder.OFFLINE.tableHasTypeSuffix(iterator.next())) { iterator.remove(); } } return resourceNames; }
/** * Get all Pinot realtime table names. * * @return List of Pinot realtime table names */ @Nonnull public List<String> getAllRealtimeTables() { List<String> resourceNames = getAllResources(); Iterator<String> iterator = resourceNames.iterator(); while (iterator.hasNext()) { if (!TableNameBuilder.REALTIME.tableHasTypeSuffix(iterator.next())) { iterator.remove(); } } return resourceNames; }
/** * Get all segments' metadata for the given REALTIME table name. * * @param realtimeTableName Realtime table name * @return List of segments' metadata */ @Nonnull public List<RealtimeSegmentZKMetadata> getRealtimeSegmentsMetadata(@Nonnull String realtimeTableName) { Preconditions.checkArgument(TableNameBuilder.REALTIME.tableHasTypeSuffix(realtimeTableName)); return ZKMetadataProvider .getRealtimeSegmentZKMetadataListForTable(_pinotHelixResourceManager.getPropertyStore(), realtimeTableName); }
/** * Get all segments' metadata for the given OFFLINE table name. * * @param offlineTableName Offline table name * @return List of segments' metadata */ @Nonnull public List<OfflineSegmentZKMetadata> getOfflineSegmentsMetadata(@Nonnull String offlineTableName) { Preconditions.checkArgument(TableNameBuilder.OFFLINE.tableHasTypeSuffix(offlineTableName)); return ZKMetadataProvider .getOfflineSegmentZKMetadataListForTable(_pinotHelixResourceManager.getPropertyStore(), offlineTableName); }
@Override public void reloadSegment(@Nonnull String tableNameWithType, @Nonnull String segmentName) throws Exception { LOGGER.info("Reloading single segment: {} in table: {}", segmentName, tableNameWithType); SegmentMetadata segmentMetadata = getSegmentMetadata(tableNameWithType, segmentName); if (segmentMetadata == null) { return; } TableConfig tableConfig = ZKMetadataProvider.getTableConfig(_propertyStore, tableNameWithType); Preconditions.checkNotNull(tableConfig); Schema schema = null; // For OFFLINE table, try to get schema for default columns if (TableNameBuilder.OFFLINE.tableHasTypeSuffix(tableNameWithType)) { schema = ZKMetadataProvider.getTableSchema(_propertyStore, tableNameWithType); } reloadSegment(tableNameWithType, segmentMetadata, tableConfig, schema); LOGGER.info("Reloaded single segment: {} in table: {}", segmentName, tableNameWithType); }
@Override public void reloadAllSegments(@Nonnull String tableNameWithType) throws Exception { LOGGER.info("Reloading all segments in table: {}", tableNameWithType); TableConfig tableConfig = ZKMetadataProvider.getTableConfig(_propertyStore, tableNameWithType); Preconditions.checkNotNull(tableConfig); Schema schema = null; // For OFFLINE table, try to get schema for default columns if (TableNameBuilder.OFFLINE.tableHasTypeSuffix(tableNameWithType)) { schema = ZKMetadataProvider.getTableSchema(_propertyStore, tableNameWithType); } for (SegmentMetadata segmentMetadata : getAllSegmentsMetadata(tableNameWithType)) { reloadSegment(tableNameWithType, segmentMetadata, tableConfig, schema); } LOGGER.info("Reloaded all segments in table: {}", tableNameWithType); }
private void manageRetentionForTable(String tableNameWithType) { // Build retention strategy from table config TableConfig tableConfig = _pinotHelixResourceManager.getTableConfig(tableNameWithType); if (tableConfig == null) { LOGGER.error("Failed to get table config for table: {}", tableNameWithType); return; } SegmentsValidationAndRetentionConfig validationConfig = tableConfig.getValidationConfig(); String segmentPushType = validationConfig.getSegmentPushType(); if (!"APPEND".equalsIgnoreCase(segmentPushType)) { LOGGER.info("Segment push type is not APPEND for table: {}, skip", tableNameWithType); return; } String retentionTimeUnit = validationConfig.getRetentionTimeUnit(); String retentionTimeValue = validationConfig.getRetentionTimeValue(); RetentionStrategy retentionStrategy; try { retentionStrategy = new TimeRetentionStrategy(TimeUnit.valueOf(retentionTimeUnit.toUpperCase()), Long.parseLong(retentionTimeValue)); } catch (Exception e) { LOGGER.warn("Invalid retention time: {} {} for table: {}, skip", retentionTimeUnit, retentionTimeValue); return; } // Scan all segment ZK metadata and purge segments if necessary if (TableNameBuilder.OFFLINE.tableHasTypeSuffix(tableNameWithType)) { manageRetentionForOfflineTable(tableNameWithType, retentionStrategy); } else { manageRetentionForRealtimeTable(tableNameWithType, retentionStrategy); } }
/** * Returns the segment metadata. * * @param tableNameWithType Table name with type suffix * @param segmentName Segment name * @return Singleton JSON array of the segment metadata */ private ArrayNode getSegmentMetaData(String tableNameWithType, String segmentName) { ZkHelixPropertyStore<ZNRecord> propertyStore = _pinotHelixResourceManager.getPropertyStore(); if (!ZKMetadataProvider.isSegmentExisted(propertyStore, tableNameWithType, segmentName)) { return null; } ArrayNode ret = JsonUtils.newArrayNode(); ObjectNode jsonObj = JsonUtils.newObjectNode(); jsonObj.put(TABLE_NAME, tableNameWithType); if (TableNameBuilder.OFFLINE.tableHasTypeSuffix(tableNameWithType)) { // OFFLINE table OfflineSegmentZKMetadata offlineSegmentZKMetadata = ZKMetadataProvider.getOfflineSegmentZKMetadata(propertyStore, tableNameWithType, segmentName); jsonObj.set(STATE, JsonUtils.objectToJsonNode(offlineSegmentZKMetadata.toMap())); } else { // REALTIME table RealtimeSegmentZKMetadata realtimeSegmentZKMetadata = ZKMetadataProvider.getRealtimeSegmentZKMetadata(propertyStore, tableNameWithType, segmentName); jsonObj.set(STATE, JsonUtils.objectToJsonNode(realtimeSegmentZKMetadata.toMap())); } ret.add(jsonObj); return ret; }