/** * Returns all the segments that have been scheduled but not finished * * @param taskType Task type * @param clusterInfoProvider Cluster info provider * @return Set of running segments */ public static Set<Segment> getRunningSegments(@Nonnull String taskType, @Nonnull ClusterInfoProvider clusterInfoProvider) { Set<Segment> runningSegments = new HashSet<>(); Map<String, TaskState> taskStates = clusterInfoProvider.getTaskStates(taskType); for (Map.Entry<String, TaskState> entry : taskStates.entrySet()) { TaskState taskState = entry.getValue(); if (taskState == TaskState.NOT_STARTED || taskState == TaskState.IN_PROGRESS || taskState == TaskState.STOPPED) { for (PinotTaskConfig pinotTaskConfig : clusterInfoProvider.getTaskConfigs(entry.getKey())) { Map<String, String> configs = pinotTaskConfig.getConfigs(); runningSegments.add( new Segment(configs.get(MinionConstants.TABLE_NAME_KEY), configs.get(MinionConstants.SEGMENT_NAME_KEY))); } } } return runningSegments; } }
@Override protected SegmentConversionResult convert(@Nonnull PinotTaskConfig pinotTaskConfig, @Nonnull File originalIndexDir, @Nonnull File workingDir) throws Exception { Map<String, String> configs = pinotTaskConfig.getConfigs(); new RawIndexConverter(originalIndexDir, workingDir, configs.get(MinionConstants.ConvertToRawIndexTask.COLUMNS_TO_CONVERT_KEY)).convert(); return new SegmentConversionResult.Builder().setFile(workingDir) .setTableNameWithType(configs.get(MinionConstants.TABLE_NAME_KEY)) .setSegmentName(configs.get(MinionConstants.SEGMENT_NAME_KEY)).build(); }
throws Exception { String taskType = pinotTaskConfig.getTaskType(); Map<String, String> configs = pinotTaskConfig.getConfigs(); String tableNameWithType = configs.get(MinionConstants.TABLE_NAME_KEY); String inputSegmentNames = configs.get(MinionConstants.SEGMENT_NAME_KEY);
throws Exception { String taskType = pinotTaskConfig.getTaskType(); Map<String, String> configs = pinotTaskConfig.getConfigs(); String tableNameWithType = configs.get(MinionConstants.TABLE_NAME_KEY); String segmentName = configs.get(MinionConstants.SEGMENT_NAME_KEY);
@Override public Boolean executeTask(@Nonnull PinotTaskConfig pinotTaskConfig) { assertTrue(MINION_CONTEXT.getDataDir().exists()); assertNotNull(MINION_CONTEXT.getMinionMetrics()); assertNotNull(MINION_CONTEXT.getMinionVersion()); assertEquals(pinotTaskConfig.getTaskType(), TestTaskGenerator.TASK_TYPE); Map<String, String> configs = pinotTaskConfig.getConfigs(); assertEquals(configs.size(), 2); String offlineTableName = configs.get("tableName"); assertEquals(TableNameBuilder.getTableTypeFromTableName(offlineTableName), TableType.OFFLINE); String rawTableName = TableNameBuilder.extractRawTableName(offlineTableName); assertTrue(rawTableName.equals(TABLE_NAME_1) || rawTableName.equals(TABLE_NAME_2)); assertEquals(configs.get("tableType"), TableType.OFFLINE.toString()); do { if (_cancelled) { throw new TaskCancelledException("Task has been cancelled"); } } while (HOLD.get()); return true; } };
@Override protected SegmentConversionResult convert(@Nonnull PinotTaskConfig pinotTaskConfig, @Nonnull File originalIndexDir, @Nonnull File workingDir) throws Exception { Map<String, String> configs = pinotTaskConfig.getConfigs(); String tableNameWithType = configs.get(MinionConstants.TABLE_NAME_KEY); String rawTableName = TableNameBuilder.extractRawTableName(tableNameWithType); SegmentPurger.RecordPurgerFactory recordPurgerFactory = MINION_CONTEXT.getRecordPurgerFactory(); SegmentPurger.RecordPurger recordPurger = recordPurgerFactory != null ? recordPurgerFactory.getRecordPurger(rawTableName) : null; SegmentPurger.RecordModifierFactory recordModifierFactory = MINION_CONTEXT.getRecordModifierFactory(); SegmentPurger.RecordModifier recordModifier = recordModifierFactory != null ? recordModifierFactory.getRecordModifier(rawTableName) : null; SegmentPurger segmentPurger = new SegmentPurger(originalIndexDir, workingDir, recordPurger, recordModifier); File purgedSegmentFile = segmentPurger.purgeSegment(); if (purgedSegmentFile == null) { purgedSegmentFile = originalIndexDir; } return new SegmentConversionResult.Builder().setFile(purgedSegmentFile).setTableNameWithType(tableNameWithType) .setSegmentName(configs.get(MinionConstants.SEGMENT_NAME_KEY)) .setCustomProperty(RECORD_PURGER_KEY, segmentPurger.getRecordPurger()) .setCustomProperty(RECORD_MODIFIER_KEY, segmentPurger.getRecordModifier()) .setCustomProperty(NUM_RECORDS_PURGED_KEY, segmentPurger.getNumRecordsPurged()) .setCustomProperty(NUM_RECORDS_MODIFIED_KEY, segmentPurger.getNumRecordsModified()).build(); }