@Nonnull public static PinotTaskConfig fromHelixTaskConfig(@Nonnull TaskConfig helixTaskConfig) { Map<String, String> configs = new HashMap<>(helixTaskConfig.getConfigMap()); // Inside Helix task config map, there are 3 extra Helix properties: TASK_COMMAND, TASK_ID, TASK_TARGET_PARTITION configs.remove(TASK_ID_KEY); configs.remove(TASK_COMMAND_KEY); configs.remove(TASK_TARGET_PARTITION_KEY); return new PinotTaskConfig(helixTaskConfig.getCommand(), configs); }
configs.put(MinionConstants.ConvertToRawIndexTask.COLUMNS_TO_CONVERT_KEY, columnsToConvertConfig); pinotTaskConfigs.add(new PinotTaskConfig(MinionConstants.ConvertToRawIndexTask.TASK_TYPE, configs)); tableNumTasks++;
@Nonnull @Override public List<PinotTaskConfig> generateTasks(@Nonnull List<TableConfig> tableConfigs) { assertEquals(tableConfigs.size(), 2); // Generate at most 2 tasks if (_clusterInfoProvider.getTaskStates(TASK_TYPE).size() >= 2) { return Collections.emptyList(); } List<PinotTaskConfig> taskConfigs = new ArrayList<>(); for (TableConfig tableConfig : tableConfigs) { Map<String, String> configs = new HashMap<>(); configs.put("tableName", tableConfig.getTableName()); configs.put("tableType", tableConfig.getTableType().toString()); taskConfigs.add(new PinotTaskConfig(TASK_TYPE, configs)); } return taskConfigs; }
@Test public void testConvert() throws Exception { PurgeTaskExecutor purgeTaskExecutor = new PurgeTaskExecutor(); PinotTaskConfig pinotTaskConfig = new PinotTaskConfig(MinionConstants.PurgeTask.TASK_TYPE, Collections .singletonMap(MinionConstants.TABLE_NAME_KEY, TableNameBuilder.OFFLINE.tableNameWithType(TABLE_NAME))); File purgedIndexDir = purgeTaskExecutor.convert(pinotTaskConfig, _originalIndexDir, PURGED_SEGMENT_DIR).getFile(); try (PinotSegmentRecordReader pinotSegmentRecordReader = new PinotSegmentRecordReader(purgedIndexDir)) { int numRecordsRemaining = 0; int numRecordsModified = 0; GenericRow row = new GenericRow(); while (pinotSegmentRecordReader.hasNext()) { row = pinotSegmentRecordReader.next(row); numRecordsRemaining++; if (row.getValue(D1).equals(Integer.MAX_VALUE)) { numRecordsModified++; } } Assert.assertEquals(numRecordsRemaining, NUM_ROWS - 1); Assert.assertEquals(numRecordsModified, NUM_ROWS - 1); } }