public CompactionFailedTask (TaskContext taskContext) { super(taskContext); this.suite = CompactionSuiteUtils.getCompactionSuiteFactory (taskContext.getTaskState()). createSuite(taskContext.getTaskState()); this.dataset = this.suite.load(taskContext.getTaskState()); this.failedReason = taskContext.getTaskState().getProp(CompactionVerifier.COMPACTION_VERIFICATION_FAIL_REASON); }
public TaskIFaceWrapper(TaskIFace underlyingTask, TaskContext taskContext, CountDownLatch countDownLatch, TaskStateTracker taskStateTracker) { super(); this.underlyingTask = underlyingTask; this.taskContext = taskContext; this.jobId = taskContext.getTaskState().getJobId(); this.taskId = taskContext.getTaskState().getTaskId(); this.countDownLatch = countDownLatch; this.taskStateTracker = taskStateTracker; }
public SleepingTask(TaskContext taskContext) { super(taskContext); sleepTime = taskContext.getTaskState().getPropAsLong("data.publisher.sleep.time.in.seconds", 10L); }
/** * Create the {@link Job} to run in this task. * @return the {@link Job} to run. If this method returns null, no job will be run and the task will be marked as successful. */ protected Job createJob() throws IOException { Job job = Job.getInstance(new Configuration()); for (Map.Entry<Object, Object> entry : this.taskContext.getTaskState().getProperties().entrySet()) { if (entry.getKey() instanceof String && ((String) entry.getKey()).startsWith(JOB_CONFIGURATION_PREFIX)) { String actualKey = ((String) entry.getKey()).substring(JOB_CONFIGURATION_PREFIX.length()); job.getConfiguration().set(actualKey, (String) entry.getValue()); } } return job; }
@Override public void run() { try { DatasetCleaner datasetCleaner = new DatasetCleaner(FileSystem.get(new Configuration()), this.taskContext.getTaskState().getProperties()); datasetCleaner.clean(); } catch (IOException e) { throw new RuntimeException(e); } } }
public BaseAbstractTask(TaskContext taskContext) { this.metricContext = TaskMetrics.get(taskContext.getTaskState()).getMetricContext(); }
protected boolean areSingleBranchTasksSynchronous(TaskContext taskContext) { return BooleanUtils.toBoolean(taskContext.getTaskState() .getProp(TaskConfigurationKeys.TASK_IS_SINGLE_BRANCH_SYNCHRONOUS, TaskConfigurationKeys.DEFAULT_TASK_IS_SINGLE_BRANCH_SYNCHRONOUS)); }
/** * Constructor */ public MRCompactionTask(TaskContext taskContext) throws IOException { super(taskContext); this.suite = CompactionSuiteUtils.getCompactionSuiteFactory (taskContext.getTaskState()). createSuite(taskContext.getTaskState()); this.dataset = this.suite.load(taskContext.getTaskState()); this.eventSubmitter = new EventSubmitter.Builder(this.metricContext, MRCompactor.COMPACTION_TRACKING_EVENTS_NAMESPACE) .addMetadata(additionalEventMetadata()).build(); }
private void onStart(long startTime) { Long queueTime = queuedTasks.remove(this.underlyingTask.getTaskId()); long workUnitCreationTime = this.underlyingTask.getTaskContext().getTaskState().getPropAsLong(ConfigurationKeys.WORK_UNIT_CREATION_TIME_IN_MILLIS, 0); long timeInQueue = startTime - queueTime; long timeSinceWorkUnitCreation = startTime - workUnitCreationTime; taskCreateAndRunTimer.update(timeSinceWorkUnitCreation, TimeUnit.MILLISECONDS); LOG.debug(String.format("Task %s started. Saving queued time of %d ms to history.", underlyingTask.getTaskId(), timeInQueue)); queuedTaskTimeHistorical.putIfAbsent(System.currentTimeMillis(), timeInQueue); runningTaskCount.inc(); } }
public HiveTask(TaskContext taskContext) { super(taskContext); this.taskContext = taskContext; this.workUnitState = taskContext.getTaskState(); this.workUnit = new HiveWorkUnit(this.workUnitState.getWorkunit()); this.eventSubmitter = new EventSubmitter.Builder(this.metricContext, "gobblin.HiveTask") .build(); this.hiveExecutionQueries = Lists.newArrayList(); this.publishEntity = new QueryBasedHivePublishEntity(); try { this.hiveJdbcConnector = HiveJdbcConnector.newConnectorWithProps(this.workUnitState.getProperties()); } catch (SQLException se) { throw new RuntimeException("Error in creating JDBC Connector", se); } this.addFiles = this.workUnitState.getPropAsList(ADD_FILES, ""); this.addJars = this.workUnitState.getPropAsList(ADD_JARS, ""); this.setupQueries = Splitter.on(";").trimResults().omitEmptyStrings().splitToList(this.workUnitState.getProp(SETUP_QUERIES, "")); }
@Override public TaskIFace createTask(TaskContext taskContext) { String taskId = taskContext.getTaskState().getProp(TASK_ID_KEY); EventBus eventBus = null; if (taskContext.getTaskState().contains(EVENTBUS_ID_KEY)) { String eventbusId = taskContext.getTaskState().getProp(EVENTBUS_ID_KEY); eventBus = TestingEventBuses.getEventBus(eventbusId); } return new Task(taskContext, taskId, eventBus); }
/** * Build a {@link org.apache.gobblin.writer.DataWriter} for writing fetched data records. */ private DataWriter<Object> buildWriter() throws IOException { DataWriterBuilder<Object, Object> builder = this.taskContext.getDataWriterBuilder(this.branches, this.index) .writeTo(Destination.of(this.taskContext.getDestinationType(this.branches, this.index), this.taskState)) .writeInFormat(this.taskContext.getWriterOutputFormat(this.branches, this.index)).withWriterId(this.taskId) .withSchema(this.convertedSchema.orNull()).withBranches(this.branches).forBranch(this.index); if (this.taskAttemptId.isPresent()) { builder.withAttemptId(this.taskAttemptId.get()); } DataWriter<Object> writer = new PartitionedDataWriter<>(builder, this.taskContext.getTaskState()); logger.info("Wrapping writer " + writer); return new DataWriterWrapperBuilder<>(writer, this.taskState).build(); }
private TaskContext getMockTaskContext(TaskState taskState, Extractor mockExtractor, ArrayList<ArrayList<Object>> writerCollectors, ForkOperator mockForkOperator) throws Exception { int numForks = writerCollectors.size(); // Create a mock RowLevelPolicyChecker RowLevelPolicyChecker mockRowLevelPolicyChecker = spy(new RowLevelPolicyChecker(Lists.newArrayList(), "ss", FileSystem.getLocal(new Configuration()))); when(mockRowLevelPolicyChecker.executePolicies(any(Object.class), any(RowLevelPolicyCheckResults.class))) .thenReturn(true); when(mockRowLevelPolicyChecker.getFinalState()).thenReturn(new State()); // Create a mock TaskPublisher TaskPublisher mockTaskPublisher = mock(TaskPublisher.class); when(mockTaskPublisher.canPublish()).thenReturn(TaskPublisher.PublisherState.SUCCESS); // Create a mock TaskContext TaskContext mockTaskContext = mock(TaskContext.class); when(mockTaskContext.getExtractor()).thenReturn(mockExtractor); when(mockTaskContext.getRawSourceExtractor()).thenReturn(mockExtractor); when(mockTaskContext.getForkOperator()).thenReturn(mockForkOperator); when(mockTaskContext.getTaskState()).thenReturn(taskState); when(mockTaskContext.getTaskPublisher(any(TaskState.class), any(TaskLevelPolicyCheckResults.class))) .thenReturn(mockTaskPublisher); when(mockTaskContext.getRowLevelPolicyChecker()).thenReturn(mockRowLevelPolicyChecker); when(mockTaskContext.getRowLevelPolicyChecker(anyInt())).thenReturn(mockRowLevelPolicyChecker); when(mockTaskContext.getTaskLevelPolicyChecker(any(TaskState.class), anyInt())).thenReturn(mock(TaskLevelPolicyChecker.class)); for (int i =0; i < numForks; ++i) { when(mockTaskContext.getDataWriterBuilder(numForks, i)).thenReturn(new RecordCollectingWriterBuilder(writerCollectors.get(i))); } return mockTaskContext; }
when(mockTaskContext.getExtractor()).thenReturn(extractor); when(mockTaskContext.getForkOperator()).thenReturn(new IdentityForkOperator()); when(mockTaskContext.getTaskState()).thenReturn(taskState); when(mockTaskContext.getConverters()).thenReturn(converters); when(mockTaskContext.getRecordStreamProcessors()).thenReturn(recordStreamProcessors);
private TaskContext getMockTaskContext(ArrayList<Object> recordCollector, Extractor mockExtractor) throws Exception { TaskState taskState = getStreamingTaskState(); // Create a mock RowLevelPolicyChecker RowLevelPolicyChecker mockRowLevelPolicyChecker = new RowLevelPolicyChecker(Lists.newArrayList(), "stateId", FileSystem.getLocal(new Configuration())); WatermarkStorage mockWatermarkStorage = new MockWatermarkStorage(); // Create a mock TaskPublisher TaskPublisher mockTaskPublisher = mock(TaskPublisher.class); when(mockTaskPublisher.canPublish()).thenReturn(TaskPublisher.PublisherState.SUCCESS); // Create a mock TaskContext TaskContext mockTaskContext = mock(TaskContext.class); when(mockTaskContext.getTaskMetrics()).thenReturn(TaskMetrics.get(taskState)); when(mockTaskContext.getExtractor()).thenReturn(mockExtractor); when(mockTaskContext.getRawSourceExtractor()).thenReturn(mockExtractor); when(mockTaskContext.getWatermarkStorage()).thenReturn(mockWatermarkStorage); when(mockTaskContext.getForkOperator()).thenReturn(new IdentityForkOperator()); when(mockTaskContext.getTaskState()).thenReturn(taskState); when(mockTaskContext.getTaskPublisher(any(TaskState.class), any(TaskLevelPolicyCheckResults.class))) .thenReturn(mockTaskPublisher); when(mockTaskContext.getRowLevelPolicyChecker()).thenReturn(mockRowLevelPolicyChecker); when(mockTaskContext.getRowLevelPolicyChecker(anyInt())).thenReturn(mockRowLevelPolicyChecker); when(mockTaskContext.getTaskLevelPolicyChecker(any(TaskState.class), anyInt())).thenReturn(mock(TaskLevelPolicyChecker.class)); when(mockTaskContext.getDataWriterBuilder(anyInt(), anyInt())).thenReturn(new TestStreamingDataWriterBuilder(recordCollector)); return mockTaskContext; }
public AsynchronousFork(TaskContext taskContext, Object schema, int branches, int index, ExecutionModel executionModel) throws Exception { super(taskContext, schema, branches, index, executionModel); TaskState taskState = taskContext.getTaskState(); this.recordQueue = BoundedBlockingRecordQueue.newBuilder() .hasCapacity(taskState.getPropAsInt( ConfigurationKeys.FORK_RECORD_QUEUE_CAPACITY_KEY, ConfigurationKeys.DEFAULT_FORK_RECORD_QUEUE_CAPACITY)) .useTimeout(taskState.getPropAsLong( ConfigurationKeys.FORK_RECORD_QUEUE_TIMEOUT_KEY, ConfigurationKeys.DEFAULT_FORK_RECORD_QUEUE_TIMEOUT)) .useTimeoutTimeUnit(TimeUnit.valueOf(taskState.getProp( ConfigurationKeys.FORK_RECORD_QUEUE_TIMEOUT_UNIT_KEY, ConfigurationKeys.DEFAULT_FORK_RECORD_QUEUE_TIMEOUT_UNIT))) .collectStats() .build(); }
when(mockTaskContext.getExtractor()).thenReturn(new FailOnceExtractor()); when(mockTaskContext.getForkOperator()).thenReturn(new IdentityForkOperator()); when(mockTaskContext.getTaskState()).thenReturn(taskState); when(mockTaskContext.getTaskLevelPolicyChecker(any(TaskState.class), anyInt())) .thenReturn(mock(TaskLevelPolicyChecker.class));
this.taskState = this.taskContext.getTaskState(); this.broker = this.taskState.getTaskBrokerNullable();
when(mockTaskContext.getExtractor()).thenReturn(extractor); when(mockTaskContext.getForkOperator()).thenReturn(new IdentityForkOperator()); when(mockTaskContext.getTaskState()).thenReturn(taskState); when(mockTaskContext.getConverters()).thenReturn(Lists.newArrayList(converter)); when(mockTaskContext.getTaskLevelPolicyChecker(any(TaskState.class), anyInt()))
@Override public TaskState getTaskState() { TaskState taskState = this.taskContext.getTaskState(); taskState.setTaskId(getTaskId()); taskState.setJobId(getJobId()); taskState.setWorkingState(getWorkingState()); taskState.addAll(getExecutionMetadata()); taskState.addAll(getPersistentState()); return taskState; }