public CompactionFailedTask (TaskContext taskContext) { super(taskContext); this.suite = CompactionSuiteUtils.getCompactionSuiteFactory (taskContext.getTaskState()). createSuite(taskContext.getTaskState()); this.dataset = this.suite.load(taskContext.getTaskState()); this.failedReason = taskContext.getTaskState().getProp(CompactionVerifier.COMPACTION_VERIFICATION_FAIL_REASON); }
private Task createTaskRunnable(WorkUnitState workUnitState, CountDownLatch countDownLatch) { Optional<TaskFactory> taskFactoryOpt = TaskUtils.getTaskFactory(workUnitState); if (taskFactoryOpt.isPresent()) { return new TaskIFaceWrapper(taskFactoryOpt.get().createTask(new TaskContext(workUnitState)), new TaskContext(workUnitState), countDownLatch, this.taskStateTracker); } else { return new Task(new TaskContext(workUnitState), this.taskStateTracker, this.taskExecutor, Optional.of(countDownLatch)); } }
Optional<CountDownLatch> countDownLatch) { this.taskContext = context; this.taskState = context.getTaskState(); this.jobId = this.taskState.getJobId(); this.taskId = this.taskState.getTaskId(); this.closer.register(this.taskState.getTaskBrokerNullable()); this.extractor = closer.register(new InstrumentedExtractorDecorator<>(this.taskState, this.taskContext.getExtractor())); this.recordStreamProcessors = this.taskContext.getRecordStreamProcessors(); List<Converter<?,?,?,?>> converters = this.taskContext.getConverters(); this.rowChecker = closer.register(this.taskContext.getRowLevelPolicyChecker()); } catch (Exception e) { try { Extractor underlyingExtractor = this.taskContext.getRawSourceExtractor(); if (!(underlyingExtractor instanceof StreamingExtractor)) { LOG.error( this.watermarkStorage = Optional.of(taskContext.getWatermarkStorage()); Config config; try {
@Test public void testOtherMethods() { Assert.assertTrue(this.taskContext.getSource() instanceof TestSource); Assert.assertEquals(this.taskContext.getStatusReportingInterval(), ConfigurationKeys.DEFAULT_TASK_STATUS_REPORT_INTERVAL_IN_MS); Assert.assertEquals(this.taskContext.getDestinationType(1, 0), Destination.DestinationType.HDFS); Assert.assertEquals(this.taskContext.getWriterOutputFormat(1, 0), WriterOutputFormat.AVRO); Assert.assertTrue(this.taskContext.getConverters().isEmpty()); } }
/** * Build a {@link org.apache.gobblin.writer.DataWriter} for writing fetched data records. */ private DataWriter<Object> buildWriter() throws IOException { DataWriterBuilder<Object, Object> builder = this.taskContext.getDataWriterBuilder(this.branches, this.index) .writeTo(Destination.of(this.taskContext.getDestinationType(this.branches, this.index), this.taskState)) .writeInFormat(this.taskContext.getWriterOutputFormat(this.branches, this.index)).withWriterId(this.taskId) .withSchema(this.convertedSchema.orNull()).withBranches(this.branches).forBranch(this.index); if (this.taskAttemptId.isPresent()) { builder.withAttemptId(this.taskAttemptId.get()); } DataWriter<Object> writer = new PartitionedDataWriter<>(builder, this.taskContext.getTaskState()); logger.info("Wrapping writer " + writer); return new DataWriterWrapperBuilder<>(writer, this.taskState).build(); }
private TaskContext getMockTaskContext(ArrayList<Object> recordCollector, Extractor mockExtractor) throws Exception { TaskState taskState = getStreamingTaskState(); // Create a mock RowLevelPolicyChecker RowLevelPolicyChecker mockRowLevelPolicyChecker = new RowLevelPolicyChecker(Lists.newArrayList(), "stateId", FileSystem.getLocal(new Configuration())); WatermarkStorage mockWatermarkStorage = new MockWatermarkStorage(); // Create a mock TaskPublisher TaskPublisher mockTaskPublisher = mock(TaskPublisher.class); when(mockTaskPublisher.canPublish()).thenReturn(TaskPublisher.PublisherState.SUCCESS); // Create a mock TaskContext TaskContext mockTaskContext = mock(TaskContext.class); when(mockTaskContext.getTaskMetrics()).thenReturn(TaskMetrics.get(taskState)); when(mockTaskContext.getExtractor()).thenReturn(mockExtractor); when(mockTaskContext.getRawSourceExtractor()).thenReturn(mockExtractor); when(mockTaskContext.getWatermarkStorage()).thenReturn(mockWatermarkStorage); when(mockTaskContext.getForkOperator()).thenReturn(new IdentityForkOperator()); when(mockTaskContext.getTaskState()).thenReturn(taskState); when(mockTaskContext.getTaskPublisher(any(TaskState.class), any(TaskLevelPolicyCheckResults.class))) .thenReturn(mockTaskPublisher); when(mockTaskContext.getRowLevelPolicyChecker()).thenReturn(mockRowLevelPolicyChecker); when(mockTaskContext.getRowLevelPolicyChecker(anyInt())).thenReturn(mockRowLevelPolicyChecker); when(mockTaskContext.getTaskLevelPolicyChecker(any(TaskState.class), anyInt())).thenReturn(mock(TaskLevelPolicyChecker.class)); when(mockTaskContext.getDataWriterBuilder(anyInt(), anyInt())).thenReturn(new TestStreamingDataWriterBuilder(recordCollector)); return mockTaskContext; }
when(mockTaskContext.getExtractor()).thenReturn(extractor); when(mockTaskContext.getForkOperator()).thenReturn(new IdentityForkOperator()); when(mockTaskContext.getTaskState()).thenReturn(taskState); when(mockTaskContext.getConverters()).thenReturn(converters); when(mockTaskContext.getRecordStreamProcessors()).thenReturn(recordStreamProcessors); when(mockTaskContext.getTaskLevelPolicyChecker(any(TaskState.class), anyInt())) .thenReturn(mock(TaskLevelPolicyChecker.class)); when(mockTaskContext.getRowLevelPolicyChecker()). thenReturn(new RowLevelPolicyChecker(Lists.newArrayList(), "ss", FileSystem.getLocal(new Configuration()))); when(mockTaskContext.getRowLevelPolicyChecker(anyInt())). thenReturn(new RowLevelPolicyChecker(Lists.newArrayList(), "ss", FileSystem.getLocal(new Configuration()))); when(mockTaskContext.getDataWriterBuilder(anyInt(), anyInt())).thenReturn(writer); when(mockTaskContext.getTaskPublisher(any(TaskState.class), any(TaskLevelPolicyCheckResults.class))) .thenReturn(mockTaskPublisher);
when(mockTaskContext.getTaskMetrics()).thenReturn(TaskMetrics.get(taskState)); when(mockTaskContext.getExtractor()).thenReturn(new FailOnceExtractor()); when(mockTaskContext.getForkOperator()).thenReturn(new IdentityForkOperator()); when(mockTaskContext.getTaskState()).thenReturn(taskState); when(mockTaskContext.getTaskLevelPolicyChecker(any(TaskState.class), anyInt())) .thenReturn(mock(TaskLevelPolicyChecker.class)); when(mockTaskContext.getRowLevelPolicyChecker()). thenReturn(new RowLevelPolicyChecker(Lists.newArrayList(), "ss", FileSystem.getLocal(new Configuration()))); when(mockTaskContext.getRowLevelPolicyChecker(anyInt())). thenReturn(new RowLevelPolicyChecker(Lists.newArrayList(), "ss", FileSystem.getLocal(new Configuration()))); when(mockTaskContext.getTaskPublisher(any(TaskState.class), any(TaskLevelPolicyCheckResults.class))) .thenReturn(mockTaskPublisher);
this.taskState = this.taskContext.getTaskState(); this.broker = this.taskState.getTaskBrokerNullable(); this.closer.register(new MultiConverter(this.taskContext.getConverters(this.index, this.forkTaskState))); this.convertedSchema = Optional.fromNullable(this.converter.convertSchema(schema, this.taskState)); this.rowLevelPolicyChecker = this.closer.register(this.taskContext.getRowLevelPolicyChecker(this.index)); this.rowLevelPolicyCheckingResult = new RowLevelPolicyCheckResults(); .get(getForkMetricsName(taskContext.getTaskMetrics(), this.taskState, index), taskContext.getTaskMetrics().getMetricContext(), getForkMetricsTags(this.taskState, index)); this.closer.register(forkMetrics.getMetricContext()); Instrumented.setMetricContextName(this.taskState, forkMetrics.getMetricContext().getName());
@Deprecated private void runSynchronousModel() throws Exception { ForkOperator forkOperator = closer.register(this.taskContext.getForkOperator()); forkOperator.init(this.taskState); int branches = forkOperator.getBranches(this.taskState); ((StreamingExtractor) this.taskContext.getRawSourceExtractor()).start(this.watermarkStorage.get());
/** * Get the list of pre-fork {@link RecordStreamProcessor}s. * * @return list (possibly empty) of {@link RecordStreamProcessor}s */ public List<RecordStreamProcessor<?, ?, ?, ?>> getRecordStreamProcessors() { return getRecordStreamProcessors(-1, this.taskState); }
/** * Get a pre-fork {@link RowLevelPolicyChecker} for executing row-level * {@link org.apache.gobblin.qualitychecker.row.RowLevelPolicy}. * * @return a {@link RowLevelPolicyChecker} */ public RowLevelPolicyChecker getRowLevelPolicyChecker() throws Exception { return getRowLevelPolicyChecker(-1); }
/** * Get the list of pre-fork {@link Converter}s. * * @return list (possibly empty) of {@link Converter}s */ public List<Converter<?, ?, ?, ?>> getConverters() { return getConverters(-1, this.taskState); }
private TaskContext getMockTaskContext(TaskState taskState, Extractor mockExtractor, ArrayList<ArrayList<Object>> writerCollectors, ForkOperator mockForkOperator) throws Exception { int numForks = writerCollectors.size(); // Create a mock RowLevelPolicyChecker RowLevelPolicyChecker mockRowLevelPolicyChecker = spy(new RowLevelPolicyChecker(Lists.newArrayList(), "ss", FileSystem.getLocal(new Configuration()))); when(mockRowLevelPolicyChecker.executePolicies(any(Object.class), any(RowLevelPolicyCheckResults.class))) .thenReturn(true); when(mockRowLevelPolicyChecker.getFinalState()).thenReturn(new State()); // Create a mock TaskPublisher TaskPublisher mockTaskPublisher = mock(TaskPublisher.class); when(mockTaskPublisher.canPublish()).thenReturn(TaskPublisher.PublisherState.SUCCESS); // Create a mock TaskContext TaskContext mockTaskContext = mock(TaskContext.class); when(mockTaskContext.getExtractor()).thenReturn(mockExtractor); when(mockTaskContext.getRawSourceExtractor()).thenReturn(mockExtractor); when(mockTaskContext.getForkOperator()).thenReturn(mockForkOperator); when(mockTaskContext.getTaskState()).thenReturn(taskState); when(mockTaskContext.getTaskPublisher(any(TaskState.class), any(TaskLevelPolicyCheckResults.class))) .thenReturn(mockTaskPublisher); when(mockTaskContext.getRowLevelPolicyChecker()).thenReturn(mockRowLevelPolicyChecker); when(mockTaskContext.getRowLevelPolicyChecker(anyInt())).thenReturn(mockRowLevelPolicyChecker); when(mockTaskContext.getTaskLevelPolicyChecker(any(TaskState.class), anyInt())).thenReturn(mock(TaskLevelPolicyChecker.class)); for (int i =0; i < numForks; ++i) { when(mockTaskContext.getDataWriterBuilder(numForks, i)).thenReturn(new RecordCollectingWriterBuilder(writerCollectors.get(i))); } return mockTaskContext; }
when(mockTaskContext.getExtractor()).thenReturn(extractor); when(mockTaskContext.getForkOperator()).thenReturn(new IdentityForkOperator()); when(mockTaskContext.getTaskState()).thenReturn(taskState); when(mockTaskContext.getConverters()).thenReturn(Lists.newArrayList(converter)); when(mockTaskContext.getTaskLevelPolicyChecker(any(TaskState.class), anyInt())) .thenReturn(mock(TaskLevelPolicyChecker.class)); when(mockTaskContext.getRowLevelPolicyChecker()). thenReturn(new RowLevelPolicyChecker(Lists.newArrayList(), "ss", FileSystem.getLocal(new Configuration()))); when(mockTaskContext.getRowLevelPolicyChecker(anyInt())). thenReturn(new RowLevelPolicyChecker(Lists.newArrayList(), "ss", FileSystem.getLocal(new Configuration()))); when(mockTaskContext.getDataWriterBuilder(anyInt(), anyInt())).thenReturn(writer); when(mockTaskContext.getTaskPublisher(any(TaskState.class), any(TaskLevelPolicyCheckResults.class))) .thenReturn(mockTaskPublisher);
/** * Build a {@link org.apache.gobblin.writer.DataWriter} for writing fetched data records. */ private DataWriter<Object> buildWriter() throws IOException { DataWriterBuilder<Object, Object> builder = this.taskContext.getDataWriterBuilder(this.branches, this.index) .writeTo(Destination.of(this.taskContext.getDestinationType(this.branches, this.index), this.taskState)) .writeInFormat(this.taskContext.getWriterOutputFormat(this.branches, this.index)).withWriterId(this.taskId) .withSchema(this.convertedSchema.orNull()).withBranches(this.branches).forBranch(this.index); if (this.taskAttemptId.isPresent()) { builder.withAttemptId(this.taskAttemptId.get()); } DataWriter<Object> writer = new PartitionedDataWriter<>(builder, this.taskContext.getTaskState()); logger.info("Wrapping writer " + writer); return new DataWriterWrapperBuilder<>(writer, this.taskState).build(); }
this.taskState = this.taskContext.getTaskState(); this.broker = this.taskState.getTaskBrokerNullable(); this.closer.register(new MultiConverter(this.taskContext.getConverters(this.index, this.forkTaskState))); this.convertedSchema = Optional.fromNullable(this.converter.convertSchema(schema, this.taskState)); this.rowLevelPolicyChecker = this.closer.register(this.taskContext.getRowLevelPolicyChecker(this.index)); this.rowLevelPolicyCheckingResult = new RowLevelPolicyCheckResults(); .get(getForkMetricsName(taskContext.getTaskMetrics(), this.taskState, index), taskContext.getTaskMetrics().getMetricContext(), getForkMetricsTags(this.taskState, index)); this.closer.register(forkMetrics.getMetricContext()); Instrumented.setMetricContextName(this.taskState, forkMetrics.getMetricContext().getName());
ForkOperator forkOperator = closer.register(this.taskContext.getForkOperator()); ((StreamingExtractor) this.taskContext.getRawSourceExtractor()).start(this.watermarkStorage.get());
/** * Get the list of pre-fork {@link RecordStreamProcessor}s. * * @return list (possibly empty) of {@link RecordStreamProcessor}s */ public List<RecordStreamProcessor<?, ?, ?, ?>> getRecordStreamProcessors() { return getRecordStreamProcessors(-1, this.taskState); }
/** * Get a pre-fork {@link RowLevelPolicyChecker} for executing row-level * {@link org.apache.gobblin.qualitychecker.row.RowLevelPolicy}. * * @return a {@link RowLevelPolicyChecker} */ public RowLevelPolicyChecker getRowLevelPolicyChecker() throws Exception { return getRowLevelPolicyChecker(-1); }