public BaseRecordHandler(HdfsTaskWriter hdfsTaskWriter, TaskWriterContext taskWriterContext, HdfsWriterParameter hdfsWriterParameter) { this.hdfsTaskWriter = hdfsTaskWriter; this.hdfsWriterParameter = hdfsWriterParameter; this.hdfsFilePathGenerator = new HdfsFilePathGenerator(taskWriterContext, hdfsWriterParameter); this.fileStreamHolder = new FileStreamHolder(taskWriterContext.taskId(), hdfsWriterParameter); }
public RecordChunk<T> transform(RecordChunk<T> recordChunk, TaskWriterContext context) { Long taskId = Long.valueOf(context.taskId()); RecordChunk<T> newChunk = recordChunk.copyWithoutRecords(); for (T record : recordChunk.getRecords()) { T result = transformOne(record, RecordMeta.mediaMapping(record), context); if (result != null) { newChunk.merge(result); } } return newChunk; }
.getMediaMappingsByTask(Long.valueOf(context.taskId()), false) .stream() .filter(i -> parameter.getSupportedSourceTypes().contains(i.getTargetMediaSource().getType())) new SynchronousQueue<>(), new NamedThreadFactory( MessageFormat.format("Task-{0}-Writer-{1}-chunk", context.taskId(), parameter.getPluginName()) ), new ThreadPoolExecutor.CallerRunsPolicy() new LinkedBlockingQueue<>(), new NamedThreadFactory( MessageFormat.format("Task-{0}-Writer-{1}-load", context.taskId(), parameter.getPluginName()) ));
public void prePush() { context.beginSession(); context.taskWriterSession().setData(WriterStatistic.KEY, new WriterStatistic(context.taskId(), parameter)); }
private String getHbaseFilePath(String schemaName, String tableName, MediaMappingInfo mappingInfo) { FileSplitMode splitMode = getFileSplitMode(mappingInfo); String dateStr = getDateStr(); String taskFileName = tableName + "-" + taskWriterContext.taskId(); if ("default".equals(schemaName)) { if (splitMode.equals(FileSplitMode.DAY)) { return MessageFormat.format(FILE_PATH_HBASE_PATTERN_DAY,hdfsWriterParameter.getHbasePath(), tableName, dateStr, taskFileName); } else { return MessageFormat.format(FILE_PATH_HBASE_PATTERN_DAY_SPLIT,hdfsWriterParameter.getHbasePath(), tableName, dateStr, buildSplitRange(splitMode), taskFileName); } } else { if (splitMode.equals(FileSplitMode.DAY)) { return MessageFormat.format(FILE_PATH_HBASE_NAMESPACE_PATTERN_DAY, hdfsWriterParameter.getHbasePath(),schemaName, tableName, dateStr, taskFileName); } else { return MessageFormat.format(FILE_PATH_HBASE_NAMESPACE_PATTERN_DAY_SPLIT, hdfsWriterParameter.getHbasePath(),schemaName, tableName, dateStr, buildSplitRange(splitMode), taskFileName); } } }
Long taskId = Long.valueOf(context.taskId()); RecordChunk<T> newChunk = recordChunk.copyWithoutRecords();