/** * Initialize this TaskWriter with the specified context object. */ @SuppressWarnings("unchecked") public void initialize(TaskWriterContext context) { this.context = context; this.parameter = (T) context.getWriterParameter(); this.handlers = CacheBuilder.newBuilder().build(new CacheLoader<Class<? extends Record>, Handler>() { @Override public Handler load(Class<? extends Record> clazz) throws Exception { Handler handler = getHandler(clazz); if (handler == null) { throw new RecordNotSupportException(clazz); } handler.initialize(context); return handler; } }); }
private void loadInGlobalMode(List<RdbEventRecord> records, TaskWriterContext context) { boolean originalUseBatch = context.getWriterParameter().isUseBatch(); try { context.getWriterParameter().setUseBatch(false);//禁用batch模式,因为按表分组并没有根据相同sql做聚合 List<List<RdbEventRecord>> list = new ArrayList<>(); list.add(records); doHandle(context, list); } finally { context.getWriterParameter().setUseBatch(originalUseBatch);//结束时把useBatch设置为之前的值,避免影响其它地方用 } }
private void loadDeletedRecords(Function<List<List<RdbEventRecord>>, Void> function) { List<List<RdbEventRecord>> batchRecords = new ArrayList<>(); for (RecordGroupHolder.TableLoadData tableData : this.tables) { if (context.getWriterParameter().isUseBatch()) { batchRecords.addAll(split(tableData.getDeleteDatas(), context.getWriterParameter().getBatchSize())); } else { for (RdbEventRecord data : tableData.getDeleteDatas()) { batchRecords.add(Arrays.asList(data)); } } } function.apply(batchRecords); }
public void loadByGroup(Function<List<List<RdbEventRecord>>, Void> function) { if (context.getWriterParameter().isMerging()) { // 优先执行delete语句,针对uniqe更新,一般会进行delete + insert的处理模式,避免并发更新 loadDeletedRecords(function); loadInsertAndUpdatedRecords(function); } else { boolean originalUseBatch = context.getWriterParameter().isUseBatch(); try { context.getWriterParameter().setUseBatch(false);//禁用batch模式,因为按表分组并没有根据相同sql做聚合 loadRecordsByTable(function); } finally { context.getWriterParameter().setUseBatch(originalUseBatch);//结束时把useBatch设置为之前的值,避免影响其它地方用 } } }
private int loadOne(TaskWriterContext context, T record, DbDialect dbDialect, LobCreator lobCreator, JdbcTemplate template) { return template.update(getSql(record), ps -> { fillPreparedStatement(ps, lobCreator, record, dbDialect, context.getWriterParameter().isSyncAutoAddColumn()); }); }
private void loadInsertAndUpdatedRecords(Function<List<List<RdbEventRecord>>, Void> function) { List<List<RdbEventRecord>> batchRecords = new ArrayList<>(); for (RecordGroupHolder.TableLoadData tableData : this.tables) { if (context.getWriterParameter().isUseBatch()) { batchRecords.addAll(split(tableData.getInsertDatas(), context.getWriterParameter().getBatchSize())); batchRecords.addAll(split(tableData.getUpdateDatas(), context.getWriterParameter().getBatchSize())); } else { for (RdbEventRecord data : tableData.getInsertDatas()) { batchRecords.add(Arrays.asList(data)); } for (RdbEventRecord data : tableData.getUpdateDatas()) { batchRecords.add(Arrays.asList(data)); } } } function.apply(batchRecords); }
.forEach(i -> results.add(executorService.submit(() -> { if (context.getWriterParameter().isUseBatch()) { List<List<RdbEventRecord>> list = BatchSplitter.splitForBatch(i.getValue(), context.getWriterParameter().getBatchSize()); for (List<RdbEventRecord> item : list) { writeData(item);
.forEach(i -> results.add(executorService.submit(() -> { if (context.getWriterParameter().isUseBatch()) { List<List<HRecord>> list = BatchSplitter.splitForBatch(i.getValue(), context.getWriterParameter().getBatchSize()); for (List<HRecord> item : list) { writeData(item);
private void doHandle(TaskWriterContext context, List<List<RdbEventRecord>> records) { if (context.getWriterParameter().isDryRun()) { doDryRun(context, records); } else { doLoad(context, records); } }
private LoadResult doCall(TaskWriterContext context, List<T> records, DbDialect dbDialect) { if (context.getWriterParameter().isUseBatch()) { try { return loadInBatch(context, records, dbDialect); } catch (Throwable t) { //批量模式出错,转为single模式,保证: //1.尽可能把不报错的数据写入到目标库 //2.改为single模式,才能知道具体出错的record是哪一条 logger.error("something goes wrong when load records with batch, now try to load with single.", t); return loadInSingle(context, records, dbDialect); } } else { return loadInSingle(context, records, dbDialect); } }
@Override public void initialize(TaskWriterContext context) { PluginWriterParameter parameter = context.getWriterParameter();
private void groupForOneRecord(RdbEventRecord record) { TableLoadData tableData = findTableData(record); if (context.getWriterParameter().isMerging()) { EventType type = record.getEventType(); if (type.isInsert()) { tableData.getInsertDatas().add(record); } else if (type.isUpdate()) { tableData.getUpdateDatas().add(record); } else if (type.isDelete()) { tableData.getDeleteDatas().add(record); } } else { tableData.getTableDatas().add(record); } }
@Override public void doWrite(List<RdbEventRecord> records, TaskWriterContext context) { buildSql(records, context); RdbmsWriterParameter parameter = (RdbmsWriterParameter) context.getWriterParameter(); if (parameter.getSyncMode() == RdbmsWriterParameter.SyncMode.TablePartlyOrdered) { loadInTableMode(records, context); } else if (parameter.getSyncMode() == RdbmsWriterParameter.SyncMode.DbGlobalOrdered) { loadInGlobalMode(records, context); } else { throw new UnsupportedOperationException("Invalid SyncMode " + parameter.getSyncMode()); } }
MediaSourceInfo targetMediaSourceInfo = mappingInfo.getTargetMediaSource(); HTable hTable = HTableFactory.getHTable(tableName, targetMediaSourceInfo); List<List<RdbEventRecord>> batchRecords = BatchSplitter.splitForBatch(mr.getValue(), context.getWriterParameter().getBatchSize()); for (List<RdbEventRecord> recordList : batchRecords) { writeToHBase(recordList, hTable, hBaseSyncParameter != null && hBaseSyncParameter.isSyncDelete());
MediaMappingInfo mappingInfo = RecordMeta.mediaMapping(mr.getValue().get(0)); List<List<T>> list = BatchSplitter.splitForBatch(mr.getValue(), context.getWriterParameter().getBatchSize()); for (List<T> item : list) { Map<String, List<String>> datas = buildData(item, context);
/** * 需要的话,可以对数据进行merge操作 */ protected RecordChunk<T> merge(RecordChunk<T> recordChunk, TaskWriterContext context) { //statistic before WriterStatistic writerStatistic = context.taskWriterSession().getData(WriterStatistic.KEY); writerStatistic.setRecordsCountBeforeMerge(recordChunk.getRecords().size()); Long startTime = System.currentTimeMillis(); //do merge if (context.getWriterParameter().isMerging()) { Merger<T> merger = MergerFactory. getMerger(recordChunk.getRecords().get(0).getClass()); RecordChunk<T> newChunk = merger.merge(recordChunk); //statistic after writerStatistic.setTimeForMerge(System.currentTimeMillis() - startTime); writerStatistic.setRecordsCountAfterMerge(newChunk.getRecords().size()); return newChunk; } else { //statistic after writerStatistic.setTimeForMerge(System.currentTimeMillis() - startTime); writerStatistic.setRecordsCountAfterMerge(recordChunk.getRecords().size()); return recordChunk; } }
List<MediaMappingInfo> list = context.getService(MediaService.class). getMediaMappingsByMediaAndTarget(taskId, record.RSI().getNamespace(), record.RSI().getName(), context.getWriterParameter().getSupportedSourceTypes(), true);