/** * This method does not actually move data, but it creates an {@link FsRenameCommitStep}. */ @Override protected void movePath(ParallelRunner parallelRunner, State state, Path src, Path dst, int branchId) throws IOException { log.info(String.format("Creating CommitStep for moving %s to %s", src, dst)); boolean overwrite = state.getPropAsBoolean(ConfigurationKeys.DATA_PUBLISHER_OVERWRITE_ENABLED, false); FsRenameCommitStep.Builder<?> builder = this.commitSequenceBuilder.get().beginStep(FsRenameCommitStep.Builder.class) .withProps(this.state).from(src).withSrcFs(this.writerFileSystemByBranches.get(branchId)).to(dst) .withDstFs(this.publisherFileSystemByBranches.get(branchId)); if (overwrite) { builder.overwrite(); } builder.endStep(); }
@BeforeClass public void setUp() throws IOException { this.fs = FileSystem.getLocal(new Configuration()); this.fs.delete(new Path(ROOT_DIR), true); Path dir1 = new Path(ROOT_DIR, "dir1"); Path dir2 = new Path(ROOT_DIR, "dir2"); this.fs.mkdirs(dir1); this.fs.mkdirs(dir2); Path src = new Path(dir1, "file"); Path dst = new Path(dir2, "file"); this.fs.createNewFile(src); this.step = (FsRenameCommitStep) new FsRenameCommitStep.Builder<>().from(src).to(dst).withProps(new State()).build(); }
@BeforeClass public void setUp() throws IOException { this.fs = FileSystem.getLocal(new Configuration()); this.fs.delete(new Path(ROOT_DIR), true); Path storeRootDir = new Path(ROOT_DIR, "store"); Path dir1 = new Path(ROOT_DIR, "dir1"); Path dir2 = new Path(ROOT_DIR, "dir2"); this.fs.mkdirs(dir1); this.fs.mkdirs(dir2); Path src1 = new Path(dir1, "file1"); Path src2 = new Path(dir2, "file2"); Path dst1 = new Path(dir2, "file1"); Path dst2 = new Path(dir1, "file2"); this.fs.createNewFile(src1); this.fs.createNewFile(src2); DatasetState ds = new DatasetState("job-name", "job-id"); ds.setDatasetUrn("urn"); ds.setNoJobFailure(); State state = new State(); state.setProp(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, storeRootDir.toString()); this.sequence = new CommitSequence.Builder().withJobName("testjob").withDatasetUrn("testurn") .beginStep(FsRenameCommitStep.Builder.class).from(src1).to(dst1).withProps(state).endStep() .beginStep(FsRenameCommitStep.Builder.class).from(src2).to(dst2).withProps(state).endStep() .beginStep(DatasetStateCommitStep.Builder.class).withDatasetUrn("urn").withDatasetState(ds).withProps(state) .endStep().build(); }
/** * This method does not actually move data, but it creates an {@link FsRenameCommitStep}. */ @Override protected void movePath(ParallelRunner parallelRunner, State state, Path src, Path dst, int branchId) throws IOException { log.info(String.format("Creating CommitStep for moving %s to %s", src, dst)); boolean overwrite = state.getPropAsBoolean(ConfigurationKeys.DATA_PUBLISHER_OVERWRITE_ENABLED, false); FsRenameCommitStep.Builder<?> builder = this.commitSequenceBuilder.get().beginStep(FsRenameCommitStep.Builder.class) .withProps(this.state).from(src).withSrcFs(this.writerFileSystemByBranches.get(branchId)).to(dst) .withDstFs(this.publisherFileSystemByBranches.get(branchId)); if (overwrite) { builder.overwrite(); } builder.endStep(); }
@BeforeClass public void setUp() throws IOException { FileSystem fs = FileSystem.getLocal(new Configuration()); this.store = new FsCommitSequenceStore(fs, new Path("commit-sequence-store-test")); State props = new State(); props.setId("propsId"); props.setProp("prop1", "valueOfProp1"); props.setProp("prop2", "valueOfProp2"); DatasetState datasetState = new DatasetState(); datasetState.setDatasetUrn(this.datasetUrn); datasetState.incrementJobFailures(); this.sequence = new CommitSequence.Builder().withJobName("testjob").withDatasetUrn("testurn") .beginStep(FsRenameCommitStep.Builder.class).from(new Path("/ab/cd")).to(new Path("/ef/gh")).withProps(props) .endStep().beginStep(DatasetStateCommitStep.Builder.class).withDatasetUrn(this.datasetUrn) .withDatasetState(datasetState).withProps(props).endStep().build(); }