/** * Update destination path to put db and table name in format "dbname.tablename" using {@link #getDbTableName(String)} * and include timestamp * * Input dst format: {finaldir}/{schemaName} * Output dst format: {finaldir}/{dbname.tablename}/{currenttimestamp} */ @Override protected void movePath(ParallelRunner parallelRunner, State state, Path src, Path dst, int branchId) throws IOException { String outputDir = dst.getParent().toString(); String schemaName = dst.getName(); Path newDst = new Path(new Path(outputDir, getDbTableName(schemaName)), timestamp); if (!this.publisherFileSystemByBranches.get(branchId).exists(newDst)) { WriterUtils.mkdirsWithRecursivePermissionWithRetry(this.publisherFileSystemByBranches.get(branchId), newDst.getParent(), this.permissions.get(branchId), this.retrierConfig); } super.movePath(parallelRunner, state, src, newDst, branchId); }
/** * Make sure directory exists before running {@link BaseDataPublisher#publishData(WorkUnitState, int, boolean, Set)} * so that tables will be moved one at a time rather than all at once */ @Override protected void publishData(WorkUnitState state, int branchId, boolean publishSingleTaskData, Set<Path> writerOutputPathsMoved) throws IOException { Path publisherOutputDir = getPublisherOutputDir(state, branchId); if (!this.publisherFileSystemByBranches.get(branchId).exists(publisherOutputDir)) { WriterUtils.mkdirsWithRecursivePermissionWithRetry(this.publisherFileSystemByBranches.get(branchId), publisherOutputDir, this.permissions.get(branchId), this.retrierConfig); } super.publishData(state, branchId, publishSingleTaskData, writerOutputPathsMoved); }
@Override public void publish(Collection<? extends WorkUnitState> states) throws IOException { super.publish(states); // PUBLISHER_DIRS key must be updated for HiveRegistrationPublisher for (Path path : this.publisherOutputDirs) { this.state.appendToSetProp(ConfigurationKeys.PUBLISHER_DIRS, path.toString()); } this.hivePublisher.publish(states); } }
@Override public void publish(Collection<? extends WorkUnitState> states) throws IOException { super.publish(states); // PUBLISHER_DIRS key must be updated for HiveRegistrationPublisher for (Path path : this.publisherOutputDirs) { this.state.appendToSetProp(ConfigurationKeys.PUBLISHER_DIRS, path.toString()); } this.hivePublisher.publish(states); } }
/** * Make sure directory exists before running {@link BaseDataPublisher#publishData(WorkUnitState, int, boolean, Set)} * so that tables will be moved one at a time rather than all at once */ @Override protected void publishData(WorkUnitState state, int branchId, boolean publishSingleTaskData, Set<Path> writerOutputPathsMoved) throws IOException { Path publisherOutputDir = getPublisherOutputDir(state, branchId); if (!this.publisherFileSystemByBranches.get(branchId).exists(publisherOutputDir)) { WriterUtils.mkdirsWithRecursivePermissionWithRetry(this.publisherFileSystemByBranches.get(branchId), publisherOutputDir, this.permissions.get(branchId), this.retrierConfig); } super.publishData(state, branchId, publishSingleTaskData, writerOutputPathsMoved); }
/** * Update destination path to put db and table name in format "dbname.tablename" using {@link #getDbTableName(String)} * and include timestamp * * Input dst format: {finaldir}/{schemaName} * Output dst format: {finaldir}/{dbname.tablename}/{currenttimestamp} */ @Override protected void movePath(ParallelRunner parallelRunner, State state, Path src, Path dst, int branchId) throws IOException { String outputDir = dst.getParent().toString(); String schemaName = dst.getName(); Path newDst = new Path(new Path(outputDir, getDbTableName(schemaName)), timestamp); if (!this.publisherFileSystemByBranches.get(branchId).exists(newDst)) { WriterUtils.mkdirsWithRecursivePermissionWithRetry(this.publisherFileSystemByBranches.get(branchId), newDst.getParent(), this.permissions.get(branchId), this.retrierConfig); } super.movePath(parallelRunner, state, src, newDst, branchId); }