private void moveCustomLocationTaskOutputs(FileSystem fs, Table table, Configuration conf) throws IOException { // in case of custom dynamic partitions, we can't just move the sub-tree of partition root // directory since the partitions location contain regex pattern. We need to first find the // final destination of each partition and move its output. for (Entry<String, Map<String, String>> entry : partitionsDiscoveredByPath.entrySet()) { Path src = new Path(entry.getKey()); Path destPath = new Path(getFinalDynamicPartitionDestination(table, entry.getValue(), jobInfo)); moveTaskOutputs(fs, src, src, destPath, true, true); // dryRun = true, immutable = true moveTaskOutputs(fs, src, src, destPath, false, true); } // delete the parent temp directory of all custom dynamic partitions Path parentPath = new Path(getCustomPartitionRootLocation(jobInfo, conf)); if (fs.exists(parentPath)) { fs.delete(parentPath, true); } }
moveTaskOutputs(fs, child.getPath(), srcDir, destDir, dryRun, immutable); moveTaskOutputs(fs, child.getPath(), srcDir, destDir, dryRun, immutable);
moveTaskOutputs(fs, src, src, tblPath, false, table.isImmutable()); if (!src.equals(tblPath)) { fs.delete(src, true); moveTaskOutputs(fs, src, src, tblPath, true, true); // dryRun = true, immutable = true moveTaskOutputs(fs, src, src, tblPath, false, true); if (!src.equals(tblPath)){ fs.delete(src, true); Path src = new Path(jobInfo.getLocation()); Path dest = new Path(p.getSd().getLocation()); moveTaskOutputs(fs, src, src, dest, true, table.isImmutable()); moveTaskOutputs(fs,src,src,dest,false,table.isImmutable()); if (!src.equals(dest)){ if (src.toString().matches(".*" + Path.SEPARATOR + SCRATCH_DIR_NAME + "\\d\\.?\\d+.*")){ moveTaskOutputs(fs, src, src, tblPath, true, true); // dryRun = true, immutable = true moveTaskOutputs(fs, src, src, tblPath, false, true); if (!src.equals(tblPath)){ fs.delete(src, true);
private void moveCustomLocationTaskOutputs(FileSystem fs, Table table, Configuration conf) throws IOException { // in case of custom dynamic partitions, we can't just move the sub-tree of partition root // directory since the partitions location contain regex pattern. We need to first find the // final destination of each partition and move its output. for (Entry<String, Map<String, String>> entry : partitionsDiscoveredByPath.entrySet()) { Path src = new Path(entry.getKey()); Path destPath = new Path(getFinalDynamicPartitionDestination(table, entry.getValue(), jobInfo)); moveTaskOutputs(fs, src, src, destPath, true, true); // dryRun = true, immutable = true moveTaskOutputs(fs, src, src, destPath, false, true); } // delete the parent temp directory of all custom dynamic partitions Path parentPath = new Path(getCustomPartitionRootLocation(jobInfo, conf)); if (fs.exists(parentPath)) { fs.delete(parentPath, true); } }
private void moveCustomLocationTaskOutputs(FileSystem fs, Table table, Configuration conf) throws IOException { // in case of custom dynamic partitions, we can't just move the sub-tree of partition root // directory since the partitions location contain regex pattern. We need to first find the // final destination of each partition and move its output. for (Entry<String, Map<String, String>> entry : partitionsDiscoveredByPath.entrySet()) { Path src = new Path(entry.getKey()); Path destPath = new Path(getFinalDynamicPartitionDestination(table, entry.getValue(), jobInfo)); moveTaskOutputs(fs, src, src, destPath, true, true); // dryRun = true, immutable = true moveTaskOutputs(fs, src, src, destPath, false, true); } // delete the parent temp directory of all custom dynamic partitions Path parentPath = new Path(getCustomPartitionRootLocation(jobInfo, conf)); if (fs.exists(parentPath)) { fs.delete(parentPath, true); } }
private void moveCustomLocationTaskOutputs(FileSystem fs, Table table, Configuration conf) throws IOException { // in case of custom dynamic partitions, we can't just move the sub-tree of partition root // directory since the partitions location contain regex pattern. We need to first find the // final destination of each partition and move its output. for (Entry<String, Map<String, String>> entry : partitionsDiscoveredByPath.entrySet()) { Path src = new Path(entry.getKey()); Path destPath = new Path(getFinalDynamicPartitionDestination(table, entry.getValue(), jobInfo)); moveTaskOutputs(fs, src, src, destPath, true, true); // dryRun = true, immutable = true moveTaskOutputs(fs, src, src, destPath, false, true); } // delete the parent temp directory of all custom dynamic partitions Path parentPath = new Path(getCustomPartitionRootLocation(jobInfo, conf)); if (fs.exists(parentPath)) { fs.delete(parentPath, true); } }
private void moveCustomLocationTaskOutputs(FileSystem fs, Table table, Configuration conf) throws IOException { // in case of custom dynamic partitions, we can't just move the sub-tree of partition root // directory since the partitions location contain regex pattern. We need to first find the // final destination of each partition and move its output. for (Entry<String, Map<String, String>> entry : partitionsDiscoveredByPath.entrySet()) { Path src = new Path(entry.getKey()); Path destPath = new Path(getFinalDynamicPartitionDestination(table, entry.getValue(), jobInfo)); moveTaskOutputs(fs, src, src, destPath, true, true); // dryRun = true, immutable = true moveTaskOutputs(fs, src, src, destPath, false, true); } // delete the parent temp directory of all custom dynamic partitions Path parentPath = new Path(getCustomPartitionRootLocation(jobInfo, conf)); if (fs.exists(parentPath)) { fs.delete(parentPath, true); } }
moveTaskOutputs(fs, child.getPath(), srcDir, destDir, dryRun, immutable); moveTaskOutputs(fs, child.getPath(), srcDir, destDir, dryRun, immutable);
moveTaskOutputs(fs, child.getPath(), srcDir, destDir, dryRun, immutable); moveTaskOutputs(fs, child.getPath(), srcDir, destDir, dryRun, immutable);
moveTaskOutputs(fs, child.getPath(), srcDir, destDir, dryRun, immutable); moveTaskOutputs(fs, child.getPath(), srcDir, destDir, dryRun, immutable);
moveTaskOutputs(fs, child.getPath(), srcDir, destDir, dryRun, immutable); moveTaskOutputs(fs, child.getPath(), srcDir, destDir, dryRun, immutable);
moveTaskOutputs(fs, src, src, tblPath, false, table.isImmutable()); if (!src.equals(tblPath)) { fs.delete(src, true); moveTaskOutputs(fs, src, src, tblPath, true, true); // dryRun = true, immutable = true moveTaskOutputs(fs, src, src, tblPath, false, true); if (!src.equals(tblPath)){ fs.delete(src, true); Path src = new Path(jobInfo.getLocation()); Path dest = new Path(p.getSd().getLocation()); moveTaskOutputs(fs, src, src, dest, true, table.isImmutable()); moveTaskOutputs(fs,src,src,dest,false,table.isImmutable()); if (!src.equals(dest)){ if (src.toString().matches(".*" + Path.SEPARATOR + SCRATCH_DIR_NAME + "\\d\\.?\\d+.*")){ moveTaskOutputs(fs, src, src, tblPath, true, true); // dryRun = true, immutable = true moveTaskOutputs(fs, src, src, tblPath, false, true); if (!src.equals(tblPath)){ fs.delete(src, true);
moveTaskOutputs(fs, src, src, tblPath, false, table.isImmutable()); if (!src.equals(tblPath)){ fs.delete(src, true); moveTaskOutputs(fs, src, src, tblPath, true, true); // dryRun = true, immutable = true moveTaskOutputs(fs, src, src, tblPath, false, true); if (!src.equals(tblPath)){ fs.delete(src, true); Path src = new Path(jobInfo.getLocation()); Path dest = new Path(p.getSd().getLocation()); moveTaskOutputs(fs, src, src, dest, true, table.isImmutable()); moveTaskOutputs(fs,src,src,dest,false,table.isImmutable()); if (!src.equals(dest)){ fs.delete(src, true); moveTaskOutputs(fs, src, src, tblPath, true, true); // dryRun = true, immutable = true moveTaskOutputs(fs, src, src, tblPath, false, true); if (!src.equals(tblPath)){ fs.delete(src, true);
moveTaskOutputs(fs, src, src, tblPath, false, table.isImmutable()); if (!src.equals(tblPath)){ fs.delete(src, true); moveTaskOutputs(fs, src, src, tblPath, true, true); // dryRun = true, immutable = true moveTaskOutputs(fs, src, src, tblPath, false, true); if (!src.equals(tblPath)){ fs.delete(src, true); Path src = new Path(jobInfo.getLocation()); Path dest = new Path(p.getSd().getLocation()); moveTaskOutputs(fs, src, src, dest, true, table.isImmutable()); moveTaskOutputs(fs,src,src,dest,false,table.isImmutable()); if (!src.equals(dest)){ fs.delete(src, true); moveTaskOutputs(fs, src, src, tblPath, true, true); // dryRun = true, immutable = true moveTaskOutputs(fs, src, src, tblPath, false, true); if (!src.equals(tblPath)){ fs.delete(src, true);
moveTaskOutputs(fs, src, src, tblPath, false, table.isImmutable()); if (!src.equals(tblPath)){ fs.delete(src, true); moveTaskOutputs(fs, src, src, tblPath, true, true); // dryRun = true, immutable = true moveTaskOutputs(fs, src, src, tblPath, false, true); if (!src.equals(tblPath)){ fs.delete(src, true); Path src = new Path(jobInfo.getLocation()); Path dest = new Path(p.getSd().getLocation()); moveTaskOutputs(fs, src, src, dest, true, table.isImmutable()); moveTaskOutputs(fs,src,src,dest,false,table.isImmutable()); if (!src.equals(dest)){ fs.delete(src, true); moveTaskOutputs(fs, src, src, tblPath, true, true); // dryRun = true, immutable = true moveTaskOutputs(fs, src, src, tblPath, false, true); if (!src.equals(tblPath)){ fs.delete(src, true);