@Override public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) { ((PathTarget) target).configureForMapReduce(job, ptype, outputPath, name); }
@Override public Path getPath() { return ((PathTarget) target).getPath(); } }
@Override public void handleOutputs(Configuration conf, Path workingPath, int index) throws IOException { ((PathTarget) target).handleOutputs(conf, workingPath, index); } }
if (t instanceof PathTarget) { PathTarget pt = (PathTarget) t; pt.configureForMapReduce(job, ptype, pt.getPath(), "out0"); CrunchOutputs.OutputConfig outConfig = CrunchOutputs.getNamedOutputs(job.getConfiguration()).get("out0"); job.getOutputFormatClass(), job.getConfiguration()); pt.handleOutputs(job.getConfiguration(), tmpPath, -1); } else { //if (t instanceof MapReduceTarget) { MapReduceTarget mrt = (MapReduceTarget) t;
private synchronized void handleMultiPaths() throws IOException { if (!multiPaths.isEmpty()) { // Need to handle moving the data from the output directory of the // job to the output locations specified in the paths. FileSystem srcFs = workingPath.getFileSystem(job.getConfiguration()); for (Map.Entry<Integer, PathTarget> entry : multiPaths.entrySet()) { final int i = entry.getKey(); final Path dst = entry.getValue().getPath(); FileNamingScheme fileNamingScheme = entry.getValue().getFileNamingScheme(); Path src = new Path(workingPath, PlanningParameters.MULTI_OUTPUT_PREFIX + i + "-*"); Path[] srcs = FileUtil.stat2Paths(srcFs.globStatus(src), src); Configuration conf = job.getConfiguration(); FileSystem dstFs = dst.getFileSystem(conf); if (!dstFs.exists(dst)) { dstFs.mkdirs(dst); } boolean sameFs = isCompatible(srcFs, dst); for (Path s : srcs) { Path d = getDestFile(conf, s, dst, fileNamingScheme); if (sameFs) { srcFs.rename(s, d); } else { FileUtil.copy(srcFs, s, dstFs, d, true, true, job.getConfiguration()); } } } } }
if (t instanceof PathTarget) { PathTarget pt = (PathTarget) t; pt.configureForMapReduce(job, ptype, pt.getPath(), "out0"); CrunchOutputs.OutputConfig outConfig = CrunchOutputs.getNamedOutputs(job.getConfiguration()).get("out0"); job.getOutputFormatClass(), job.getConfiguration()); pt.handleOutputs(job.getConfiguration(), tmpPath, -1); } else { //if (t instanceof MapReduceTarget) { MapReduceTarget mrt = (MapReduceTarget) t;
@Override public Path getPath() { return ((PathTarget) target).getPath(); }
@Override public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) { ((PathTarget) target).configureForMapReduce(job, ptype, outputPath, name); }
private synchronized void handleMultiPaths(Job job) throws IOException { try { if (job.isSuccessful()) { if (!multiPaths.isEmpty()) { for (Map.Entry<Integer, PathTarget> entry : multiPaths.entrySet()) { entry.getValue().handleOutputs(job.getConfiguration(), workingPath, entry.getKey()); } } } } catch(Exception ie) { throw new IOException(ie); } } }
@Override public Path getPath() { return ((PathTarget) target).getPath(); }
@Override public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) { ((PathTarget) target).configureForMapReduce(job, ptype, outputPath, name); }
/** * Returns the {@code Path} that contains this data, or null if no such path exists. */ public Path getPath() { if (source instanceof FileSourceImpl) { return ((FileSourceImpl) source).getPath(); } if (source instanceof PathTarget) { return ((PathTarget) source).getPath(); } return null; }
public Path getPath() { if (source instanceof FileSourceImpl) { return ((FileSourceImpl) source).getPath(); } else if (source instanceof PathTarget) { return ((PathTarget) source).getPath(); } return null; }
public boolean configure(Target target, PType<?> ptype) { if (target instanceof MapReduceTarget && target instanceof PathTarget) { String name = PlanningParameters.MULTI_OUTPUT_PREFIX + multiPaths.size(); multiPaths.add(((PathTarget) target).getPath()); workingNode.setOutputName(name); ((MapReduceTarget) target).configureForMapReduce(job, ptype, path, name); return true; } if (target instanceof MapReduceTarget) { ((MapReduceTarget) target).configureForMapReduce(job, ptype, null, null); return true; } return false; }
@Override public void write(PCollection<?> collection, Target target) { if (target instanceof PathTarget) { Path path = ((PathTarget) target).getPath(); try { FileSystem fs = FileSystem.get(conf); FSDataOutputStream os = fs.create(new Path(path, "out")); if (collection instanceof PTable) { for (Object o : collection.materialize()) { Pair p = (Pair) o; os.writeBytes(p.first().toString()); os.writeBytes("\t"); os.writeBytes(p.second().toString()); os.writeBytes("\r\n"); } } else { for (Object o : collection.materialize()) { os.writeBytes(o.toString() + "\r\n"); } } os.close(); } catch (IOException e) { LOG.error("Exception writing target: " + target, e); } } else { LOG.error("Target " + target + " is not a PathTarget instance"); } }
collection.getPType().initialize(getConfiguration()); Path path = ((PathTarget) target).getPath(); try { FileSystem fs = path.getFileSystem(conf);
Path path = ((PathTarget) target).getPath(); try { FileSystem fs = path.getFileSystem(conf);