private void setupMiniDfsAndMrClusters() { try { final int dataNodes = 1; // There will be 4 data nodes final int taskTrackers = 1; // There will be 4 task tracker nodes Configuration config = new Configuration(); // Builds and starts the mini dfs and mapreduce clusters if(System.getProperty("hadoop.log.dir") == null) { System.setProperty("hadoop.log.dir", "target/tmp/logs/"); } m_dfs = new MiniDFSCluster(config, dataNodes, true, null); m_fileSys = m_dfs.getFileSystem(); m_mr = new MiniMRCluster(taskTrackers, m_fileSys.getUri().toString(), 1); // Create the configuration hadoop-site.xml file File conf_dir = new File(System.getProperty("user.home"), "pigtest/conf/"); conf_dir.mkdirs(); File conf_file = new File(conf_dir, "hadoop-site.xml"); // Write the necessary config info to hadoop-site.xml m_conf = m_mr.createJobConf(); m_conf.setInt("mapred.submit.replication", 1); m_conf.set("dfs.datanode.address", "0.0.0.0:0"); m_conf.set("dfs.datanode.http.address", "0.0.0.0:0"); m_conf.writeXml(new FileOutputStream(conf_file)); // Set the system properties needed by Pig System.setProperty("cluster", m_conf.get("mapred.job.tracker")); System.setProperty("namenode", m_conf.get("fs.default.name")); System.setProperty("junit.hadoop.conf", conf_dir.getPath()); } catch (IOException e) { throw new RuntimeException(e); } }
protected void reconfigureLocalJobConf( JobConf localJobConf, Path localJobFile, TaskInProgress tip, boolean changed) throws IOException { if (!changed) { return; } OutputStream out = localFs.create(localJobFile); try { localJobConf.writeXml(out); } finally { out.close(); } }
public static void main(String[] args) throws Exception { new JobConf(new Configuration()).writeXml(System.out); }
private void persistConf(FileSystem fs, Path file, JobConf conf) throws IOException { new File(file.toUri().getPath()).delete(); FSDataOutputStream out = FileSystem.create( fs, file, FsPermission.getDefault()); conf.writeXml(out); out.close(); }
public static void main(String... args) throws Exception { Configuration conf = new Configuration(); conf.writeXml(System.out); System.out.println("\n\n"); JobConf jobConf = new JobConf(conf); jobConf.writeXml(System.out); }
/** * Write the task specific job-configuration file. * * @param localFs * @throws IOException */ private static void writeLocalTaskFile(String jobFile, JobConf conf) throws IOException { Path localTaskFile = new Path(jobFile); FileSystem localFs = FileSystem.getLocal(conf); localFs.delete(localTaskFile, true); OutputStream out = localFs.create(localTaskFile); try { conf.writeXml(out); } finally { out.close(); } }
/** * Write the task specific job-configuration file. * @throws IOException */ private static void writeLocalJobFile(Path jobFile, JobConf conf) throws IOException { FileSystem localFs = FileSystem.getLocal(conf); localFs.delete(jobFile); OutputStream out = null; try { out = FileSystem.create(localFs, jobFile, urw_gr); conf.writeXml(out); } finally { IOUtils.cleanupWithLogger(LOG, out); } }
/** * Write the task specific job-configuration file. * @throws IOException */ private static void writeLocalJobFile(Path jobFile, JobConf conf) throws IOException { FileSystem localFs = FileSystem.getLocal(conf); localFs.delete(jobFile); OutputStream out = null; try { out = FileSystem.create(localFs, jobFile, urw_gr); conf.writeXml(out); } finally { IOUtils.cleanup(LOG, out); } }
/** * Write the task specific job-configuration file. * @throws IOException */ private static void writeLocalJobFile(Path jobFile, JobConf conf) throws IOException { FileSystem localFs = FileSystem.getLocal(conf); localFs.delete(jobFile); OutputStream out = null; try { out = FileSystem.create(localFs, jobFile, urw_gr); conf.writeXml(out); } finally { IOUtils.cleanup(LOG, out); } }
/** * Creates the localized copy of job conf * @param jobConf * @param id */ void setUpLocalizedJobConf(JobConf jobConf, org.apache.hadoop.mapreduce.JobID id) { String localJobFilePath = jobtracker.getLocalJobFilePath(id); File localJobFile = new File(localJobFilePath); FileOutputStream jobOut = null; try { jobOut = new FileOutputStream(localJobFile); jobConf.writeXml(jobOut); if (LOG.isDebugEnabled()) { LOG.debug("Job conf for " + id + " stored at " + localJobFile.getAbsolutePath()); } } catch (IOException ioe) { LOG.error("Failed to store job conf on the local filesystem ", ioe); } finally { if (jobOut != null) { try { jobOut.close(); } catch (IOException ie) { LOG.info("Failed to close the job configuration file " + StringUtils.stringifyException(ie)); } } } }
conf.writeXml(writer); writer.flush(); String xml = writer.getBuffer().toString();
@Override public void configure(JobConf jobConf) { try { String loc = jobConf.get(JOB_XML_OUTPUT_LOCATION); if (loc != null) { Path p = new Path(loc); FileSystem fs = p.getFileSystem(jobConf); if (!fs.exists(p)) { FSDataOutputStream out = fs.create(p); try { jobConf.writeXml(out); } finally { out.close(); } } } } catch (IOException ioe) { ioe.printStackTrace(); } }
logDirFs.getDefaultReplication(), logDirFs.getDefaultBlockSize(), null); jobConf.writeXml(jobFileOut); jobFileOut.close();
aclConf.writeXml(out); } finally { out.close();
conf.writeXml(out); } finally { out.close();
conf.writeXml(out); } finally { out.close();
conf.writeXml(out); } finally { out.close();
conf.writeXml(out); } finally { out.close();
conf.writeXml(out); } finally { out.close();
conf.writeXml(fos); fos.close();