Refine search
private void setStatus(String line) { reporter.setStatus(line.substring(statusPrefix.length()).trim()); } }
private void createDpDir(final Path dpPath) throws IOException { if (!fs.exists(dpPath)) { fs.mkdirs(dpPath); if (reporter != null) { reporter.incrCounter(counterGroup, Operator.HIVE_COUNTER_CREATED_DYNAMIC_PARTITIONS, 1); } } }
private static void oldApiCounter(Reporter reporter, Enum<?> counter, long value) { try { org.apache.hadoop.mapred.Counters.Counter c = reporter.getCounter(counter); if (c != null) { c.increment(value); } } catch (Exception ex) { // counter unavailable } }
@Override public void map(LongWritable key, Text value, OutputCollector<LongWritable, Text> output, Reporter reporter) throws IOException { int slotId = Integer.parseInt(value.toString().trim()); long[] range = HtmlCore.getPageRange(slotId, pages, slotpages); for (long i=range[0]; i<range[1]; i++) { key.set(i); Text v = new Text(Long.toString(i)); output.collect(key, v); reporter.incrCounter(HiBench.Counters.BYTES_DATA_GENERATED, 8+v.getLength()); } } }
public Object doIO(Reporter reporter, String name, long totalSize ) throws IOException { totalSize *= MEGA; // open file DataInputStream in = fs.open(new Path(DATA_DIR, name)); try { long actualSize = 0; for(int curSize = bufferSize; curSize == bufferSize;) { curSize = in.read(buffer, 0, bufferSize); actualSize += curSize; reporter.setStatus("reading " + name + "@" + actualSize + "/" + totalSize + " ::host = " + hostName); } } finally { in.close(); } return new Long(totalSize); } }
@SuppressWarnings("unchecked") @Test public void testInitNextRecordReader() throws IOException{ JobConf conf = new JobConf(); Path[] paths = new Path[3]; long[] fileLength = new long[3]; File[] files = new File[3]; LongWritable key = new LongWritable(1); Text value = new Text(); try { for(int i=0;i<3;i++){ fileLength[i] = i; File dir = new File(outDir.toString()); dir.mkdir(); files[i] = new File(dir,"testfile"+i); FileWriter fileWriter = new FileWriter(files[i]); fileWriter.close(); paths[i] = new Path(outDir+"/testfile"+i); CombineFileRecordReader cfrr = new CombineFileRecordReader(conf, combineFileSplit, reporter, TextRecordReaderWrapper.class); verify(reporter).progress(); Assert.assertFalse(cfrr.next(key,value)); verify(reporter, times(3)).progress(); } finally { FileUtil.fullyDelete(new File(outDir.toString()));
public Object doIO(Reporter reporter, String name, long totalSize ) throws IOException { // create file totalSize *= MEGA; OutputStream out; out = fs.create(new Path(DATA_DIR, name), true, bufferSize); try { // write to the file long nrRemaining; for (nrRemaining = totalSize; nrRemaining > 0; nrRemaining -= bufferSize) { int curSize = (bufferSize < nrRemaining) ? bufferSize : (int)nrRemaining; out.write(buffer, 0, curSize); reporter.setStatus("writing " + name + "@" + (totalSize - nrRemaining) + "/" + totalSize + " ::host = " + hostName); } } finally { out.close(); } return new Long(totalSize); } }
public void map(LongWritable key, Text val, OutputCollector<Text, Text> output, Reporter reporter) throws IOException { FileSplit fileSplit = (FileSplit) reporter.getInputSplit(); String fileName = fileSplit.getPath().getName(); location.set(fileName); String line = val.toString(); StringTokenizer itr = new StringTokenizer(line.toLowerCase()); while (itr.hasMoreTokens()) { word.set(itr.nextToken()); output.collect(word, location); } } }
public RecordReader<Text, Text> getRecordReader( InputSplit genericSplit, JobConf job, Reporter reporter) throws IOException { reporter.setStatus(genericSplit.toString()); FileSplit split = (FileSplit)genericSplit; final Path file = split.getPath(); FileSystem fs = file.getFileSystem(job); FSDataInputStream fileIn = fs.open(split.getPath()); if (compressionCodecs != null && compressionCodecs.getCodec(file) != null) throw new RuntimeException("Not handling compression!"); return new StreamXmlRecordReader(fileIn, split, reporter, job, FileSystem.get(job)); }
public void map(IntWritable key, PhrasePair value, OutputCollector<IntWritable,Text> output, Reporter reporter) throws IOException { if (m1 == null) { Path pathTTable = new Path(ttable); TTable tt = loadTTable(pathTTable); m1 = new Model1(tt, true); } cr.reset(); Alignment a = m1.viterbiAlign(value, cr); out.set(a.toString()); output.collect(key, out); reporter.incrCounter(CrossEntropyCounters.LOGPROB, (long)(cr.getTotalLogProb())); reporter.incrCounter(CrossEntropyCounters.WORDCOUNT, cr.getTotalWordCount()); }
public Long doIO(Reporter reporter, String name, long totalSize // in bytes ) throws IOException { // create file OutputStream out; out = fs.append(new Path(getDataDir(getConf()), name), bufferSize); try { // write to the file long nrRemaining; for (nrRemaining = totalSize; nrRemaining > 0; nrRemaining -= bufferSize) { int curSize = (bufferSize < nrRemaining) ? bufferSize : (int)nrRemaining; out.write(buffer, 0, curSize); reporter.setStatus("writing " + name + "@" + (totalSize - nrRemaining) + "/" + totalSize + " ::host = " + hostName); } } finally { out.close(); } return Long.valueOf(totalSize); } }
reporter.setStatus(orcSplit.toString()); readerOptions = OrcInputFormat.createOptionsForReader(conf); String txnString = conf.get(ValidWriteIdList.VALID_WRITEIDS_KEY); this.validWriteIdList = (txnString == null) ? new ValidReaderWriteIdList() : new ValidReaderWriteIdList(txnString); LOG.info("Read ValidWriteIdList: " + this.validWriteIdList.toString() if(conf.getBoolean(ConfVars.OPTIMIZE_ACID_META_COLUMNS.varname, true)) { Path parent = orcSplit.getPath().getParent(); while (parent != null && !rootPath.equals(parent)) { if (parent.getName().startsWith(AcidUtils.BASE_PREFIX)) {
/** * Skip copying this file if already exists at the destination. * Updates counters and copy status if skipping this file. * @return true if copy of this file can be skipped */ private boolean skipCopyFile(FileStatus srcstat, Path absdst, OutputCollector<WritableComparable<?>, Text> outc, Reporter reporter) throws IOException { if (destFileSys.exists(absdst) && !overwrite && !needsUpdate(srcstat, destFileSys, absdst)) { outc.collect(null, new Text("SKIP: " + srcstat.getPath())); ++skipcount; reporter.incrCounter(Counter.SKIP, 1); updateStatus(reporter); return true; } return false; }
Reporter reporter ) throws IOException { String field = key.toString(); reporter.setStatus("starting " + field + " ::host = " + hostName); String sSum = ""; while (values.hasNext()) sSum += values.next().toString() + ";"; output.collect(key, new Text(sSum)); reporter.setStatus("finished " + field + " ::host = " + hostName); return; while (values.hasNext()) fSum += Float.parseFloat(values.next().toString()); output.collect(key, new Text(String.valueOf(fSum))); reporter.setStatus("finished " + field + " ::host = " + hostName); return; lSum += Long.parseLong(values.next().toString()); output.collect(key, new Text(String.valueOf(lSum))); reporter.setStatus("finished " + field + " ::host = " + hostName);
public BlockRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException { FileBlockSplit fsplit = (FileBlockSplit) split; buffer = new byte[1024 * 1024]; this.key = new FileBlockIndex(fsplit.getPath(), fsplit.getIndex()); // Initialize input stream Path inPath = fsplit.getPath(); FileSystem inFs = inPath.getFileSystem(job); in = inFs.open(inPath); in.seek(this.start = fsplit.getStart()); this.end = fsplit.getStart() + fsplit.getLength(); reporter.setStatus("Copying "+inPath+"["+start+","+end+")"); }
@Override protected void copyFile(FileSystem fsSource, Path source, FileSystem fsDest, Path target, Reporter reporter) throws IOException { FSDataInputStream fin = fsSource.open(source); FSDataOutputStream fout = fsDest.create(target); try { int amt; while((amt = fin.read(buffer)) >= 0) { fout.write(buffer, 0, amt); reporter.progress(); } } finally { fin.close(); } //don't complete files that aren't done yet. prevents partial files from being written //doesn't really matter though since files are written to tmp file and renamed fout.close(); } }
public void solution(List<List<Pentomino.ColumnName>> answer) { String board = Pentomino.stringifySolution(width, height, answer); try { output.collect(prefixString, new Text("\n" + board)); reporter.incrCounter(pent.getCategory(answer), 1); } catch (IOException e) { System.err.println(StringUtils.stringifyException(e)); } } }
private String oomErrorMessage(Reporter reporter) { return "Got OOM while mapping record #: " + recordCounter + " from input: " + ((FileSplit) reporter.getInputSplit()).getPath().getName(); }
String[] numbers = value.toString().split("\t"); int i = 0; long numSamples = Long.parseLong(numbers[i++]); Vector p = new RandomAccessSparseVector(dimension); p.assign(vec); output.collect(new LongWritable(count), new VectorWritable(p)); reporter.setStatus(Long.toString(count + 1) + " samples generated"); reporter.incrCounter(HiBench.Counters.BYTES_DATA_GENERATED, 8 + p.getNumNondefaultElements() * 8);
@Override public RecordReader<AvroWrapper<T>, NullWritable> getRecordReader(InputSplit split, final JobConf job, Reporter reporter) throws IOException { final FileSplit file = (FileSplit)split; reporter.setStatus(file.toString()); final AvroColumnReader.Params params = new AvroColumnReader.Params(new HadoopInput(file.getPath(), job)); params.setModel(ReflectData.get()); if (job.get(AvroJob.INPUT_SCHEMA) != null) params.setSchema(AvroJob.getInputSchema(job)); return new RecordReader<AvroWrapper<T>, NullWritable>() { private AvroColumnReader<T> reader = new AvroColumnReader<>(params); private float rows = reader.getRowCount(); private long row; public AvroWrapper<T> createKey() { return new AvroWrapper<>(null); } public NullWritable createValue() { return NullWritable.get(); } public boolean next(AvroWrapper<T> wrapper, NullWritable ignore) throws IOException { if (!reader.hasNext()) return false; wrapper.datum(reader.next()); row++; return true; } public float getProgress() throws IOException { return row / rows; } public long getPos() throws IOException { return row; } public void close() throws IOException { reader.close(); } }; }