private Writable allocateWritable(Type type) { switch (type) { case BYTE: return new ByteWritable(); case BOOL: return new BooleanWritable(); case INT: return new IntWritable(); case SHORT: return new ShortWritable(); case LONG: return new LongWritable(); case FLOAT: return new FloatWritable(); case DOUBLE: return new DoubleWritable(); case STRING: return new Text(); default: assert false; // not supported } return null; }
@Override public void readFields(DataInput dataInput) { try { isXLearningTrainCompleted.readFields(dataInput); interResultTimeStamp.readFields(dataInput); } catch (IOException e) { LOG.error("containerStatus read error:" + e); } } }
public void addToStat(String statType, long amount) { LongWritable currentValue = statsMap.get(statType); if (currentValue == null) { statsMap.put(statType, new LongWritable(amount)); } else { currentValue.set(currentValue.get() + amount); } }
@BeforeClass public static void testWriteSequenceFile() throws IOException { Configuration c = new Configuration(); URI uri = file().toURI(); try(SequenceFile.Writer writer = new SequenceFile.Writer(FileSystem.get(uri, c), c, new Path(uri.toString()), LongWritable.class, Text.class)) { final LongWritable key = new LongWritable(); final Text val = new Text(); for (int i = 0; i < COUNT; ++i) { key.set(i); val.set(Integer.toString(i)); writer.append(key, val); } } }
@Override protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { String line = value.toString(); String[] lineSplits = line.split(" "); Configuration conf = context.getConfiguration(); final FileSystem fs = FileSystem.get(conf); final Path hdfsInputFilePath = new Path(_inputFilePath); new Path(localInputDataDir.getAbsolutePath() + "/" + hdfsInputFilePath.getName()); LOGGER.info("Copy from " + hdfsInputFilePath + " to " + localInputFilePath); fs.copyToLocalFile(hdfsInputFilePath, localInputFilePath); String schemaString = context.getConfiguration().get("data.schema"); try { schema = Schema.fromString(schemaString); context.write(new LongWritable(Long.parseLong(lineSplits[2])), new Text( FileSystem.get(_properties).listStatus(new Path(_localHdfsSegmentTarPath + "/"))[0].getPath().getName())); LOGGER.info("Finished the job successfully");
private static final void shareMapFile(String symbol, int slots, Path mfile, JobConf job) throws IOException, URISyntaxException { FileSystem fs = FileSystem.get(mfile.toUri(), job); if (fs.exists(mfile) && fs.getFileStatus(mfile).isDir()) { DistributedCache.createSymlink(job); FileStatus[] fstats = fs.listStatus(mfile, getPassDirectoriesFilter(fs)); LongWritable key = new LongWritable(); Text value = new Text(); for (int i=0; i<fstats.length; i++) { Path curMap = fstats[i].getPath(); MapFile.Reader mreader = new MapFile.Reader(fs, curMap.toString(), job); if (mreader.next(key, value)) { int rid = (int) (key.get() % slots); String uriWithLink = curMap.toUri().toString() + "#" + symbol + "-" + Integer.toString(rid); DistributedCache.addCacheFile(new URI(uriWithLink), job); } else { System.exit(-1); } mreader.close(); } } job.setInt(symbol, slots); }
fileSizes = true; } else if (fileName == null){ fileName = new Path(arg); } else { printUsage(null); FileSystem fs = FileSystem.get(fileName.toUri(), conf); long fileLen = fs.getFileStatus(fileName).getLen(); if (start < 0) { start = 0; LongWritable key = new LongWritable(); BytesRefArrayWritable value = new BytesRefArrayWritable(); StringBuilder buf = new StringBuilder(STRING_BUFFER_SIZE); // extra capacity in case we overrun, to avoid resizing
throws IOException, InterruptedException { int index = key.get(); String dictInfoPath = mergedDictInfo == null ? "" : mergedDictInfo.getResourcePath(); context.write(new IntWritable(-1), new Text(tblCol + "=" + dictInfoPath)); final String cubeName = context.getConfiguration().get(BatchConstants.ARG_CUBE_NAME); conf = HadoopUtil.getCurrentConfiguration(); if (keyW.get() == 0L) { } else if (keyW.get() > 0) { HLLCounter hll = new HLLCounter(kylinConfig.getCubeStatsHLLPrecision()); ByteArray byteArray = new ByteArray(valueW.getBytes()); hll.readRegisters(byteArray.asBuffer()); if (cuboidHLLMap.get(keyW.get()) != null) { cuboidHLLMap.get(keyW.get()).merge(hll); } else { cuboidHLLMap.put(keyW.get(), hll); FSDataInputStream fis = fs.open(statisticsFilePath); context.write(new IntWritable(-1), new Text(""));
row.setFieldValue(0, new IntWritable(0)); row.setFieldValue(1, new LongWritable(1)); row.setFieldValue(2, new IntWritable(0)); LongWritable rowId = new LongWritable(); row.setFieldValue(3, rowId); row.setFieldValue(4, new LongWritable(1)); OrcStruct rowField = new OrcStruct(3); row.setFieldValue(5, rowField); IntWritable a = new IntWritable(); rowField.setFieldValue(0, a); OrcStruct b = new OrcStruct(1); IntWritable c = new IntWritable(); b.setFieldValue(0, c); Text d = new Text(); rowField.setFieldValue(2, d); rowId.set(r); a.set(r * 42); c.set(r * 10001); d.set(Integer.toHexString(r)); indexBuilder.addKey(OrcRecordUpdater.INSERT_OPERATION, 1, 0, rowId.get()); writer.addRow(row);
String dictInfoPath = mergedDictInfo == null ? "" : mergedDictInfo.getResourcePath(); return new Tuple2<>(new Text(tblCol), new Text(dictInfoPath)); try(SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(tempFile.getAbsolutePath()), conf)) { if (key.get() == 0L) { } else if (key.get() > 0) { HLLCounter hll = new HLLCounter(kylinConfig.getCubeStatsHLLPrecision()); ByteArray byteArray = new ByteArray(value.getBytes()); hll.readRegisters(byteArray.asBuffer()); if (cuboidHLLMap.get(key.get()) != null) { cuboidHLLMap.get(key.get()).merge(hll); } else { cuboidHLLMap.put(key.get(), hll); CubeStatsWriter.writeCuboidStatistics(conf, new Path(statOutputPath), cuboidHLLMap, averageSamplingPercentage); Path statisticsFilePath = new Path(statOutputPath, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME); FSDataInputStream fis = fs.open(statisticsFilePath);
LongWritable val = sizes.get(host); if (val == null) { val = new LongWritable(); sizes.put(host, val); val.set(val.get() + overlap); maxSize = Math.max(maxSize, val.get()); throw new IOException("File " + file.getPath().toString() + " should have had overlap on block starting at " + block.getOffset()); for(String host: block.getHosts()) { if (sizes.containsKey(host)) { if (sizes.get(host).get() >= threshold) { hostList.add(host);
private static void createControlFile( FileSystem fs, int fileSize, // in MB int nrFiles ) throws IOException { LOG.info("creating control file: "+fileSize+" mega bytes, "+nrFiles+" files"); fs.delete(CONTROL_DIR, true); for(int i=0; i < nrFiles; i++) { String name = getFileName(i); Path controlFile = new Path(CONTROL_DIR, "in_file_" + name); SequenceFile.Writer writer = null; try { writer = SequenceFile.createWriter(fs, fsConfig, controlFile, Text.class, LongWritable.class, CompressionType.NONE); writer.append(new Text(name), new LongWritable(fileSize)); } catch(Exception e) { throw new IOException(e.getLocalizedMessage()); } finally { if (writer != null) writer.close(); writer = null; } } LOG.info("created control files for: "+nrFiles+" files"); }
Configuration conf) throws Exception { String dr = (dryrun ? "[DRY RUN ] " : ""); Path data = new Path(dir, DATA_FILE_NAME); Path index = new Path(dir, INDEX_FILE_NAME); int indexInterval = conf.getInt(Writer.INDEX_INTERVAL, 128); if (!fs.exists(data)) { if (fs.exists(index)) { LongWritable position = new LongWritable(); long nextBlock = pos; boolean blockCompressed = dataReader.isBlockCompressed(); position.set(pos); if (!dryrun) { indexWriter.append(key, position);
switch (poi.getPrimitiveCategory()) { case INT: return Bytes.toBytes(((IntWritable) writable).get()); case BOOLEAN: return Bytes.toBytes(((BooleanWritable) writable).get()); case LONG: return Bytes.toBytes(((LongWritable) writable).get()); case FLOAT: return Bytes.toBytes(((FloatWritable) writable).get()); case DOUBLE: return Bytes.toBytes(((DoubleWritable) writable).get()); case SHORT: return Bytes.toBytes(((ShortWritable) writable).get()); case STRING: return Bytes.toBytes(((Text) writable).toString()); case BYTE: return Bytes.toBytes(((ByteWritable) writable).get());
SequenceFile.Reader reader = null; try { reader = new SequenceFile.Reader(fs, new Path(tempFile.getAbsolutePath()), conf); LongWritable key = (LongWritable) ReflectionUtils.newInstance(reader.getKeyClass(), conf); BytesWritable value = (BytesWritable) ReflectionUtils.newInstance(reader.getValueClass(), conf); while (reader.next(key, value)) { if (key.get() == 0L) { } else if (key.get() > 0) { HLLCounter hll = new HLLCounter(kylinConf.getCubeStatsHLLPrecision()); ByteArray byteArray = new ByteArray(value.getBytes()); hll.readRegisters(byteArray.asBuffer()); if (cuboidHLLMap.get(key.get()) != null) { cuboidHLLMap.get(key.get()).merge(hll); } else { cuboidHLLMap.put(key.get(), hll); CubeStatsWriter.writeCuboidStatistics(conf, new Path(CubingExecutableUtil.getMergedStatisticsPath(this.getParams())), cuboidHLLMap, averageSamplingPercentage); Path statisticsFilePath = new Path(CubingExecutableUtil.getMergedStatisticsPath(this.getParams()), BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME); FileSystem fs = HadoopUtil.getFileSystem(statisticsFilePath, conf); FSDataInputStream is = fs.open(statisticsFilePath); try {
public SequenceFileWriter(LogFilePath path, CompressionCodec codec) throws IOException { Configuration config = new Configuration(); fsPath = new Path(path.getLogFilePath()); FileSystem fs = FileUtil.getFileSystem(path.getLogFilePath()); if (codec != null) { this.mWriter = SequenceFile.createWriter(fs, config, fsPath, LongWritable.class, BytesWritable.class, SequenceFile.CompressionType.BLOCK, codec); } else { this.mWriter = SequenceFile.createWriter(fs, config, fsPath, LongWritable.class, BytesWritable.class); } this.mKey = new LongWritable(); this.mValue = new BytesWritable(); LOG.info("Created sequence file writer: {}", fsPath); }
public void printFile(String path) throws Exception { FileSystem fileSystem = FileUtil.getFileSystem(path); Path fsPath = new Path(path); SequenceFile.Reader reader = new SequenceFile.Reader(fileSystem, fsPath, new Configuration()); LongWritable key = (LongWritable) reader.getKeyClass().newInstance(); BytesWritable value = (BytesWritable) reader.getValueClass().newInstance(); System.out.println("reading file " + path); while (reader.next(key, value)) { if (mPrintOffsetsOnly) { System.out.println(Long.toString(key.get())); } else { byte[] nonPaddedBytes = new byte[value.getLength()]; System.arraycopy(value.getBytes(), 0, nonPaddedBytes, 0, value.getLength()); System.out.println(Long.toString(key.get()) + ": " + new String(nonPaddedBytes)); } } } }
@Override public void map(LongWritable k, Text v, OutputCollector<Text, LongWritable> out, Reporter rep) throws IOException { // normalize and split the line String line = v.toString(); String[] tokens = line.toLowerCase().split("\\W+"); // emit the pairs for (String token : tokens) { if (token.length() > 0) { out.collect(new Text(token), new LongWritable(1L)); } } }