public SequenceFileReader(FileSystem fs, Path file, Map<String, Object> conf) throws IOException { super(fs, file); int bufferSize = !conf.containsKey(BUFFER_SIZE) ? DEFAULT_BUFF_SIZE : Integer.parseInt(conf.get(BUFFER_SIZE).toString()); this.reader = new SequenceFile.Reader(fs.getConf(), SequenceFile.Reader.file(file), SequenceFile.Reader.bufferSize(bufferSize)); this.key = (Key) ReflectionUtils.newInstance(reader.getKeyClass(), fs.getConf()); this.value = (Value) ReflectionUtils.newInstance(reader.getValueClass(), fs.getConf()); this.offset = new SequenceFileReader.Offset(0, 0, 0); }
public SequenceFileReader(FileSystem fs, Path file, Map<String, Object> conf, String offset) throws IOException { super(fs, file); int bufferSize = !conf.containsKey(BUFFER_SIZE) ? DEFAULT_BUFF_SIZE : Integer.parseInt(conf.get(BUFFER_SIZE).toString()); this.offset = new SequenceFileReader.Offset(offset); this.reader = new SequenceFile.Reader(fs.getConf(), SequenceFile.Reader.file(file), SequenceFile.Reader.bufferSize(bufferSize)); this.key = (Key) ReflectionUtils.newInstance(reader.getKeyClass(), fs.getConf()); this.value = (Value) ReflectionUtils.newInstance(reader.getValueClass(), fs.getConf()); skipToOffset(this.reader, this.offset, this.key); }
@SuppressWarnings("deprecation") public static Map<String, String> readFromSequenceFile(Configuration conf, String inputPath) throws IOException { try (SequenceFile.Reader reader = new SequenceFile.Reader(getWorkingFileSystem(conf), new Path(inputPath), conf)) { Map<String, String> map = Maps.newHashMap(); Text key = (Text) ReflectionUtils.newInstance(reader.getKeyClass(), conf); Text value = (Text) ReflectionUtils.newInstance(reader.getValueClass(), conf); while (reader.next(key, value)) { map.put(key.toString(), value.toString()); } return map; } }
BytesWritable valueW = (BytesWritable) ReflectionUtils.newInstance(reader.getValueClass(), conf);
BytesWritable value = (BytesWritable) ReflectionUtils.newInstance(reader.getValueClass(), conf);
reader = new SequenceFile.Reader(fs, new Path(tempFile.getAbsolutePath()), conf); LongWritable key = (LongWritable) ReflectionUtils.newInstance(reader.getKeyClass(), conf); BytesWritable value = (BytesWritable) ReflectionUtils.newInstance(reader.getValueClass(), conf); while (reader.next(key, value)) { if (key.get() == 0L) {
Text value = (Text) ReflectionUtils.newInstance(reader.getValueClass(), conf);
Mockito.<Class<?>>when(reader.getValueClass()).thenReturn( (Class<?>) BytesWritable.class);
deserializeConf)) { Object writable = reader.getValueClass() == JobState.class ? new JobState() : new JobState.DatasetState();
/** Fills up the rawKey object with the key returned by the Reader * @return true if there is a key returned; false, otherwise * @throws IOException */ public boolean nextRawKey() throws IOException { if (in == null) { int bufferSize = getBufferSize(conf); Reader reader = new Reader(conf, Reader.file(segmentPathName), Reader.bufferSize(bufferSize), Reader.start(segmentOffset), Reader.length(segmentLength)); //sometimes we ignore syncs especially for temp merge files if (ignoreSync) reader.ignoreSync(); if (reader.getKeyClass() != keyClass) throw new IOException("wrong key class: " + reader.getKeyClass() + " is not " + keyClass); if (reader.getValueClass() != valClass) throw new IOException("wrong value class: "+reader.getValueClass()+ " is not " + valClass); this.in = reader; rawKey = new DataOutputBuffer(); } rawKey.reset(); int keyLength = in.nextRawKey(rawKey); return (keyLength >= 0); }
/** * Check if there's partition files for hfile, if yes replace the table splits, to make the job more reducers * @param conf the job configuration * @param path the hfile partition file * @throws IOException */ @SuppressWarnings("deprecation") private void reconfigurePartitions(Configuration conf, Path path) throws IOException { FileSystem fs = path.getFileSystem(conf); if (fs.exists(path)) { try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf)) { int partitionCount = 0; Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf); Writable value = (Writable) ReflectionUtils.newInstance(reader.getValueClass(), conf); while (reader.next(key, value)) { partitionCount++; } TotalOrderPartitioner.setPartitionFile(job.getConfiguration(), path); // The reduce tasks should be one more than partition keys job.setNumReduceTasks(partitionCount + 1); } } else { logger.info("File '" + path.toString() + " doesn't exist, will not reconfigure hfile Partitions"); } }
public CubeStatsResult(Path path, int precision) throws IOException { Configuration hadoopConf = HadoopUtil.getCurrentConfiguration(); Option seqInput = SequenceFile.Reader.file(path); try (Reader reader = new SequenceFile.Reader(hadoopConf, seqInput)) { LongWritable key = (LongWritable) ReflectionUtils.newInstance(reader.getKeyClass(), hadoopConf); BytesWritable value = (BytesWritable) ReflectionUtils.newInstance(reader.getValueClass(), hadoopConf); while (reader.next(key, value)) { if (key.get() == 0L) { percentage = Bytes.toInt(value.getBytes()); } else if (key.get() == -1) { mapperOverlapRatio = Bytes.toDouble(value.getBytes()); } else if (key.get() == -2) { mapperNumber = Bytes.toInt(value.getBytes()); } else if (key.get() == -3) { sourceRecordCount = Bytes.toLong(value.getBytes()); } else if (key.get() > 0) { HLLCounter hll = new HLLCounter(precision); ByteArray byteArray = new ByteArray(value.getBytes()); hll.readRegisters(byteArray.asBuffer()); counterMap.put(key.get(), hll); } } } }
public void printFile(String path) throws Exception { FileSystem fileSystem = FileUtil.getFileSystem(path); Path fsPath = new Path(path); SequenceFile.Reader reader = new SequenceFile.Reader(fileSystem, fsPath, new Configuration()); LongWritable key = (LongWritable) reader.getKeyClass().newInstance(); BytesWritable value = (BytesWritable) reader.getValueClass().newInstance(); System.out.println("reading file " + path); while (reader.next(key, value)) { if (mPrintOffsetsOnly) { System.out.println(Long.toString(key.get())); } else { byte[] nonPaddedBytes = new byte[value.getLength()]; System.arraycopy(value.getBytes(), 0, nonPaddedBytes, 0, value.getLength()); System.out.println(Long.toString(key.get()) + ": " + new String(nonPaddedBytes)); } } } }
public SequenceFileReader(SequenceFile.Reader reader, Configuration conf) { this.reader = reader; this.schema = Pair.getPairSchema(WritableData.get().getSchema(reader.getKeyClass()), WritableData.get().getSchema(reader.getValueClass())); this.key = (Writable)ReflectionUtils.newInstance(reader.getKeyClass(), conf); this.spareKey = (Writable)ReflectionUtils.newInstance(reader.getKeyClass(), conf); this.value = (Writable)ReflectionUtils.newInstance(reader.getValueClass(), conf); if (WRITABLE_CONVERTERS.containsKey(reader.getKeyClass()) ) keyConverter = WRITABLE_CONVERTERS.get(reader.getKeyClass()); if (WRITABLE_CONVERTERS.containsKey(reader.getValueClass()) ) valConverter = WRITABLE_CONVERTERS.get(reader.getValueClass()); }
/** Returns the class of values in this file. */ public Class<?> getValueClass() { return data.getValueClass(); }
/** Read the next key/value pair in the file into <code>key</code> and * <code>val</code>. Returns true if such a pair exists and false when at * end of file */ public synchronized boolean next(Writable key, Writable val) throws IOException { if (val.getClass() != getValueClass()) throw new IOException("wrong value class: "+val+" is not "+valClass); boolean more = next(key); if (more) { getCurrentValue(val); } return more; }
public SequenceFileReader(LogFilePath path) throws Exception { Configuration config = new Configuration(); Path fsPath = new Path(path.getLogFilePath()); FileSystem fs = FileUtil.getFileSystem(path.getLogFilePath()); this.mReader = new SequenceFile.Reader(fs, fsPath, config); this.mKey = (LongWritable) mReader.getKeyClass().newInstance(); this.mValue = (BytesWritable) mReader.getValueClass().newInstance(); }
public TextRecordInputStream(FileStatus f) throws IOException { final Path fpath = f.getPath(); final Configuration lconf = getConf(); r = new SequenceFile.Reader(lconf, SequenceFile.Reader.file(fpath)); key = ReflectionUtils.newInstance( r.getKeyClass().asSubclass(Writable.class), lconf); val = ReflectionUtils.newInstance( r.getValueClass().asSubclass(Writable.class), lconf); inbuf = new DataInputBuffer(); outbuf = new DataOutputBuffer(); }
public MessagePackSequenceFileReader(LogFilePath path) throws Exception { Configuration config = new Configuration(); Path fsPath = new Path(path.getLogFilePath()); FileSystem fs = FileUtil.getFileSystem(path.getLogFilePath()); this.mReader = new SequenceFile.Reader(fs, fsPath, config); this.mKey = (BytesWritable) mReader.getKeyClass().newInstance(); this.mValue = (BytesWritable) mReader.getValueClass().newInstance(); }
private List<String> getSeqFileContents(FileSystem fs, Path... seqFiles) throws IOException { ArrayList<String> result = new ArrayList<>(); for (Path seqFile : seqFiles) { Path file = new Path(fs.getUri().toString() + seqFile.toString()); SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(file)); try { Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf); Writable value = (Writable) ReflectionUtils.newInstance(reader.getValueClass(), conf); while (reader.next(key, value)) { String keyValStr = Arrays.asList(key, value).toString(); result.add(keyValStr); } } finally { reader.close(); } }// for return result; }