public SymlinkTextInputSplit() { super((Path)null, 0, 0, (String[])null); split = new FileSplit((Path)null, 0, 0, (String[])null); }
public SymlinkTextInputSplit() { super((Path)null, 0, 0, (String[])null); split = new FileSplit((Path)null, 0, 0, (String[])null); }
private FileSplit getFileSplit() { return new FileSplit(new Path("path"), 1, 2, new String[]{}); }
FileSplit fileSplit = new FileSplit(path, start, length, (String[]) null);
public static HiveInputSplit createMockSplit(String pathString, long start, long length) { InputSplit inputSplit = new FileSplit(new Path(pathString), start, length, HOSTS); return new HiveInputSplit(inputSplit, INPUT_FORMAT_CLASS_NAME); } }
@Test public void testCompare1() throws Exception { FileSplit split1 = new FileSplit(new Path("/abc/def"), 2000L, 500L, EMPTY); FileSplit split2 = new FileSplit(new Path("/abc/def"), 1000L, 500L, EMPTY); InputSplitComparator comparator = new InputSplitComparator(); assertEquals(1, comparator.compare(split1, split2)); } }
private HiveInputSplit adjustSplit(HiveInputSplit split) throws IOException { long adjustedStart = adjustStart(split.getStart()); return new HiveInputSplit(new FileSplit(split.getPath(), adjustedStart, split.getStart() - adjustedStart + split.getLength(), split.getLocations()), split.inputFormatClassName()); }
public void testProjectionNoreducer(String inputDirectory) throws Exception { JobConf job = new JobConf(); long onel = 1; Schema readerSchema = Schema.create(Schema.Type.STRING); AvroJob.setInputSchema(job, readerSchema); Path inputPath = new Path(inputDirectory + "/myavro2-m-00000.avro"); FileStatus fileStatus = FileSystem.get(job).getFileStatus(inputPath); FileSplit fileSplit = new FileSplit(inputPath, 0, fileStatus.getLen(), job); AvroRecordReader<Utf8> recordReader = new AvroRecordReader<>(job, fileSplit); AvroWrapper<Utf8> inputPair = new AvroWrapper<>(null); NullWritable ignore = NullWritable.get(); while (recordReader.next(inputPair, ignore)) { long testl = Long.parseLong(inputPair.datum().toString().split(":")[2].replace("}", "").trim()); Assert.assertEquals(onel, testl); } } }
path.getFileSystem(CONFIGURATION).setVerifyChecksum(true); File file = new File(filePath); return new FileSplit(path, 0, file.length(), new String[0]);
protected static FileSplit getFileSplit(Job vectorJob) throws IOException, InterruptedException { ParquetInputFormat parquetInputFormat = new ParquetInputFormat(GroupReadSupport.class); InputSplit split = (InputSplit) parquetInputFormat.getSplits(vectorJob).get(0); FileSplit fsplit = new FileSplit(file, 0L, split.getLength(), split.getLocations()); return fsplit; }
@SuppressWarnings("deprecation") // Test for a different schema output public void testProjection1(String inputDirectory) throws Exception { JobConf job = new JobConf(); Schema readerSchema = Schema.create(Schema.Type.STRING); AvroJob.setInputSchema(job, readerSchema); Path inputPath = new Path(inputDirectory + "/myavro1-r-00000.avro"); FileStatus fileStatus = FileSystem.get(job).getFileStatus(inputPath); FileSplit fileSplit = new FileSplit(inputPath, 0, fileStatus.getLen(), job); AvroWrapper<Utf8> inputPair = new AvroWrapper<>(null); NullWritable ignore = NullWritable.get(); AvroRecordReader<Utf8> recordReader = new AvroRecordReader<>(job, fileSplit); long sumOfCounts = 0; long numOfCounts = 0; while (recordReader.next(inputPair, ignore)) { sumOfCounts += Long.parseLong(inputPair.datum().toString().split(":")[2].replace("}", "").trim()); numOfCounts++; } Assert.assertEquals(numOfCounts, WordCountUtil.COUNTS.size()); long actualSumOfCounts = 0; for (Long count : WordCountUtil.COUNTS.values()) { actualSumOfCounts += count; } Assert.assertEquals(sumOfCounts, actualSumOfCounts); }
@SuppressWarnings("deprecation") // Test for a different schema output public void testProjectionNewMethodsTwo(String inputDirectory) throws Exception { JobConf job = new JobConf(); Schema readerSchema = Schema.create(Schema.Type.STRING); AvroJob.setInputSchema(job, readerSchema); Path inputPath = new Path(inputDirectory + "/testavrofile1-r-00000.avro"); FileStatus fileStatus = FileSystem.get(job).getFileStatus(inputPath); FileSplit fileSplit = new FileSplit(inputPath, 0, fileStatus.getLen(), job); AvroWrapper<Utf8> inputPair = new AvroWrapper<>(null); NullWritable ignore = NullWritable.get(); AvroRecordReader<Utf8> recordReader = new AvroRecordReader<>(job, fileSplit); long sumOfCounts = 0; long numOfCounts = 0; while (recordReader.next(inputPair, ignore)) { sumOfCounts += Long.parseLong(inputPair.datum().toString().split(":")[2].replace("}", "").trim()); numOfCounts++; } Assert.assertEquals(numOfCounts, WordCountUtil.COUNTS.size()); long actualSumOfCounts = 0; for (Long count : WordCountUtil.COUNTS.values()) { actualSumOfCounts += count; } Assert.assertEquals(sumOfCounts, actualSumOfCounts); }
path.getFileSystem(new Configuration()).setVerifyChecksum(true); File file = new File(filePath); return new FileSplit(path, 0, file.length(), new String[0]);
public static List<ArrayWritable> read(Path parquetFile) throws IOException { List<ArrayWritable> records = new ArrayList<ArrayWritable>(); RecordReader<NullWritable, ArrayWritable> reader = new MapredParquetInputFormat(). getRecordReader(new FileSplit( parquetFile, 0, fileLength(parquetFile), (String[]) null), new JobConf(), null); NullWritable alwaysNull = reader.createKey(); ArrayWritable record = reader.createValue(); while (reader.next(alwaysNull, record)) { records.add(record); record = reader.createValue(); // a new value so the last isn't clobbered } return records; }
new FileSplit(new Path(tempFile.getFile().getAbsolutePath()), 0, tempFile.getFile().length(), (String[]) null), configuration, NULL);
FileSplit fileSplit = new FileSplit(inputPath, 0, fileStatus.getLen(), job);
FileSplit fileSplit = new FileSplit(inputPath, 0, fileStatus.getLen(), job);
FileSplit fileSplit = new FileSplit(inputPath, 0, fileStatus.getLen(), job);
@Test public void testSplitStartsWithOffset() throws IOException { createOrcFile(DEFAULT_STRIPE_SIZE, DEFAULT_STRIPE_SIZE + 1); FileStatus fileStatus = fs.getFileStatus(tmpPath); long length = fileStatus.getLen(); long offset = length / 2; FileSplit split = new FileSplit(tmpPath, offset, length, (String[])null); OrcFileStripeMergeRecordReader reader = new OrcFileStripeMergeRecordReader(conf, split); reader.next(key, value); Assert.assertEquals("InputPath", tmpPath, key.getInputPath()); Assert.assertEquals("NumberOfValues", 1L, value.getStripeStatistics().getColStats(0).getNumberOfValues()); reader.close(); }
FileSplit fsplit = new FileSplit(hsplit.getPaths()[partition], hsplit .getStartOffsets()[partition], hsplit.getLengths()[partition], hsplit .getLocations());