@Override public boolean validateInput(FileSystem fs, HiveConf conf, List<FileStatus> files) throws IOException { if (files.size() <= 0) { return false; } for (int fileId = 0; fileId < files.size(); fileId++) { RCFile.Reader reader = null; try { reader = new RCFile.Reader(fs, files.get(fileId) .getPath(), conf); reader.close(); reader = null; } catch (IOException e) { return false; } finally { if (null != reader) { reader.close(); } } } return true; } }
@Override public boolean validateInput(FileSystem fs, HiveConf conf, List<FileStatus> files) throws IOException { if (files.size() <= 0) { return false; } for (int fileId = 0; fileId < files.size(); fileId++) { RCFile.Reader reader = null; try { reader = new RCFile.Reader(fs, files.get(fileId) .getPath(), conf); reader.close(); reader = null; } catch (IOException e) { return false; } finally { if (null != reader) { reader.close(); } } } return true; } }
public void initIOContext(FileSplit split, JobConf job, Class inputFormatClass, RecordReader recordReader) throws IOException { boolean blockPointer = false; long blockStart = -1; FileSplit fileSplit = split; Path path = fileSplit.getPath(); FileSystem fs = path.getFileSystem(job); if (inputFormatClass.getName().contains("SequenceFile")) { SequenceFile.Reader in = new SequenceFile.Reader(fs, path, job); blockPointer = in.isBlockCompressed(); in.sync(fileSplit.getStart()); blockStart = in.getPosition(); in.close(); } else if (recordReader instanceof RCFileRecordReader) { blockPointer = true; blockStart = ((RCFileRecordReader) recordReader).getStart(); } else if (inputFormatClass.getName().contains("RCFile")) { blockPointer = true; RCFile.Reader in = new RCFile.Reader(fs, path, job); in.sync(fileSplit.getStart()); blockStart = in.getPosition(); in.close(); } this.jobConf = job; this.initIOContext(blockStart, blockPointer, path.makeQualified(fs)); this.initIOContextSortedProps(split, recordReader, job); }
public void close() throws IOException { in.close(); } }
@Override public void close() throws IOException { in.close(); }
public void close() throws IOException { in.close(); } }
@Override public void close() { // some hive input formats are broken and bad things can happen if you close them multiple times if (closed) { return; } closed = true; recordReader.close(); }
@Override public void close() throws IOException { in.close(); }
reader.close();
reader.close();
reader.close();
assertEquals("Serialized data", patialS, serializedBytes); reader.close(); long cost = System.currentTimeMillis() - start; LOG.debug("reading fully costs:" + cost + " milliseconds");
actualRead++; reader.close(); assertEquals("Expect " + count + " rows, actual read " + actualRead, actualRead, count);
public void initIOContext(FileSplit split, JobConf job, Class inputFormatClass, RecordReader recordReader) throws IOException { boolean blockPointer = false; long blockStart = -1; FileSplit fileSplit = split; Path path = fileSplit.getPath(); FileSystem fs = path.getFileSystem(job); if (inputFormatClass.getName().contains("SequenceFile")) { SequenceFile.Reader in = new SequenceFile.Reader(fs, path, job); blockPointer = in.isBlockCompressed(); in.sync(fileSplit.getStart()); blockStart = in.getPosition(); in.close(); } else if (recordReader instanceof RCFileRecordReader) { blockPointer = true; blockStart = ((RCFileRecordReader) recordReader).getStart(); } else if (inputFormatClass.getName().contains("RCFile")) { blockPointer = true; RCFile.Reader in = new RCFile.Reader(fs, path, job); in.sync(fileSplit.getStart()); blockStart = in.getPosition(); in.close(); } this.jobConf = job; this.initIOContext(blockStart, blockPointer, path.makeQualified(fs)); this.initIOContextSortedProps(split, recordReader, job); }
public void close() throws IOException { in.close(); }
@Test public void testReadOldFileHeader() throws IOException { String[] row = new String[]{"Tester", "Bart", "333 X St.", "Reno", "NV", "USA"}; RCFile.Reader reader = new RCFile.Reader(fs, new Path(HiveTestUtils.getFileFromClasspath("rc-file-v0.rc")), conf); LongWritable rowID = new LongWritable(); BytesRefArrayWritable cols = new BytesRefArrayWritable(); assertTrue("old file reader first row", reader.next(rowID)); reader.getCurrentRow(cols); assertEquals(row.length, cols.size()); for (int i=0; i < cols.size(); ++i) { assertEquals(row[i], new String(cols.get(i).getBytesCopy())); } assertFalse("old file reader end", reader.next(rowID)); reader.close(); }
public void close() throws IOException { in.close(); } }
public void close() throws IOException { in.close(); }
public void close() throws IOException { in.close(); } }
@Override public void close() throws IOException { in.close(); }