long length = fs.getFileStatus(path).getLen(); FSDataInputStream fsdis = fs.open(path); TFile.Reader reader = new TFile.Reader(fsdis, length, conf); try { LinkedHashMap<String, String> properties = properties.put("Data Compression", reader.readerBCF .getDefaultCompressionName()); properties.put("Record Count", Long.toString(reader.getEntryCount())); properties.put("Sorted", Boolean.toString(reader.isSorted())); if (reader.isSorted()) { properties.put("Comparator", reader.getComparatorName()); reader.checkTFileDataIndex(); if (blockCnt > 0) { String blkID = "Data-Block"; String records = "Records"; int recordsWidth = Align.calculateWidth(records, reader.getEntryCount() / blockCnt * 10); String endKey = "End-Key";
/** * Get a scanner that covers a specific key range. * * @param beginKey * Begin key of the scan (inclusive). If null, scan from the first * key-value entry of the TFile. * @param endKey * End key of the scan (exclusive). If null, scan up to the last * key-value entry of the TFile. * @return The actual coverage of the returned scanner will cover all keys * greater than or equal to the beginKey and less than the endKey. * @throws IOException * * @deprecated Use {@link #createScannerByKey(RawComparable, RawComparable)} * instead. */ @Deprecated public Scanner createScanner(RawComparable beginKey, RawComparable endKey) throws IOException { return createScannerByKey(beginKey, endKey); }
FSDataInputStream fin = fs.open(ncTFile); Reader reader = new Reader(fs.open(ncTFile), fs.getFileStatus(ncTFile).getLen(), conf); Scanner scanner = reader.createScanner(); readAllRecords(scanner); scanner.seekTo(getSomeKey(50)); .compareTo(reader.end()) < 0); readKey = readKey(scanner); assertTrue("seeked key does not match", Arrays.equals(readKey, .compareTo(reader.end()) < 0); readKey = readKey(scanner); assertTrue("seeked key does not match", Arrays.equals(readKey, scanner = reader.createScannerByKey(getSomeKey(10), getSomeKey(60)); readAndCheckbytes(scanner, 10, 50); assertFalse(scanner.advance()); scanner.close(); reader.close(); fin.close(); fs.delete(ncTFile, true);
FSDataInputStream fin = fs.open(ncTFile); Reader reader = new Reader(fs.open(ncTFile), fs.getFileStatus(ncTFile).getLen(), conf); Scanner scanner = reader.createScanner(); readAllRecords(scanner); scanner.seekTo(getSomeKey(50)); .compareTo(reader.end()) < 0); readKey = readKey(scanner); assertTrue("seeked key does not match", Arrays.equals(readKey, .compareTo(reader.end()) < 0); readKey = readKey(scanner); assertTrue("seeked key does not match", Arrays.equals(readKey, scanner = reader.createScannerByKey(getSomeKey(10), getSomeKey(60)); readAndCheckbytes(scanner, 10, 50); assertFalse(scanner.advance()); scanner.close(); reader.close(); fin.close(); fs.delete(ncTFile, true);
FSDataInputStream fin = fs.open(ncTFile); Reader reader = new Reader(fs.open(ncTFile), fs.getFileStatus(ncTFile).getLen(), conf); Scanner scanner = reader.createScanner(); readAllRecords(scanner); scanner.seekTo(getSomeKey(50)); .compareTo(reader.end()) < 0); readKey = readKey(scanner); assertTrue("seeked key does not match", Arrays.equals(readKey, .compareTo(reader.end()) < 0); readKey = readKey(scanner); assertTrue("seeked key does not match", Arrays.equals(readKey, scanner = reader.createScannerByKey(getSomeKey(10), getSomeKey(60)); readAndCheckbytes(scanner, 10, 50); assertFalse(scanner.advance()); scanner.close(); reader.close(); fin.close(); fs.delete(ncTFile, true);
void checkRecNums() throws IOException { long fileLen = fs.getFileStatus(path).getLen(); Reader reader = new Reader(fs.open(path), fileLen, conf); long totalRecs = reader.getEntryCount(); long begin = random.nextLong() % (totalRecs / 2); if (begin < 0) .getRecordNumNear(0)); for (long x : new long[] { fileLen, fileLen + 1, 2 * fileLen }) { assertEquals("RecNum for offset>=fileLen should be total entries", totalRecs, reader.getRecordNumNear(x)); .getRecordNumByLocation(reader.getLocationByRecordNum(i))); long x = totalRecs - i; assertEquals("Locaton to RecNum conversion not symmetric", x, reader .getRecordNumByLocation(reader.getLocationByRecordNum(x))); .getRecordNumByLocation(reader.getLocationByRecordNum(i))); if (x < 0) x += totalRecs; assertEquals("Locaton to RecNum conversion not symmetric", x, reader .getRecordNumByLocation(reader.getLocationByRecordNum(x)));
void checkRecNums() throws IOException { long fileLen = fs.getFileStatus(path).getLen(); Reader reader = new Reader(fs.open(path), fileLen, conf); long totalRecs = reader.getEntryCount(); long begin = random.nextLong() % (totalRecs / 2); if (begin < 0) .getRecordNumNear(0)); for (long x : new long[] { fileLen, fileLen + 1, 2 * fileLen }) { assertEquals("RecNum for offset>=fileLen should be total entries", totalRecs, reader.getRecordNumNear(x)); .getRecordNumByLocation(reader.getLocationByRecordNum(i))); long x = totalRecs - i; assertEquals("Locaton to RecNum conversion not symmetric", x, reader .getRecordNumByLocation(reader.getLocationByRecordNum(x))); .getRecordNumByLocation(reader.getLocationByRecordNum(i))); if (x < 0) x += totalRecs; assertEquals("Locaton to RecNum conversion not symmetric", x, reader .getRecordNumByLocation(reader.getLocationByRecordNum(x)));
void checkRecNums() throws IOException { long fileLen = fs.getFileStatus(path).getLen(); Reader reader = new Reader(fs.open(path), fileLen, conf); long totalRecs = reader.getEntryCount(); long begin = random.nextLong() % (totalRecs / 2); if (begin < 0) .getRecordNumNear(0)); for (long x : new long[] { fileLen, fileLen + 1, 2 * fileLen }) { assertEquals("RecNum for offset>=fileLen should be total entries", totalRecs, reader.getRecordNumNear(x)); .getRecordNumByLocation(reader.getLocationByRecordNum(i))); long x = totalRecs - i; assertEquals("Locaton to RecNum conversion not symmetric", x, reader .getRecordNumByLocation(reader.getLocationByRecordNum(x))); .getRecordNumByLocation(reader.getLocationByRecordNum(i))); if (x < 0) x += totalRecs; assertEquals("Locaton to RecNum conversion not symmetric", x, reader .getRecordNumByLocation(reader.getLocationByRecordNum(x)));
public void testScanRange() throws IOException { Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf); Assert.assertFalse(reader.isSorted()); Assert.assertEquals((int) reader.getEntryCount(), 4); Scanner scanner = reader.createScanner(); reader.close();
public void testScan() throws IOException { Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf); Assert.assertFalse(reader.isSorted()); Assert.assertEquals((int) reader.getEntryCount(), 4); Scanner scanner = reader.createScanner(); reader.close();
void readRowSplits(int numSplits) throws IOException { new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf); long totalRecords = reader.getEntryCount(); for (int i=0; i<numSplits; i++) { long startRec = i*totalRecords/numSplits; endRec = totalRecords; Scanner scanner = reader.createScannerByRecordNum(startRec, endRec); int count = 0; BytesWritable key = new BytesWritable(); Scanner scanner = reader.createScannerByRecordNum(totalRecords, -1); Assert.assertTrue(scanner.atEnd());
/** * Constructor * * @param reader * The TFile reader object. * @param beginKey * Begin key of the scan. If null, scan from the first <K,V> * entry of the TFile. * @param endKey * End key of the scan. If null, scan up to the last <K, V> entry * of the TFile. * @throws IOException */ protected Scanner(Reader reader, RawComparable beginKey, RawComparable endKey) throws IOException { this(reader, (beginKey == null) ? reader.begin() : reader .getBlockContainsKey(beginKey, false), reader.end()); if (beginKey != null) { inBlockAdvance(beginKey, false); beginLocation.set(currentLocation); } if (endKey != null) { seekTo(endKey, false); endLocation.set(currentLocation); seekTo(beginLocation); } }
void readRowSplits(int numSplits) throws IOException { new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf); long totalRecords = reader.getEntryCount(); for (int i=0; i<numSplits; i++) { long startRec = i*totalRecords/numSplits; endRec = totalRecords; Scanner scanner = reader.createScannerByRecordNum(startRec, endRec); int count = 0; BytesWritable key = new BytesWritable(); Scanner scanner = reader.createScannerByRecordNum(totalRecords, -1); Assert.assertTrue(scanner.atEnd());
public void testScan() throws IOException { Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf); Assert.assertFalse(reader.isSorted()); Assert.assertEquals((int) reader.getEntryCount(), 4); Scanner scanner = reader.createScanner(); reader.close();
public void testScan() throws IOException { Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf); Assert.assertFalse(reader.isSorted()); Assert.assertEquals((int) reader.getEntryCount(), 4); Scanner scanner = reader.createScanner(); reader.close();
void readRowSplits(int numSplits) throws IOException { new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf); long totalRecords = reader.getEntryCount(); for (int i=0; i<numSplits; i++) { long startRec = i*totalRecords/numSplits; endRec = totalRecords; Scanner scanner = reader.createScannerByRecordNum(startRec, endRec); int count = 0; BytesWritable key = new BytesWritable(); Scanner scanner = reader.createScannerByRecordNum(totalRecords, -1); Assert.assertTrue(scanner.atEnd());
/** * Constructor * * @param reader * The TFile reader object. * @param beginKey * Begin key of the scan. If null, scan from the first <K,V> * entry of the TFile. * @param endKey * End key of the scan. If null, scan up to the last <K, V> entry * of the TFile. * @throws IOException */ protected Scanner(Reader reader, RawComparable beginKey, RawComparable endKey) throws IOException { this(reader, (beginKey == null) ? reader.begin() : reader .getBlockContainsKey(beginKey, false), reader.end()); if (beginKey != null) { inBlockAdvance(beginKey, false); beginLocation.set(currentLocation); } if (endKey != null) { seekTo(endKey, false); endLocation.set(currentLocation); seekTo(beginLocation); } }
/** * Constructor * * @param reader * The TFile reader object. * @param beginKey * Begin key of the scan. If null, scan from the first <K,V> * entry of the TFile. * @param endKey * End key of the scan. If null, scan up to the last <K, V> entry * of the TFile. * @throws IOException */ protected Scanner(Reader reader, RawComparable beginKey, RawComparable endKey) throws IOException { this(reader, (beginKey == null) ? reader.begin() : reader .getBlockContainsKey(beginKey, false), reader.end()); if (beginKey != null) { inBlockAdvance(beginKey, false); beginLocation.set(currentLocation); } if (endKey != null) { seekTo(endKey, false); endLocation.set(currentLocation); seekTo(beginLocation); } }
/** * Constructor * * @param reader * The TFile reader object. * @param beginKey * Begin key of the scan. If null, scan from the first <K,V> * entry of the TFile. * @param endKey * End key of the scan. If null, scan up to the last <K, V> entry * of the TFile. * @throws IOException */ protected Scanner(Reader reader, RawComparable beginKey, RawComparable endKey) throws IOException { this(reader, (beginKey == null) ? reader.begin() : reader .getBlockContainsKey(beginKey, false), reader.end()); if (beginKey != null) { inBlockAdvance(beginKey, false); beginLocation.set(currentLocation); } if (endKey != null) { seekTo(endKey, false); endLocation.set(currentLocation); seekTo(beginLocation); } }
/** * Constructor * * @param reader * The TFile reader object. * @param beginKey * Begin key of the scan. If null, scan from the first <K,V> * entry of the TFile. * @param endKey * End key of the scan. If null, scan up to the last <K, V> entry * of the TFile. * @throws IOException */ protected Scanner(Reader reader, RawComparable beginKey, RawComparable endKey) throws IOException { this(reader, (beginKey == null) ? reader.begin() : reader .getBlockContainsKey(beginKey, false), reader.end()); if (beginKey != null) { inBlockAdvance(beginKey, false); beginLocation.set(currentLocation); } if (endKey != null) { seekTo(endKey, false); endLocation.set(currentLocation); seekTo(beginLocation); } }