public void addSampleFile(Path inputPath, JobConf job) throws IOException { FileSystem fs = inputPath.getFileSystem(job); FSDataInputStream input = fs.open(inputPath); try { int count = input.readInt(); for (int i = 0; i < count; i++) { byte[] key = new byte[input.readInt()]; input.readFully(key); sampled.add(key); } } finally { IOUtils.closeStream(input); } }
public void addSampleFile(Path inputPath, JobConf job) throws IOException { FileSystem fs = inputPath.getFileSystem(job); FSDataInputStream input = fs.open(inputPath); try { int count = input.readInt(); for (int i = 0; i < count; i++) { byte[] key = new byte[input.readInt()]; input.readFully(key); sampled.add(key); } } finally { IOUtils.closeStream(input); } }
@Override public GlobalDictMetadata readIndexFile(Path dir) throws IOException { Path indexFile = new Path(dir, V1_INDEX_NAME); try (FSDataInputStream in = fs.open(indexFile)) { int baseId = in.readInt(); int maxId = in.readInt(); int maxValueLength = in.readInt(); int nValues = in.readInt(); String converterName = in.readUTF(); BytesConverter converter; try { converter = ClassUtil.forName(converterName, BytesConverter.class).getDeclaredConstructor().newInstance(); } catch (Exception e) { throw new RuntimeException("Fail to instantiate BytesConverter: " + converterName, e); } int nSlices = in.readInt(); TreeMap<AppendDictSliceKey, String> sliceFileMap = new TreeMap<>(); for (int i = 0; i < nSlices; i++) { AppendDictSliceKey key = new AppendDictSliceKey(); key.readFields(in); sliceFileMap.put(key, sliceFileName(key)); } // make sure first key is always "" String firstFile = sliceFileMap.remove(sliceFileMap.firstKey()); sliceFileMap.put(AppendDictSliceKey.START_KEY, firstFile); return new GlobalDictMetadata(baseId, maxId, maxValueLength, nValues, converter, sliceFileMap); } }
@Override public GlobalDictMetadata readIndexFile(Path dir) throws IOException { Path indexFile = new Path(dir, V2_INDEX_NAME); try (FSDataInputStream in = fs.open(indexFile)) { byte minorVersion = in.readByte(); // include a header to allow minor format changes if (minorVersion != MINOR_VERSION_V1) { throw new RuntimeException("Unsupported minor version " + minorVersion); } int baseId = in.readInt(); int maxId = in.readInt(); int maxValueLength = in.readInt(); int nValues = in.readInt(); String converterName = in.readUTF(); BytesConverter converter; try { converter = ClassUtil.forName(converterName, BytesConverter.class).getDeclaredConstructor().newInstance(); } catch (Exception e) { throw new RuntimeException("Fail to instantiate BytesConverter: " + converterName, e); } int nSlices = in.readInt(); TreeMap<AppendDictSliceKey, String> sliceFileMap = new TreeMap<>(); for (int i = 0; i < nSlices; i++) { AppendDictSliceKey key = new AppendDictSliceKey(); key.readFields(in); String sliceFileName = in.readUTF(); sliceFileMap.put(key, sliceFileName); } return new GlobalDictMetadata(baseId, maxId, maxValueLength, nValues, converter, sliceFileMap); } }
/** * Read and return the next record length, potentially skipping over * a sync block. * @return the length of the next record or -1 if there is no next record * @throws IOException */ private synchronized int readRecordLength() throws IOException { if (in.getPos() >= end) { return -1; } int length = in.readInt(); if (version > 1 && sync != null && length == SYNC_ESCAPE) { // process a sync entry in.readFully(syncCheck); // read syncCheck if (!Arrays.equals(sync, syncCheck)) // check it throw new IOException("File is corrupt!"); syncSeen = true; if (in.getPos() >= end) { return -1; } length = in.readInt(); // re-read length } else { syncSeen = false; } return length; }
/** * Read and return the next record length, potentially skipping over a sync * block. * * @return the length of the next record or -1 if there is no next record * @throws IOException */ private synchronized int readRecordLength() throws IOException { if (in.getPos() >= end) { return -1; } int length = in.readInt(); if (sync != null && length == SYNC_ESCAPE) { // process // a // sync entry lastSeenSyncPos = in.getPos() - 4; // minus SYNC_ESCAPE's length in.readFully(syncCheck); // read syncCheck if (!Arrays.equals(sync, syncCheck)) { throw new IOException("File is corrupt!"); } syncSeen = true; if (in.getPos() >= end) { return -1; } length = in.readInt(); // re-read length } else { syncSeen = false; } return length; }
this.seekOnFs(trailerSizeOffset); int trailerSize = this.inputStream.readInt(); ByteBuffer buf = ByteBuffer.allocate(ProtobufLogReader.PB_WAL_COMPLETE_MAGIC.length); this.inputStream.readFully(buf.array(), buf.arrayOffset(), buf.capacity());
/** * Read and return the next record length, potentially skipping over a sync * block. * * @return the length of the next record or -1 if there is no next record * @throws IOException */ private synchronized int readRecordLength() throws IOException { if (in.getPos() >= end) { return -1; } int length = in.readInt(); if (sync != null && length == SYNC_ESCAPE) { // process // a // sync entry lastSeenSyncPos = in.getPos() - 4; // minus SYNC_ESCAPE's length in.readFully(syncCheck); // read syncCheck if (!Arrays.equals(sync, syncCheck)) { throw new IOException("File is corrupt!"); } syncSeen = true; if (in.getPos() >= end) { return -1; } length = in.readInt(); // re-read length } else { syncSeen = false; } return length; }
/** @deprecated Call {@link #nextRaw(DataOutputBuffer,SequenceFile.ValueBytes)}. */ @Deprecated synchronized int next(DataOutputBuffer buffer) throws IOException { // Unsupported for block-compressed sequence files if (blockCompressed) { throw new IOException("Unsupported call for block-compressed" + " SequenceFiles - use SequenceFile.Reader.next(DataOutputStream, ValueBytes)"); } try { int length = readRecordLength(); if (length == -1) { return -1; } int keyLength = in.readInt(); buffer.write(in, length); return keyLength; } catch (ChecksumException e) { // checksum failure handleChecksumException(e); return next(buffer); } }
/** @throws Exception If failed. */ @Test public void testDeleteSuccessfulIfPathIsOpenedToRead() throws Exception { Path fsHome = new Path(primaryFsUri); final Path file = new Path(fsHome, "myFile"); FSDataOutputStream os = fs.create(file, false, 128); final int cnt = 5 * FileSystemConfiguration.DFLT_BLOCK_SIZE; // Write 5 blocks. for (int i = 0; i < cnt; i++) os.writeInt(i); os.close(); final FSDataInputStream is = fs.open(file, -1); for (int i = 0; i < cnt / 2; i++) assertEquals(i, is.readInt()); assert fs.delete(file, false); assert !fs.exists(file); is.close(); }
public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file, int bufferSize) throws IOException { super( file, fs.getFileStatus(file).getReplication() ); this.datas = fs.getRawFileSystem().open(file, bufferSize); this.fs = fs; Path sumFile = fs.getChecksumFile(file); try { int sumBufferSize = fs.getSumBufferSize(fs.getBytesPerSum(), bufferSize); sums = fs.getRawFileSystem().open(sumFile, sumBufferSize); byte[] version = new byte[CHECKSUM_VERSION.length]; sums.readFully(version); if (!Arrays.equals(version, CHECKSUM_VERSION)) throw new IOException("Not a checksum file: "+sumFile); this.bytesPerSum = sums.readInt(); set(fs.verifyChecksum, DataChecksum.newCrc32(), bytesPerSum, 4); } catch (IOException e) { // mincing the message is terrible, but java throws permission // exceptions as FNF because that's all the method signatures allow! if (!(e instanceof FileNotFoundException) || e.getMessage().endsWith(" (Permission denied)")) { LOG.warn("Problem opening checksum file: "+ file + ". Ignoring exception: " , e); } set(fs.verifyChecksum, null, 1, 0); } }
/** @throws Exception If failed. */ @Test public void testRenameIfSrcPathIsAlreadyBeingOpenedToRead() throws Exception { Path fsHome = new Path(primaryFsUri); Path srcFile = new Path(fsHome, "srcFile"); Path dstFile = new Path(fsHome, "dstFile"); FSDataOutputStream os = fs.create(srcFile); int cnt = 1024; for (int i = 0; i < cnt; i++) os.writeInt(i); os.close(); FSDataInputStream is = fs.open(srcFile); for (int i = 0; i < cnt; i++) { if (i == 100) // Rename file during the read process. assertTrue(fs.rename(srcFile, dstFile)); assertEquals(i, is.readInt()); } assertPathDoesNotExist(fs, srcFile); assertPathExists(fs, dstFile); os.close(); is.close(); }
return -1; keyLength = in.readInt(); key.write(in, keyLength); return keyLength;
public ChecksumFSInputChecker(ChecksumFs fs, Path file, int bufferSize) throws IOException, UnresolvedLinkException { super(file, fs.getFileStatus(file).getReplication()); this.datas = fs.getRawFs().open(file, bufferSize); this.fs = fs; Path sumFile = fs.getChecksumFile(file); try { int sumBufferSize = fs.getSumBufferSize(fs.getBytesPerSum(), bufferSize, file); sums = fs.getRawFs().open(sumFile, sumBufferSize); byte[] version = new byte[CHECKSUM_VERSION.length]; sums.readFully(version); if (!Arrays.equals(version, CHECKSUM_VERSION)) { throw new IOException("Not a checksum file: "+sumFile); } this.bytesPerSum = sums.readInt(); set(fs.verifyChecksum, DataChecksum.newCrc32(), bytesPerSum, 4); } catch (FileNotFoundException e) { // quietly ignore set(fs.verifyChecksum, null, 1, 0); } catch (IOException e) { // loudly ignore LOG.warn("Problem opening checksum file: "+ file + ". Ignoring exception: " , e); set(fs.verifyChecksum, null, 1, 0); } }
/** @throws Exception If failed. */ @Test public void testRenameIfSrcPathIsAlreadyBeingOpenedToRead() throws Exception { Path fsHome = new Path(primaryFsUri); Path srcFile = new Path(fsHome, "srcFile"); Path dstFile = new Path(fsHome, "dstFile"); FSDataOutputStream os = fs.create(srcFile, EnumSet.noneOf(CreateFlag.class), Options.CreateOpts.perms(FsPermission.getDefault())); int cnt = 1024; for (int i = 0; i < cnt; i++) os.writeInt(i); os.close(); FSDataInputStream is = fs.open(srcFile); for (int i = 0; i < cnt; i++) { if (i == 100) // Rename file during the read process. fs.rename(srcFile, dstFile); assertEquals(i, is.readInt()); } assertPathDoesNotExist(fs, srcFile); assertPathExists(fs, dstFile); os.close(); is.close(); }
/** @throws Exception If failed. */ @Test public void testDeleteSuccessfulIfPathIsOpenedToRead() throws Exception { Path fsHome = new Path(primaryFsUri); final Path file = new Path(fsHome, "myFile"); FSDataOutputStream os = fs.create(file, EnumSet.noneOf(CreateFlag.class), Options.CreateOpts.perms(FsPermission.getDefault())); final int cnt = 5 * FileSystemConfiguration.DFLT_BLOCK_SIZE; // Write 5 blocks. for (int i = 0; i < cnt; i++) os.writeInt(i); os.close(); final FSDataInputStream is = fs.open(file, -1); for (int i = 0; i < cnt / 2; i++) assertEquals(i, is.readInt()); assert fs.delete(file, false); GridTestUtils.assertThrows(log, new Callable<Object>() { @Override public Object call() throws Exception { fs.getFileStatus(file); return null; } }, FileNotFoundException.class, null); is.close(); }