/** * Closes the connection socket and lets the thread exit. */ public void close() { closeSilently(clientSocket, LOG); interrupt(); }
final InputStream inputStream, final File incomingFile, final byte[] buf) throws IOException { MessageDigest md = BlobUtils.createMessageDigest(); final int bytesExpected = readLength(inputStream); if (bytesExpected == -1) { readFully(inputStream, buf, 0, bytesExpected, "buffer"); fos.write(buf, 0, bytesExpected);
/** * Returns a temporary file inside the BLOB server's incoming directory. * * @return a temporary file inside the BLOB server's incoming directory */ File createTemporaryFilename() { return new File(BlobUtils.getIncomingDirectory(storageDir), String.format("temp-%08d", tempFileCounter.getAndIncrement())); }
/** * Constructs a new BLOB input stream. * * @param wrappedInputStream * the underlying input stream to read from * @param blobKey * the expected BLOB key for content-addressable BLOBs, <code>null</code> for non-content-addressable BLOBs. * @throws IOException * throws if an I/O error occurs while reading the BLOB data from the BLOB server */ BlobInputStream(final InputStream wrappedInputStream, final BlobKey blobKey) throws IOException { this.wrappedInputStream = wrappedInputStream; this.blobKey = blobKey; this.bytesToReceive = readLength(wrappedInputStream); if (this.bytesToReceive < 0) { throw new FileNotFoundException(); } this.md = (blobKey != null) ? BlobUtils.createMessageDigest() : null; }
/** * Reads the key of a BLOB from the given input stream. * * @param buf * auxiliary buffer to data deserialization * @param inputStream * the input stream to read the key from * @return the key of a BLOB * @throws IOException * thrown if an I/O error occurs while reading the key data from the input stream */ private static String readKey(byte[] buf, InputStream inputStream) throws IOException { final int keyLength = readLength(inputStream); if (keyLength > MAX_KEY_LENGTH) { throw new IOException("Unexpected key length " + keyLength); } readFully(inputStream, buf, 0, keyLength, "BlobKey"); return new String(buf, 0, keyLength, BlobUtils.DEFAULT_CHARSET); }
/** * Returns a file handle to the file associated with the given blob key on the blob * server. * * <p><strong>This is only called from the {@link BlobServerConnection}</strong> * * @param key identifying the file * @return file handle to the file */ File getStorageLocation(BlobKey key) { return BlobUtils.getStorageLocation(storageDir, key); }
@Override public void write(int b) throws IOException { writeLength(1, socketStream); socketStream.write(b); md.update((byte) b); }
public static HighAvailabilityServices createAvailableOrEmbeddedServices( Configuration config, Executor executor) throws Exception { HighAvailabilityMode highAvailabilityMode = LeaderRetrievalUtils.getRecoveryMode(config); switch (highAvailabilityMode) { case NONE: return new EmbeddedHaServices(executor); case FILESYSTEM: throw new UnsupportedOperationException("to be implemented"); case ZOOKEEPER: BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config); return new ZooKeeperHaServices( ZooKeeperUtils.startCuratorFramework(config), executor, config, blobStoreService); default: throw new Exception("High availability mode " + highAvailabilityMode + " is not supported."); } }
} else if (mode == JOB_RELATED_CONTENT) { byte[] jidBytes = new byte[JobID.SIZE]; readFully(inputStream, jidBytes, 0, JobID.SIZE, "JobID"); jobId = JobID.fromByteArray(jidBytes); } else { writeLength(blobLen, outputStream);
@Override public boolean put(File localFile, JobID jobId, BlobKey blobKey) throws IOException { return put(localFile, BlobUtils.getStorageLocationPath(basePath, jobId, blobKey)); }
BlobOutputStream(JobID jobID, BlobKey.BlobType blobType, Socket socket) throws IOException { this.blobType = blobType; if (socket.isClosed()) { throw new IllegalStateException("BLOB Client is not connected. " + "Client has been shut down or encountered an error before."); } this.socket = socket; this.socketStream = socket.getOutputStream(); this.md = BlobUtils.createMessageDigest(); sendPutHeader(socketStream, jobID, blobType); }
/** * Creates a BlobStore based on the parameters set in the configuration. * * @param config * configuration to use * * @return a (distributed) blob store for high availability * * @throws IOException * thrown if the (distributed) file storage cannot be created */ public static BlobStoreService createBlobStoreFromConfig(Configuration config) throws IOException { if (HighAvailabilityMode.isHighAvailabilityModeActivated(config)) { return createFileSystemBlobStore(config); } else { return new VoidBlobStore(); } }
} else if (mode == JOB_RELATED_CONTENT) { byte[] jidBytes = new byte[JobID.SIZE]; readFully(inputStream, jidBytes, 0, JobID.SIZE, "JobID"); jobId = JobID.fromByteArray(jidBytes); } else {
private static Throwable readExceptionFromStream(InputStream in) throws IOException { int len = readLength(in); byte[] bytes = new byte[len]; readFully(in, bytes, 0, len, "Error message"); try { return (Throwable) InstantiationUtil.deserializeObject(bytes, ClassLoader.getSystemClassLoader()); } catch (ClassNotFoundException e) { // should never occur throw new IOException("Could not transfer error message", e); } } }
/** * Constructs a new BLOB input stream. * * @param wrappedInputStream * the underlying input stream to read from * @param blobKey * the expected BLOB key for content-addressable BLOBs, <code>null</code> for non-content-addressable BLOBs. * @param wrappedOutputStream * the underlying output stream to write the result to * * @throws IOException * throws if an I/O error occurs while reading the BLOB data from the BLOB server */ BlobInputStream( final InputStream wrappedInputStream, final BlobKey blobKey, OutputStream wrappedOutputStream) throws IOException { this.wrappedInputStream = wrappedInputStream; this.blobKey = blobKey; this.wrappedOutputStream = wrappedOutputStream; this.bytesToReceive = readLength(wrappedInputStream); if (this.bytesToReceive < 0) { throw new FileNotFoundException(); } this.md = (blobKey != null) ? BlobUtils.createMessageDigest() : null; }
/** * Returns a file handle to the file identified by the given jobID and key. * * <p><strong>This is only called from the {@link BlobServerConnection}</strong> * * @param jobID to which the file is associated * @param key to identify the file within the job context * @return file handle to the file */ File getStorageLocation(JobID jobID, String key) { return BlobUtils.getStorageLocation(storageDir, jobID, key); }
@Override public void write(int b) throws IOException { writeLength(1, socketStream); socketStream.write(b); md.update((byte) b); }
public static HighAvailabilityServices createAvailableOrEmbeddedServices( Configuration config, Executor executor) throws Exception { HighAvailabilityMode highAvailabilityMode = LeaderRetrievalUtils.getRecoveryMode(config); switch (highAvailabilityMode) { case NONE: return new EmbeddedHaServices(executor); case ZOOKEEPER: BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config); return new ZooKeeperHaServices( ZooKeeperUtils.startCuratorFramework(config), executor, config, blobStoreService); default: throw new Exception("High availability mode " + highAvailabilityMode + " is not supported."); } }
readFully(inputStream, jidBytes, 0, JobID.SIZE, "JobID"); writeLength(blobLen, outputStream);
@Override public boolean get(JobID jobId, BlobKey blobKey, File localFile) throws IOException { return get(BlobUtils.getStorageLocationPath(basePath, jobId, blobKey), localFile, blobKey); }