final InetSocketAddress address = new InetSocketAddress(gateway.getHostname(), blobServerPort); try { ClientUtils.extractAndUploadJobGraphFiles(jobGraph, () -> new BlobClient(address, configuration)); } catch (FlinkException e) { throw new CompletionException(e);
/** * Uploads data from the given byte array to the BLOB server in a content-addressable manner. * * @param value * the buffer to upload data from * @param offset * the read offset within the buffer * @param len * the number of bytes to upload from the buffer * @return the computed BLOB key identifying the BLOB on the server * @throws IOException * thrown if an I/O error occurs while uploading the data to the BLOB server */ public BlobKey put(byte[] value, int offset, int len) throws IOException { return putBuffer(null, null, value, offset, len); }
/** * Uploads the data from the given input stream to the BLOB server in a content-addressable manner. * * @param inputStream * the input stream to read the data from * @return the computed BLOB key identifying the BLOB on the server * @throws IOException * thrown if an I/O error occurs while reading the data from the input stream or uploading the data to the * BLOB server */ public BlobKey put(InputStream inputStream) throws IOException { return putInputStream(null, null, inputStream); }
List<PermanentBlobKey> blobKeys = new ArrayList<>(); try (BlobClient blobClient = new BlobClient(serverAddress, clientConfig)) { for (final Path file : files) { final PermanentBlobKey key = blobClient.uploadFile(jobId, file); blobKeys.add(key);
while (true) { try ( final BlobClient bc = new BlobClient(serverAddress, blobClientConfig); final InputStream is = bc.getInternal(jobId, blobKey); final OutputStream os = new FileOutputStream(localJarFile) ) {
sendGetHeader(os, jobId, blobKey); receiveAndCheckGetResponse(is);
bc = new BlobClient(serverAddress, blobClientConfig); try { is = fs.open(jar); final BlobKey key = bc.put(is); this.userJarBlobKeys.add(key); bc.close();
bc = new BlobClient(serverAddress, blobClientConfig); is = bc.get(requiredBlob); os = new FileOutputStream(localJarFile); is.close(); is = null; bc.close(); bc = null;
List<BlobKey> blobKeys = new ArrayList<>(); try (BlobClient blobClient = new BlobClient(serverAddress, clientConfig)) { for (final Path jar : jars) { final FileSystem fs = jar.getFileSystem(); try { is = fs.open(jar); final BlobKey key = blobClient.put(is); blobKeys.add(key); } finally {
private static Collection<PermanentBlobKey> uploadUserJars(JobID jobId, Collection<Path> userJars, BlobClient blobClient) throws IOException { Collection<PermanentBlobKey> blobKeys = new ArrayList<>(userJars.size()); for (Path jar : userJars) { final PermanentBlobKey blobKey = blobClient.uploadFile(jobId, jar); blobKeys.add(blobKey); } return blobKeys; }
sendGetHeader(os, null, null, blobKey); receiveAndCheckResponse(is);
/** * Uploads a single file to the {@link PermanentBlobService} of the given {@link BlobServer}. * * @param jobId * ID of the job this blob belongs to (or <tt>null</tt> if job-unrelated) * @param file * file to upload * * @throws IOException * if the upload fails */ public PermanentBlobKey uploadFile(JobID jobId, Path file) throws IOException { final FileSystem fs = file.getFileSystem(); if (fs.getFileStatus(file).isDir()) { return uploadDirectory(jobId, file, fs); } else { try (InputStream is = fs.open(file)) { return (PermanentBlobKey) putInputStream(jobId, is, PERMANENT_BLOB); } } }
BlobClient.downloadFromBlobServer( jobId, blobKey, incomingFile, currentServerAddress, blobClientConfig, numFetchRetries);
/** * Uploads the previously added user JAR files to the job manager through * the job manager's BLOB server. The respective port is retrieved from the * JobManager. This function issues a blocking call. * * @param jobManager JobManager actor gateway * @param askTimeout Ask timeout * @param blobClientConfig the blob client configuration * @throws IOException Thrown, if the file upload to the JobManager failed. */ public void uploadUserJars(ActorGateway jobManager, FiniteDuration askTimeout, Configuration blobClientConfig) throws IOException { List<BlobKey> blobKeys = BlobClient.uploadJarFiles(jobManager, askTimeout, blobClientConfig, userJars); for (BlobKey blobKey : blobKeys) { if (!userJarBlobKeys.contains(blobKey)) { userJarBlobKeys.add(blobKey); } } }
List<PermanentBlobKey> blobKeys = new ArrayList<>(); try (BlobClient blobClient = new BlobClient(serverAddress, clientConfig)) { for (final Path file : files) { final PermanentBlobKey key = blobClient.uploadFile(jobId, file); blobKeys.add(key);
while (true) { try ( final BlobClient bc = new BlobClient(serverAddress, blobClientConfig); final InputStream is = bc.getInternal(jobId, blobKey); final OutputStream os = new FileOutputStream(localJarFile) ) {
sendGetHeader(os, jobId, blobKey); receiveAndCheckGetResponse(is);
private static Collection<PermanentBlobKey> uploadUserJars(JobID jobId, Collection<Path> userJars, BlobClient blobClient) throws IOException { Collection<PermanentBlobKey> blobKeys = new ArrayList<>(userJars.size()); for (Path jar : userJars) { final PermanentBlobKey blobKey = blobClient.uploadFile(jobId, jar); blobKeys.add(blobKey); } return blobKeys; }
InputStream is = this.socket.getInputStream(); sendGetHeader(os, jobID, key, null); receiveAndCheckResponse(is);
BlobClient.downloadFromBlobServer( jobId, blobKey, incomingFile, currentServerAddress, blobClientConfig, numFetchRetries);