close(); } catch (Throwable closeThrowable) { LOG.error("Could not properly close the BlobServer.", closeThrowable);
File incomingFile = createTemporaryFilename(); MessageDigest md = BlobUtils.createMessageDigest(); BlobKey blobKey = null; blobKey = moveTempFileToStore(incomingFile, jobId, md.digest(), blobType);
/** * Creates a new BLOB connection for a client request * * @param clientSocket The socket to read/write data. * @param blobServer The BLOB server. */ BlobServerConnection(Socket clientSocket, BlobServer blobServer) { super("BLOB connection for " + clientSocket.getRemoteSocketAddress()); setDaemon(true); if (blobServer == null) { throw new NullPointerException(); } this.clientSocket = clientSocket; this.blobServer = blobServer; this.blobStore = blobServer.getBlobStore(); ReadWriteLock readWriteLock = blobServer.getReadWriteLock(); this.writeLock = readWriteLock.writeLock(); this.readLock = readWriteLock.readLock(); }
commonRpcService.getExecutor()); blobServer = new BlobServer(configuration, haServices.createBlobStore()); blobServer.start(); metricRegistry, resourceManagerRpcService, new ClusterInformation("localhost", blobServer.getPort()), jobManagerMetricGroup); configuration, haServices.createBlobStore(), new InetSocketAddress(InetAddress.getLocalHost(), blobServer.getPort()) ); RestHandlerConfiguration.fromConfiguration(configuration), resourceManagerGatewayRetriever, blobServer.getTransientBlobService(), WebMonitorEndpoint.createExecutorService( configuration.getInteger(RestOptions.SERVER_NUM_THREADS, 1),
protected void initializeServices(Configuration configuration) throws Exception { LOG.info("Initializing cluster services."); synchronized (lock) { final String bindAddress = configuration.getString(JobManagerOptions.ADDRESS); final String portRange = getRPCPortRange(configuration); commonRpcService = createRpcService(configuration, bindAddress, portRange); // update the configuration used to create the high availability services configuration.setString(JobManagerOptions.ADDRESS, commonRpcService.getAddress()); configuration.setInteger(JobManagerOptions.PORT, commonRpcService.getPort()); haServices = createHaServices(configuration, commonRpcService.getExecutor()); blobServer = new BlobServer(configuration, haServices.createBlobStore()); blobServer.start(); heartbeatServices = createHeartbeatServices(configuration); metricRegistry = createMetricRegistry(configuration); // TODO: This is a temporary hack until we have ported the MetricQueryService to the new RpcEndpoint // Start actor system for metric query service on any available port metricQueryServiceActorSystem = MetricUtils.startMetricsActorSystem(configuration, bindAddress, LOG); metricRegistry.startQueryService(metricQueryServiceActorSystem, null); archivedExecutionGraphStore = createSerializableExecutionGraphStore(configuration, commonRpcService.getScheduledExecutor()); transientBlobCache = new TransientBlobCache( configuration, new InetSocketAddress( commonRpcService.getAddress(), blobServer.getPort())); } }
blobFile = blobServer.getStorageLocation(jobId, blobKey); blobServer.getFileInternal(jobId, blobKey, blobFile); } else if (blobKey instanceof TransientBlobKey && result == RETURN_OKAY) { if (!blobServer.deleteInternal(jobId, (TransientBlobKey) blobKey)) { LOG.warn("DELETE operation failed for BLOB {}/{} from {}.", jobId, blobKey, clientSocket.getInetAddress());
@Override public CompletableFuture<Integer> getBlobServerPort(Time timeout) { return CompletableFuture.completedFuture(blobServer.getPort()); }
incomingFile = blobServer.createTemporaryFilename(); fos = new FileOutputStream(incomingFile); File storageFile = blobServer.getStorageLocation(jobID, key); File storageFile = blobServer.getStorageLocation(blobKey);
/** * Creates a new BLOB connection for a client request. * * @param clientSocket The socket to read/write data. * @param blobServer The BLOB server. */ BlobServerConnection(Socket clientSocket, BlobServer blobServer) { super("BLOB connection for " + clientSocket.getRemoteSocketAddress()); setDaemon(true); this.clientSocket = clientSocket; this.blobServer = checkNotNull(blobServer); ReadWriteLock readWriteLock = blobServer.getReadWriteLock(); this.readLock = readWriteLock.readLock(); }
incomingFile = createTemporaryFilename(); blobStore.get(jobId, blobKey, incomingFile);
File blobFile = blobServer.getStorageLocation(key); File blobFile = this.blobServer.getStorageLocation(jobID, key); blobServer.deleteJobDirectory(jobID);
blobFile = blobServer.getStorageLocation(jobId, key); blobFile = blobServer.getStorageLocation(blobKey);
public static JobManagerServices fromConfiguration( Configuration config, HighAvailabilityServices haServices) throws Exception { final BlobServer blobServer = new BlobServer(config, haServices.createBlobStore()); final long cleanupInterval = config.getLong( ConfigConstants.LIBRARY_CACHE_MANAGER_CLEANUP_INTERVAL, ConfigConstants.DEFAULT_LIBRARY_CACHE_MANAGER_CLEANUP_INTERVAL) * 1000; final BlobLibraryCacheManager libraryCacheManager = new BlobLibraryCacheManager(blobServer, cleanupInterval); final FiniteDuration timeout; try { timeout = AkkaUtils.getTimeout(config); } catch (NumberFormatException e) { throw new IllegalConfigurationException(AkkaUtils.formatDurationParingErrorMessage()); } final ScheduledExecutorService futureExecutor = Executors.newScheduledThreadPool( Hardware.getNumberCPUCores(), new ExecutorThreadFactory("jobmanager-future")); return new JobManagerServices( futureExecutor, libraryCacheManager, RestartStrategyFactory.createRestartStrategyFactory(config), Time.of(timeout.length(), timeout.unit())); } }
commonRpcService.getExecutor()); blobServer = new BlobServer(configuration, haServices.createBlobStore()); blobServer.start(); metricRegistry, resourceManagerRpcService, new ClusterInformation("localhost", blobServer.getPort()), jobManagerMetricGroup); configuration, haServices.createBlobStore(), new InetSocketAddress(InetAddress.getLocalHost(), blobServer.getPort()) ); RestHandlerConfiguration.fromConfiguration(configuration), resourceManagerGatewayRetriever, blobServer.getTransientBlobService(), WebMonitorEndpoint.createExecutorService( configuration.getInteger(RestOptions.SERVER_NUM_THREADS, 1),
protected void initializeServices(Configuration configuration) throws Exception { LOG.info("Initializing cluster services."); synchronized (lock) { final String bindAddress = configuration.getString(JobManagerOptions.ADDRESS); final String portRange = getRPCPortRange(configuration); commonRpcService = createRpcService(configuration, bindAddress, portRange); // update the configuration used to create the high availability services configuration.setString(JobManagerOptions.ADDRESS, commonRpcService.getAddress()); configuration.setInteger(JobManagerOptions.PORT, commonRpcService.getPort()); haServices = createHaServices(configuration, commonRpcService.getExecutor()); blobServer = new BlobServer(configuration, haServices.createBlobStore()); blobServer.start(); heartbeatServices = createHeartbeatServices(configuration); metricRegistry = createMetricRegistry(configuration); // TODO: This is a temporary hack until we have ported the MetricQueryService to the new RpcEndpoint // Start actor system for metric query service on any available port metricQueryServiceActorSystem = MetricUtils.startMetricsActorSystem(configuration, bindAddress, LOG); metricRegistry.startQueryService(metricQueryServiceActorSystem, null); archivedExecutionGraphStore = createSerializableExecutionGraphStore(configuration, commonRpcService.getScheduledExecutor()); transientBlobCache = new TransientBlobCache( configuration, new InetSocketAddress( commonRpcService.getAddress(), blobServer.getPort())); } }
blobFile = blobServer.getStorageLocation(jobId, blobKey); blobServer.getFileInternal(jobId, blobKey, blobFile); } else if (blobKey instanceof TransientBlobKey && result == RETURN_OKAY) { if (!blobServer.deleteInternal(jobId, (TransientBlobKey) blobKey)) { LOG.warn("DELETE operation failed for BLOB {}/{} from {}.", jobId, blobKey, clientSocket.getInetAddress());