@Override public final void getManifest( ArtifactApi.GetManifestRequest request, StreamObserver<GetManifestResponse> responseObserver) { try { responseObserver.onNext(GetManifestResponse.newBuilder().setManifest(manifest).build()); responseObserver.onCompleted(); } catch (Exception e) { responseObserver.onError(Status.INTERNAL.withCause(e).asException()); } }
@Override public void cancel(CancelJobRequest request, StreamObserver<CancelJobResponse> responseObserver) { LOG.trace("{} {}", CancelJobRequest.class.getSimpleName(), request); responseObserver.onError( Status.NOT_FOUND .withDescription(String.format("Unknown Job ID %s", request.getJobId())) .asException()); }
@Override public void commitManifest( ArtifactApi.CommitManifestRequest request, StreamObserver<ArtifactApi.CommitManifestResponse> responseObserver) { try { commitManifestOrThrow(request, responseObserver); } catch (StatusRuntimeException e) { responseObserver.onError(e); LOG.error("Failed to commit Manifest {}", request.getManifest(), e); } catch (Exception e) { responseObserver.onError( Status.INTERNAL .withCause(e) .withDescription(Throwables.getStackTraceAsString(e)) .asRuntimeException()); LOG.error("Failed to commit Manifest {}", request.getManifest(), e); } }
@Before public void setup() throws IOException { stager = ArtifactServiceStager.overChannel( InProcessChannelBuilder.forName("service_stager").build(), 6); service = new InMemoryArtifactStagerService(); server = InProcessServerBuilder.forName("service_stager") .directExecutor() .addService(service) .build() .start(); }
private static CloseableResource<Server> createJobServer(JobState.Enum jobState) throws IOException { CloseableResource<Server> server = CloseableResource.of( InProcessServerBuilder.forName(ENDPOINT_URL) .addService(new TestJobService(ENDPOINT_DESCRIPTOR, "prepId", "jobId", jobState)) .addService(new InMemoryArtifactStagerService()) .build(), Server::shutdown); server.get().start(); return server; }
private void commitManifestOrThrow( ArtifactApi.CommitManifestRequest request, StreamObserver<ArtifactApi.CommitManifestResponse> responseObserver) throws IOException { Collection<ArtifactApi.ArtifactMetadata> missing = new ArrayList<>(); for (ArtifactApi.ArtifactMetadata artifact : request.getManifest().getArtifactList()) { // TODO: Validate the checksums on the server side, to fail more aggressively if require if (!location.getArtifactFile(artifact.getName()).exists()) { missing.add(artifact); } } if (!missing.isEmpty()) { throw Status.INVALID_ARGUMENT .withDescription( String.format("Attempted to commit manifest with missing Artifacts: [%s]", missing)) .asRuntimeException(); } File mf = location.getManifestFile(); checkState(mf.createNewFile(), "Could not create file to store manifest"); try (OutputStream mfOut = new FileOutputStream(mf)) { request.getManifest().writeTo(mfOut); } responseObserver.onNext( ArtifactApi.CommitManifestResponse.newBuilder() .setRetrievalToken(location.getRootPath()) .build()); responseObserver.onCompleted(); }
responseObserver.onError(actual); } else { Status status = Status.INTERNAL .withCause(actual) .withDescription(Throwables.getStackTraceAsString(actual)); responseObserver.onError(status.asException());
@After public void teardown() { server.shutdownNow(); }
@Override public void run() { try { jobServer = createJobServer(); jobServer.getServer().awaitTermination(); } catch (InterruptedException e) { LOG.warn("Job server interrupted", e); } catch (Exception e) { LOG.warn("Exception during job server creation", e); } finally { stop(); } }
private FileWritingObserver createFile(ArtifactApi.ArtifactMetadata metadata) throws IOException { File destination = location.getArtifactFile(metadata.getName()); if (!destination.createNewFile()) { throw Status.ALREADY_EXISTS .withDescription(String.format("Artifact with name %s already exists", metadata)) .asRuntimeException(); } return new FileWritingObserver( destination, new FileOutputStream(destination), responseObserver); }
CloseableResource.of(jobService, unused -> jobServiceChannel.shutdown())) {
@Override public void getArtifact( ArtifactApi.GetArtifactRequest request, StreamObserver<ArtifactApi.ArtifactChunk> responseObserver) { try { ByteBuffer artifact = getArtifact(request.getName()); do { responseObserver.onNext( ArtifactChunk.newBuilder() .setData( ByteString.copyFrom( artifact, Math.min(artifact.remaining(), DEFAULT_CHUNK_SIZE))) .build()); } while (artifact.hasRemaining()); responseObserver.onCompleted(); } catch (FileNotFoundException e) { responseObserver.onError( Status.INVALID_ARGUMENT .withDescription(String.format("No such artifact %s", request.getName())) .withCause(e) .asException()); } catch (Exception e) { responseObserver.onError( Status.INTERNAL .withDescription( String.format("Could not retrieve artifact with name %s", request.getName())) .withCause(e) .asException()); } }
@Before public void setup() throws Exception { stager = LocalFileSystemArtifactStagerService.forRootDirectory(temporaryFolder.newFolder()); server = InProcessServerBuilder.forName("fs_stager") .directExecutor() .addService(stager) .build() .start(); stub = ArtifactStagingServiceGrpc.newStub( InProcessChannelBuilder.forName("fs_stager").usePlaintext().build()); }
@Override public void onNext(ArtifactApi.PutArtifactRequest value) { try { if (writer == null) { if (!value.getContentCase().equals(ArtifactApi.PutArtifactRequest.ContentCase.METADATA)) { throw Status.INVALID_ARGUMENT .withDescription( String.format( "Expected the first %s to contain the Artifact Name, got %s", ArtifactApi.PutArtifactRequest.class.getSimpleName(), value.getContentCase())) .asRuntimeException(); } writer = createFile(value.getMetadata().getMetadata()); } else { writer.onNext(value); } } catch (StatusRuntimeException e) { responseObserver.onError(e); } catch (Exception e) { responseObserver.onError( Status.INTERNAL .withCause(e) .withDescription(Throwables.getStackTraceAsString(e)) .asRuntimeException()); } }
@After public void teardown() { server.shutdownNow(); }
private static void runServer(ServerConfiguration configuration) throws Exception { ServerFactory serverFactory = ServerFactory.createDefault(); ReferenceRunnerJobService service = ReferenceRunnerJobService.create(serverFactory); try (GrpcFnServer<ReferenceRunnerJobService> server = createServer(configuration, serverFactory, service)) { System.out.println( String.format( "Started %s at %s", ReferenceRunnerJobService.class.getSimpleName(), server.getApiServiceDescriptor().getUrl())); server.getServer().awaitTermination(); } System.out.println("Server shut down, exiting"); }
@Override public void onNext(ArtifactApi.PutArtifactRequest value) { try { if (value.getData() == null) { StatusRuntimeException e = Status.INVALID_ARGUMENT .withDescription( String.format( "Expected all chunks in the current stream state to contain data, got %s", value.getContentCase())) .asRuntimeException(); throw e; } value.getData().getData().writeTo(target); } catch (Exception e) { cleanedUp(e); } }
@Override public void getStateStream( GetJobStateRequest request, StreamObserver<GetJobStateResponse> responseObserver) { LOG.trace("{} {}", GetJobStateRequest.class.getSimpleName(), request); String invocationId = request.getJobId(); try { Thread.sleep(WAIT_MS); Enum state = jobStates.getOrDefault(request.getJobId(), Enum.UNRECOGNIZED); responseObserver.onNext(GetJobStateResponse.newBuilder().setState(state).build()); while (Enum.RUNNING.equals(state)) { Thread.sleep(WAIT_MS); state = jobStates.getOrDefault(request.getJobId(), Enum.UNRECOGNIZED); } responseObserver.onNext(GetJobStateResponse.newBuilder().setState(state).build()); } catch (Exception e) { String errMessage = String.format("Encountered Unexpected Exception for Invocation %s", invocationId); LOG.error(errMessage, e); responseObserver.onError(Status.INTERNAL.withCause(e).asException()); } responseObserver.onCompleted(); }
PreparingJob preparingJob = unpreparedJobs.get(preparationId); if (preparingJob == null) { responseObserver.onError( Status.INVALID_ARGUMENT .withDescription(String.format("Unknown Preparation Id %s", preparationId)) .asException()); return; responseObserver.onError(e); preparingJob.getStagingLocation().toFile()); String jobId = "job-" + Integer.toString(ThreadLocalRandom.current().nextInt()); responseObserver.onNext(RunJobResponse.newBuilder().setJobId(jobId).build()); responseObserver.onCompleted(); runningJobs.put(jobId, runner); jobStates.putIfAbsent(jobId, Enum.RUNNING); responseObserver.onError(e); } catch (Exception e) { responseObserver.onError(Status.INTERNAL.withCause(e).asException());
previous == null, "Unexpected existing job with preparation ID %s", preparationId); responseObserver.onNext( PrepareJobResponse.newBuilder() .setPreparationId(preparationId) responseObserver.onCompleted(); } catch (Exception e) { LOG.error("Could not prepare job with name {}", request.getJobName(), e); responseObserver.onError(Status.INTERNAL.withCause(e).asException());