static <R> CheckedSupplier<R> checked(Supplier<R> supplier) { return () -> { try { return supplier.get(); } catch (RuntimeException e) { throw new FlinkException(e); } }; } }
static FlinkException createAndLogException(PyException pe, Logger log) { StringWriter sw = new StringWriter(); try (PrintWriter pw = new PrintWriter(sw)) { pe.printStackTrace(pw); } String pythonStackTrace = sw.toString().trim(); log.error("Python function failed: " + System.lineSeparator() + pythonStackTrace); return new FlinkException("Python function failed: " + pythonStackTrace); } }
private static <R> OptionalFailure<R> wrapUnchecked(String name, Supplier<R> supplier) { return OptionalFailure.createFrom(() -> { try { return supplier.get(); } catch (RuntimeException ex) { LOG.error("Unexpected error while handling accumulator [" + name + "]", ex); throw new FlinkException(ex); } }); }
private PackagedProgram createPackagedProgram() throws FlinkException { try { final Class<?> mainClass = getClass().getClassLoader().loadClass(jobClassName); return new PackagedProgram(mainClass, programArguments); } catch (ClassNotFoundException | ProgramInvocationException e) { throw new FlinkException("Could not load the provided entrypoint class.", e); } } }
/** * @return stored value or throw a {@link FlinkException} with {@code failureCause}. */ public T get() throws FlinkException { if (value != null) { return value; } checkNotNull(failureCause); throw new FlinkException(failureCause); }
default void close() throws Exception { try { closeAsync().get(); } catch (ExecutionException e) { throw new FlinkException("Could not close resource.", ExceptionUtils.stripExecutionException(e)); } }
@Override public void invoke() throws Exception { if (hasFailed.compareAndSet(false, true)) { throw new FlinkException("One time failure."); } } }
@Override public void killCluster(ApplicationId applicationId) throws FlinkException { try { yarnClient.killApplication(applicationId); Utils.deleteApplicationFiles(Collections.singletonMap( YarnConfigKeys.FLINK_YARN_FILES, getYarnFilesDir(applicationId).toUri().toString())); } catch (YarnException | IOException e) { throw new FlinkException("Could not kill the Yarn Flink cluster with id " + applicationId + '.', e); } }
@Override public void triggerCheckpoint(long checkpointId) throws FlinkException { // TODO - we need to see how to derive those. We should probably not encode this in the // TODO - source's trigger message, but do a handshake in this task between the trigger // TODO - message from the master, and the source's trigger notification final CheckpointOptions checkpointOptions = CheckpointOptions.forCheckpointWithDefaultLocation(); final long timestamp = System.currentTimeMillis(); final CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, timestamp); try { SourceStreamTask.super.triggerCheckpoint(checkpointMetaData, checkpointOptions); } catch (RuntimeException | FlinkException e) { throw e; } catch (Exception e) { throw new FlinkException(e.getMessage(), e); } } };
@Test public void testInvalidExceptionStripping() { final FlinkException expectedException = new FlinkException(new RuntimeException(new FlinkException("inner exception"))); final Throwable strippedException = ExceptionUtils.stripException(expectedException, RuntimeException.class); assertThat(strippedException, is(equalTo(expectedException))); } }
@Test public void testExceptionStripping() { final FlinkException expectedException = new FlinkException("test exception"); final Throwable strippedException = ExceptionUtils.stripException(new RuntimeException(new RuntimeException(expectedException)), RuntimeException.class); assertThat(strippedException, is(equalTo(expectedException))); }
@Override public void close(boolean cleanup) throws Exception { Throwable exception = null; try { actorSystem.shutdown(); } catch (Throwable t) { exception = ExceptionUtils.firstOrSuppressed(t, exception); } try { artifactServer.stop(); } catch (Throwable t) { exception = ExceptionUtils.firstOrSuppressed(t, exception); } if (exception != null) { throw new FlinkException("Could not properly shut down the Mesos services.", exception); } } }
/** * Sends a {@link JobManagerMessages.DisposeSavepoint} message to the job manager. */ private void disposeSavepoint(ClusterClient<?> clusterClient, String savepointPath) throws FlinkException { Preconditions.checkNotNull(savepointPath, "Missing required argument: savepoint path. " + "Usage: bin/flink savepoint -d <savepoint-path>"); logAndSysout("Disposing savepoint '" + savepointPath + "'."); final CompletableFuture<Acknowledge> disposeFuture = clusterClient.disposeSavepoint(savepointPath); logAndSysout("Waiting for response..."); try { disposeFuture.get(clientTimeout.toMillis(), TimeUnit.MILLISECONDS); } catch (Exception e) { throw new FlinkException("Disposing the savepoint '" + savepointPath + "' failed.", e); } logAndSysout("Savepoint '" + savepointPath + "' disposed."); }
@Override public SnapshotResult<OperatorStateHandle> call() throws Exception { // notify that we have started the asynchronous checkpointed operation CHECKPOINTING_LATCH.trigger(); // wait until we have reached the StreamTask#cleanup --> This will already cancel this FutureTask CLEANUP_LATCH.await(); // now throw exception to fail the async checkpointing operation if it has not already been cancelled // by the StreamTask in the meantime throw new FlinkException("Checkpointing operation failed"); } }
/** * Sends a {@link org.apache.flink.runtime.messages.JobManagerMessages.TriggerSavepoint} * message to the job manager. */ private String triggerSavepoint(ClusterClient<?> clusterClient, JobID jobId, String savepointDirectory) throws FlinkException { logAndSysout("Triggering savepoint for job " + jobId + '.'); CompletableFuture<String> savepointPathFuture = clusterClient.triggerSavepoint(jobId, savepointDirectory); logAndSysout("Waiting for response..."); final String savepointPath; try { savepointPath = savepointPathFuture.get(); } catch (Exception e) { Throwable cause = ExceptionUtils.stripExecutionException(e); throw new FlinkException("Triggering a savepoint for the job " + jobId + " failed.", cause); } logAndSysout("Savepoint completed. Path: " + savepointPath); logAndSysout("You can resume your program from this savepoint with the run command."); return savepointPath; }
@Override public JobGraph retrieveJobGraph(Configuration configuration) throws FlinkException { final PackagedProgram packagedProgram = createPackagedProgram(); final int defaultParallelism = configuration.getInteger(CoreOptions.DEFAULT_PARALLELISM); try { final JobGraph jobGraph = PackagedProgramUtils.createJobGraph( packagedProgram, configuration, defaultParallelism, FIXED_JOB_ID); jobGraph.setAllowQueuedScheduling(true); jobGraph.setSavepointRestoreSettings(savepointRestoreSettings); return jobGraph; } catch (Exception e) { throw new FlinkException("Could not create the JobGraph from the provided user code jar.", e); } }
@Test public void testClosePartitionDiscovererWhenOpenThrowException() throws Exception { final RuntimeException failureCause = new RuntimeException(new FlinkException("Test partition discoverer exception")); final FailingPartitionDiscoverer failingPartitionDiscoverer = new FailingPartitionDiscoverer(failureCause); final DummyFlinkKafkaConsumer<String> consumer = new DummyFlinkKafkaConsumer<>(failingPartitionDiscoverer); testFailingConsumerLifecycle(consumer, failureCause); assertTrue("partitionDiscoverer should be closed when consumer is closed", failingPartitionDiscoverer.isClosed()); }
@Test public void testClosePartitionDiscovererWhenKafkaFetcherFails() throws Exception { final FlinkException failureCause = new FlinkException("Run Kafka fetcher failure."); // in this scenario, the partition discoverer will be concurrently accessed; // use the WakeupBeforeCloseTestingPartitionDiscoverer to verify that we always call // wakeup() before closing the discoverer final WakeupBeforeCloseTestingPartitionDiscoverer testPartitionDiscoverer = new WakeupBeforeCloseTestingPartitionDiscoverer(); final AbstractFetcher<String, ?> mock = (AbstractFetcher<String, ?>) mock(AbstractFetcher.class); doThrow(failureCause).when(mock).runFetchLoop(); final DummyFlinkKafkaConsumer<String> consumer = new DummyFlinkKafkaConsumer<>(() -> mock, testPartitionDiscoverer, 100L); testFailingConsumerLifecycle(consumer, failureCause); assertTrue("partitionDiscoverer should be closed when consumer is closed", testPartitionDiscoverer.isClosed()); }
@Test public void testClosePartitionDiscovererWhenCreateKafkaFetcherFails() throws Exception { final FlinkException failureCause = new FlinkException("Create Kafka fetcher failure."); final DummyPartitionDiscoverer testPartitionDiscoverer = new DummyPartitionDiscoverer(); final DummyFlinkKafkaConsumer<String> consumer = new DummyFlinkKafkaConsumer<>( () -> { throw failureCause; }, testPartitionDiscoverer, 100L); testFailingConsumerLifecycle(consumer, failureCause); assertTrue("partitionDiscoverer should be closed when consumer is closed", testPartitionDiscoverer.isClosed()); }
@Test public void testUnknownJobId() throws Exception { // test unknown job Id JobID jid = new JobID(); String[] parameters = { jid.toString() }; String expectedMessage = "Test exception"; FlinkException testException = new FlinkException(expectedMessage); final ClusterClient<String> clusterClient = createClusterClient(testException); MockedCliFrontend testFrontend = new MockedCliFrontend(clusterClient); try { testFrontend.stop(parameters); fail("Should have failed."); } catch (FlinkException e) { assertTrue(ExceptionUtils.findThrowableWithMessage(e, expectedMessage).isPresent()); } }