/** * Adds a list of {@link SparkHttpServiceHandler}s to runs in the Spark driver. */ protected void addHandlers(Iterable<? extends SparkHttpServiceHandler> handlers) { getConfigurer().addHandlers(handlers); } }
/** * Sets the resources requirement for the Spark client process. */ protected final void setClientResources(Resources resources) { configurer.setClientResources(resources); }
@Override public final void configure(SparkConfigurer configurer) { this.configurer = configurer; configure(); }
/** * Creates a {@link JavaPairRDD} from the given {@link Dataset} with the given set of dataset arguments. * * @param datasetName name of the dataset * @param arguments arguments for the dataset * @param <K> key type * @param <V> value type * @return A new {@link JavaPairRDD} instance that reads from the given dataset * @throws DatasetInstantiationException if the dataset doesn't exist */ public <K, V> JavaPairRDD<K, V> fromDataset(String datasetName, Map<String, String> arguments) { return fromDataset(datasetName, arguments, null); }
@Override public void run(final JavaSparkExecutionContext sec) throws Exception { sec.execute(new TxRunnable() { @Override public void run(DatasetContext context) throws Exception { KeyValueTable clusterNameTable = context.getDataset(CLUSTER_NAME_TABLE); WorkflowInfo workflowInfo = sec.getWorkflowInfo(); String prefix = workflowInfo == null ? "" : workflowInfo.getName() + "."; clusterNameTable.write(prefix + "spark.cluster.name", sec.getClusterName()); } }); } }
@Override public Void call() throws Exception { delegate.run(sec); return null; } });
@Override public void destroy() { if (delegateSpark != null) { if (delegateSpark instanceof AbstractSpark) { ((AbstractSpark) delegateSpark).destroy(); } } } }
/** * Creates a {@link JavaRDD} that represents all events from the given stream. * * @param namespace namespace in which the stream exists * @param streamName name of the stream * @return A new {@link JavaRDD} instance that reads from the given stream * @throws DatasetInstantiationException if the stream doesn't exist */ public JavaRDD<StreamEvent> fromStream(String namespace, String streamName) { return fromStream(namespace, streamName, 0, Long.MAX_VALUE); }
@Override public <K, V> JavaPairRDD<K, V> fromStream(String streamName, long startTime, long endTime, Class<? extends StreamEventDecoder<K, V>> decoderClass, Class<K> keyType, Class<V> valueType) { return sec.fromStream(streamName, startTime, endTime, decoderClass, keyType, valueType); }
/** * Sets the name of the {@link Spark}. */ protected final void setName(String name) { configurer.setName(name); }
/** * Sets the description of the {@link Spark}. */ protected final void setDescription(String description) { configurer.setDescription(description); }
/** * Sets the Spark job main class name in specification. The main method of this class will be called to run the * Spark job * * @param mainClassName the fully qualified name of class containing the main method */ protected final void setMainClassName(String mainClassName) { configurer.setMainClassName(mainClassName); }
/** * Sets a set of properties that will be available through the {@link SparkSpecification#getProperties()} * at runtime. * * @param properties the properties to set */ protected final void setProperties(Map<String, String> properties) { configurer.setProperties(properties); }
/** * Sets the resources requirement for the Spark driver process. */ protected final void setDriverResources(Resources resources) { configurer.setDriverResources(resources); }
/** * Sets the resources requirement for the Spark executor processes. */ protected final void setExecutorResources(Resources resources) { configurer.setExecutorResources(resources); }
/** * Adds a list of {@link SparkHttpServiceHandler}s to runs in the Spark driver. */ protected void addHandlers(Iterable<? extends SparkHttpServiceHandler> handlers) { getConfigurer().addHandlers(handlers); } }
/** * Creates a {@link JavaPairRDD} from the given {@link Dataset} with the given set of dataset arguments. * * @param datasetName name of the dataset * @param arguments arguments for the dataset * @param <K> key type * @param <V> value type * @return A new {@link JavaPairRDD} instance that reads from the given dataset * @throws DatasetInstantiationException if the dataset doesn't exist */ public <K, V> JavaPairRDD<K, V> fromDataset(String datasetName, Map<String, String> arguments) { return fromDataset(datasetName, arguments, null); }
/** * Creates a {@link JavaRDD} that represents all events from the given stream. * * @param streamName name of the stream * @return A new {@link JavaRDD} instance that reads from the given stream * @throws DatasetInstantiationException if the stream doesn't exist */ public JavaRDD<StreamEvent> fromStream(String streamName) { return fromStream(streamName, 0, Long.MAX_VALUE); }
@Override public <K, V> JavaPairRDD<K, V> fromStream(String streamName, long startTime, long endTime, Class<? extends StreamEventDecoder<K, V>> decoderClass, Class<K> keyType, Class<V> valueType) { return sec.fromStream(streamName, startTime, endTime, decoderClass, keyType, valueType); }
/** * Adds a list of {@link SparkHttpServiceHandler}s to runs in the Spark driver. */ protected void addHandlers(Iterable<? extends SparkHttpServiceHandler> handlers) { getConfigurer().addHandlers(handlers); } }