public static GeoWaveRDD loadRDD(final SparkContext sc, final DataStorePluginOptions storeOptions) throws IOException { final RDDOptions defaultOptions = new RDDOptions(); return GeoWaveRDDLoader.loadRDD(sc, storeOptions, defaultOptions); }
public static GeoWaveRDD loadRDD( final SparkContext sc, final DataStorePluginOptions storeOptions, final RDDOptions rddOpts) throws IOException { final JavaPairRDD<GeoWaveInputKey, SimpleFeature> rawRDD = GeoWaveRDDLoader.loadRawRDD(sc, storeOptions, rddOpts); return new GeoWaveRDD(rawRDD); }
GeoWaveRDDLoader.loadIndexedRDD(session.sparkContext(), leftStore.rdd, leftStrat)); joinRunner.setRightRDD( GeoWaveRDDLoader.loadIndexedRDD(session.sparkContext(), rightStore.rdd, rightStrat));
return GeoWaveRDDLoader.loadIndexedRDD(sc, storeOptions, rddOpts, rddStrategy);
public static GeoWaveIndexedRDD loadIndexedRDD( final SparkContext sc, final DataStorePluginOptions storeOptions, final RDDOptions rddOpts, final NumericIndexStrategy indexStrategy) throws IOException { final GeoWaveRDD wrappedRDD = GeoWaveRDDLoader.loadRDD(sc, storeOptions, rddOpts); if (wrappedRDD == null) { return null; } // Index strategy can be expensive so we will broadcast it and store it Broadcast<NumericIndexStrategy> broadcastStrategy = null; if (indexStrategy != null) { broadcastStrategy = (Broadcast<NumericIndexStrategy>) RDDUtils.broadcastIndexStrategy(sc, indexStrategy); } final GeoWaveIndexedRDD returnRDD = new GeoWaveIndexedRDD(wrappedRDD, broadcastStrategy); return returnRDD; }
private void loadStoresAndViews() throws IOException { final Collection<InputStoreInfo> addStores = inputStores.values(); for (final InputStoreInfo storeInfo : addStores) { final RDDOptions rddOpts = new RDDOptions(); rddOpts.setQuery(QueryBuilder.newBuilder().addTypeName(storeInfo.typeName).build()); storeInfo.rdd = GeoWaveRDDLoader.loadRDD(session.sparkContext(), storeInfo.storeOptions, rddOpts); // Create a DataFrame from the Left RDD final SimpleFeatureDataFrame dataFrame = new SimpleFeatureDataFrame(session); if (!dataFrame.init(storeInfo.storeOptions, storeInfo.typeName)) { LOGGER.error("Failed to initialize dataframe"); return; } LOGGER.debug(dataFrame.getSchema().json()); final Dataset<Row> dfTemp = dataFrame.getDataFrame(storeInfo.rdd); dfTemp.createOrReplaceTempView(storeInfo.viewName); } }
rddOpts.setMaxSplits(maxSplits); final JavaPairRDD<GeoWaveInputKey, SimpleFeature> javaRdd = GeoWaveRDDLoader.loadRDD(context.sc(), inputStoreOptions, rddOpts).getRawRDD();
try { count = GeoWaveRDDLoader.loadRDD( session.sparkContext(), pluginOptions,
kmeansOpts.setQuery(bldr.build()); final GeoWaveRDD kmeansRDD = GeoWaveRDDLoader.loadRDD(session.sparkContext(), inputDataStore, kmeansOpts);
queryOpts.setMinSplits(DEFAULT_SPLITS_FOR_COUNT); queryOpts.setMaxSplits(DEFAULT_SPLITS_FOR_COUNT); final GeoWaveRDD newRDD = GeoWaveRDDLoader.loadRDD(context, dataStore, queryOpts); final JavaPairRDD<GeoWaveInputKey, SimpleFeature> javaRdd = newRDD.getRawRDD(); final long count = getCount(javaRdd, dataStore.getType());