private void addJars(String addedJars) { for (String addedJar : CSV_SPLITTER.split(Strings.nullToEmpty(addedJars))) { if (!localJars.contains(addedJar)) { localJars.add(addedJar); sc.addJar(addedJar); } } }
@Override public Serializable call(JobContext jc) throws Exception { jc.sc().addJar(path); // Following remote job may refer to classes in this jar, and the remote job would be executed // in a different thread, so we add this jar path to JobContext for further usage. jc.getAddedJars().put(path, System.currentTimeMillis()); return null; }
private void addJars(String addedJars) { for (String addedJar : CSV_SPLITTER.split(Strings.nullToEmpty(addedJars))) { if (!localJars.contains(addedJar)) { localJars.add(addedJar); sc.addJar(addedJar); } } }
private LocalHiveSparkClient(SparkConf sparkConf, HiveConf hiveConf) throws FileNotFoundException, MalformedURLException { String regJar = null; // the registrator jar should already be in CP when not in test mode if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_IN_TEST)) { String kryoReg = sparkConf.get("spark.kryo.registrator", ""); if (SparkClientUtilities.HIVE_KRYO_REG_NAME.equals(kryoReg)) { regJar = SparkClientUtilities.findKryoRegistratorJar(hiveConf); SparkClientUtilities.addJarToContextLoader(new File(regJar)); } } sc = new JavaSparkContext(sparkConf); if (regJar != null) { sc.addJar(regJar); } jobMetricsListener = new JobMetricsListener(); sc.sc().addSparkListener(jobMetricsListener); }
@Override protected void loadJar(final Configuration hadoopConfiguration, final File file, final Object... params) { final JavaSparkContext sparkContext = (JavaSparkContext) params[0]; sparkContext.addJar(file.getAbsolutePath()); }
@Override protected void loadJar(final Configuration hadoopConfiguration, final File file, final Object... params) { final JavaSparkContext sparkContext = (JavaSparkContext) params[0]; sparkContext.addJar(file.getAbsolutePath()); }
@Override protected void loadJar(final Configuration hadoopConfiguration, final File file, final Object... params) { final JavaSparkContext sparkContext = (JavaSparkContext) params[0]; sparkContext.addJar(file.getAbsolutePath()); }
JavaSparkContext sc = new JavaSparkContext(conf); sc.addJar("./target/SparkPOC-0.0.1-SNAPSHOT-jar-with-dependencies.jar");
private void registerJarIfNotNull(String path) { if (path != null) this.sparkContextReference.get().addJar(path); }
private void addJars(String addedJars) { for (String addedJar : CSV_SPLITTER.split(Strings.nullToEmpty(addedJars))) { if (!localJars.contains(addedJar)) { localJars.add(addedJar); sc.addJar(addedJar); } } }
@Override public Serializable call(JobContext jc) throws Exception { jc.sc().addJar(path); // Following remote job may refer to classes in this jar, and the remote job would be executed // in a different thread, so we add this jar path to JobContext for further usage. jc.getAddedJars().add(path); return null; }
@Override public Serializable call(JobContext jc) throws Exception { jc.sc().addJar(path); // Following remote job may refer to classes in this jar, and the remote job would be executed // in a different thread, so we add this jar path to JobContext for further usage. jc.getAddedJars().add(path); return null; }
public static void main(String args[]) { if (args.length == 0) { System.out.println("JavaHBaseBulkDeleteExample {master} {tableName} "); } String master = args[0]; String tableName = args[1]; JavaSparkContext jsc = new JavaSparkContext(master, "JavaHBaseBulkDeleteExample"); jsc.addJar("SparkHBase.jar"); List<byte[]> list = new ArrayList<byte[]>(); list.add(Bytes.toBytes("1")); list.add(Bytes.toBytes("2")); list.add(Bytes.toBytes("3")); list.add(Bytes.toBytes("4")); list.add(Bytes.toBytes("5")); JavaRDD<byte[]> rdd = jsc.parallelize(list); Configuration conf = HBaseConfiguration.create(); conf.addResource(new Path("/etc/hbase/conf/core-site.xml")); conf.addResource(new Path("/etc/hbase/conf/hbase-site.xml")); JavaHBaseContext hbaseContext = new JavaHBaseContext(jsc, conf); hbaseContext.bulkDelete(rdd, tableName, new DeleteFunction(), 4); }
public static void main(String args[]) { if (args.length == 0) { System.out .println("JavaHBaseBulkGetExample {master} {tableName}"); } String master = args[0]; String tableName = args[1]; JavaSparkContext jsc = new JavaSparkContext(master, "JavaHBaseBulkGetExample"); jsc.addJar("SparkHBase.jar"); List<byte[]> list = new ArrayList<byte[]>(); list.add(Bytes.toBytes("1")); list.add(Bytes.toBytes("2")); list.add(Bytes.toBytes("3")); list.add(Bytes.toBytes("4")); list.add(Bytes.toBytes("5")); JavaRDD<byte[]> rdd = jsc.parallelize(list); Configuration conf = HBaseConfiguration.create(); conf.addResource(new Path("/etc/hbase/conf/core-site.xml")); conf.addResource(new Path("/etc/hbase/conf/hbase-site.xml")); JavaHBaseContext hbaseContext = new JavaHBaseContext(jsc, conf); hbaseContext.bulkGet(tableName, 2, rdd, new GetFunction(), new ResultFunction()); }
public static void main(String args[]) { if (args.length == 0) { System.out .println("JavaHBaseBulkIncrementExample {master} {tableName} {columnFamily}"); } String master = args[0]; String tableName = args[1]; String columnFamily = args[2]; JavaSparkContext jsc = new JavaSparkContext(master, "JavaHBaseBulkIncrementExample"); jsc.addJar("SparkHBase.jar"); List<String> list = new ArrayList<String>(); list.add("1," + columnFamily + ",counter,1"); list.add("2," + columnFamily + ",counter,2"); list.add("3," + columnFamily + ",counter,3"); list.add("4," + columnFamily + ",counter,4"); list.add("5," + columnFamily + ",counter,5"); JavaRDD<String> rdd = jsc.parallelize(list); Configuration conf = HBaseConfiguration.create(); conf.addResource(new Path("/etc/hbase/conf/core-site.xml")); conf.addResource(new Path("/etc/hbase/conf/hbase-site.xml")); JavaHBaseContext hbaseContext = new JavaHBaseContext(jsc, conf); hbaseContext.bulkIncrement(rdd, tableName, new IncrementFunction(), 4); }
public static void main(String args[]) { if (args.length == 0) { System.out .println("JavaHBaseBulkPutExample {master} {tableName} {columnFamily}"); } String master = args[0]; String tableName = args[1]; String columnFamily = args[2]; JavaSparkContext jsc = new JavaSparkContext(master, "JavaHBaseBulkPutExample"); jsc.addJar("SparkHBase.jar"); List<String> list = new ArrayList<String>(); list.add("1," + columnFamily + ",a,1"); list.add("2," + columnFamily + ",a,2"); list.add("3," + columnFamily + ",a,3"); list.add("4," + columnFamily + ",a,4"); list.add("5," + columnFamily + ",a,5"); JavaRDD<String> rdd = jsc.parallelize(list); Configuration conf = HBaseConfiguration.create(); conf.addResource(new Path("/etc/hbase/conf/core-site.xml")); conf.addResource(new Path("/etc/hbase/conf/hbase-site.xml")); JavaHBaseContext hbaseContext = new JavaHBaseContext(jsc, conf); hbaseContext.bulkPut(rdd, tableName, new PutFunction(), true); }
public static void main(String args[]) { if (args.length == 0) { System.out .println("JavaHBaseDistributedScan {master} {tableName}"); } String master = args[0]; String tableName = args[1]; JavaSparkContext jsc = new JavaSparkContext(master, "JavaHBaseDistributedScan"); jsc.addJar("SparkHBase.jar"); Configuration conf = HBaseConfiguration.create(); conf.addResource(new Path("/etc/hbase/conf/core-site.xml")); conf.addResource(new Path("/etc/hbase/conf/hbase-site.xml")); JavaHBaseContext hbaseContext = new JavaHBaseContext(jsc, conf); Scan scan = new Scan(); scan.setCaching(100); JavaRDD<Tuple2<byte[], List<Tuple3<byte[], byte[], byte[]>>>> javaRdd = hbaseContext.hbaseRDD(tableName, scan); List<Tuple2<byte[], List<Tuple3<byte[], byte[], byte[]>>>> results = javaRdd.collect(); results.size(); } }
public static void setUp() { jsc = new JavaSparkContext("local", "JavaHBaseContextSuite"); jsc.addJar("SparkHBase.jar"); tempDir = Files.createTempDir(); tempDir.deleteOnExit(); htu = HBaseTestingUtility.createLocalHTU(); try { System.out.println("cleaning up test dir"); htu.cleanupTestDir(); System.out.println("starting minicluster"); htu.startMiniZKCluster(); htu.startMiniHBaseCluster(1, 1); System.out.println(" - minicluster started"); try { htu.deleteTable(Bytes.toBytes(tableName)); } catch (Exception e) { System.out.println(" - no table " + tableName + " found"); } System.out.println(" - creating table " + tableName); htu.createTable(Bytes.toBytes(tableName), Bytes.toBytes(columnFamily)); System.out.println(" - created table"); } catch (Exception e1) { throw new RuntimeException(e1); } }
jsc.addJar("SparkHBase.jar");
protected void addJarOrPyFile(String path) throws Exception { File localCopyDir = new File(jc.getLocalTmpDir(), "__livy__"); File localCopy = copyFileToLocal(localCopyDir, path, jc.sc().sc()); addLocalFileToClassLoader(localCopy); jc.sc().addJar(path); }