@Override public Tuple1<Long> map(Long value) throws Exception { return Tuple1.of(value); }
public static void copyToLocal(final URI remotePath, final File localPath) throws Exception { // Do it in another Thread because HDFS can deadlock if being interrupted while copying String threadName = "HDFS Copy from " + remotePath + " to " + localPath; final Tuple1<Exception> asyncException = Tuple1.of(null); Thread copyThread = new Thread(threadName) { @Override public void run() { try { Configuration hadoopConf = HadoopFileSystem.getHadoopConfiguration(); FileSystem fs = FileSystem.get(remotePath, hadoopConf); fs.copyToLocalFile(new Path(remotePath), new Path(localPath.getAbsolutePath())); } catch (Exception t) { asyncException.f0 = t; } } }; copyThread.setDaemon(true); copyThread.start(); copyThread.join(); if (asyncException.f0 != null) { throw asyncException.f0; } } }
final Tuple1<Exception> asyncException = Tuple1.of(null);
udf.flatMap(Tuple1.of(embedding), new ListCollector<>(result));