/** * Gets the local temporary file copy of a distributed cache files. * * @param name distributed cache file name * @return local temporary file copy of a distributed cache file. */ public File getCachedFile(String name) { return context.getDistributedCache().getFile(name); }
public AbstractRuntimeUDFContext(TaskInfo taskInfo, ClassLoader userCodeClassLoader, ExecutionConfig executionConfig, Map<String, Accumulator<?, ?>> accumulators, Map<String, Future<Path>> cpTasks, MetricGroup metrics) { this.taskInfo = checkNotNull(taskInfo); this.userCodeClassLoader = userCodeClassLoader; this.executionConfig = executionConfig; this.distributedCache = new DistributedCache(checkNotNull(cpTasks)); this.accumulators = checkNotNull(accumulators); this.metrics = metrics; }
public void writeUserArtifactEntriesToConfiguration() { for (Map.Entry<String, DistributedCache.DistributedCacheEntry> userArtifact : userArtifacts.entrySet()) { DistributedCache.writeFileInfoToConfig( userArtifact.getKey(), userArtifact.getValue(), jobConfiguration ); } } }
DistributedCache.readFileInfoFromConfig(jobConfiguration))
public void writeUserArtifactEntriesToConfiguration() { for (Map.Entry<String, DistributedCache.DistributedCacheEntry> userArtifact : userArtifacts.entrySet()) { DistributedCache.writeFileInfoToConfig( userArtifact.getKey(), userArtifact.getValue(), jobConfiguration ); } } }
DistributedCache.readFileInfoFromConfig(jobConfiguration)) { LOG.info("Obtaining local cache file for '{}'.", entry.getKey()); Future<Path> cp = fileCache.createTmpFile(entry.getKey(), entry.getValue(), jobId, executionId);
@Override public String map(Integer value) throws Exception { final Path testFile = getRuntimeContext().getDistributedCache().getFile("test_data").toPath(); final Path testDir = getRuntimeContext().getDistributedCache().getFile("test_dir").toPath(); if (testFile.toAbsolutePath().toString().equals(initialPath)) { throw new RuntimeException(String.format("Operator should access copy from cache rather than the " + "initial file. Input file path: %s. Cache file path: %s", initialPath, testFile)); } long testFileSize = Files.size(testFile); if (testFileSize != fileSize) { throw new RuntimeException(String.format("File size does not match. Expected:%s Actual:%s", fileSize, testFileSize)); } if (testDir.toAbsolutePath().toString().equals(initialDirPath)) { throw new RuntimeException(String.format("Operator should access copy from cache rather than the " + "initial dir. Input dir path: %s. Cache dir path: %s", initialDirPath, testDir)); } try (Stream<Path> files = Files.list(testDir)) { if (files.map(Path::getFileName).map(Path::toString).noneMatch(path -> path.equals(containedFileName))) { throw new RuntimeException(String.format("Cached directory %s should not be empty.", testDir)); } } return Files.readAllLines(testFile) .stream() .collect(Collectors.joining("\n")); } }
public void writeUserArtifactEntriesToConfiguration() { for (Map.Entry<String, DistributedCache.DistributedCacheEntry> userArtifact : userArtifacts.entrySet()) { DistributedCache.writeFileInfoToConfig( userArtifact.getKey(), userArtifact.getValue(), jobConfiguration ); } } }
DistributedCache.readFileInfoFromConfig(jobConfiguration)) { LOG.info("Obtaining local cache file for '{}'.", entry.getKey()); Future<Path> cp = fileCache.createTmpFile(entry.getKey(), entry.getValue(), jobId, executionId);
public AbstractRuntimeUDFContext(TaskInfo taskInfo, ClassLoader userCodeClassLoader, ExecutionConfig executionConfig, AbstractAccumulatorRegistry accumulatorRegistry, Map<String, Future<Path>> cpTasks, MetricGroup metrics) { this.taskInfo = checkNotNull(taskInfo); this.userCodeClassLoader = userCodeClassLoader; this.executionConfig = executionConfig; this.distributedCache = new DistributedCache(checkNotNull(cpTasks)); this.accumulatorRegistry = checkNotNull(accumulatorRegistry); this.metrics = metrics; }
@Override public void open(Configuration conf) throws IOException { File file = getRuntimeContext().getDistributedCache().getFile("cache_test"); try (BufferedReader reader = new BufferedReader(new FileReader(file))) { String tempString; while ((tempString = reader.readLine()) != null) { wordList.add(tempString); } } }
private JobGraph createJobGraph() { // make sure that all vertices start immediately jobGraph.setScheduleMode(ScheduleMode.EAGER); // Generate deterministic hashes for the nodes in order to identify them across // submission iff they didn't change. Map<Integer, byte[]> hashes = defaultStreamGraphHasher.traverseStreamGraphAndGenerateHashes(streamGraph); // Generate legacy version hashes for backwards compatibility List<Map<Integer, byte[]>> legacyHashes = new ArrayList<>(legacyStreamGraphHashers.size()); for (StreamGraphHasher hasher : legacyStreamGraphHashers) { legacyHashes.add(hasher.traverseStreamGraphAndGenerateHashes(streamGraph)); } Map<Integer, List<Tuple2<byte[], byte[]>>> chainedOperatorHashes = new HashMap<>(); setChaining(hashes, legacyHashes, chainedOperatorHashes); setPhysicalEdges(); setSlotSharing(); configureCheckpointing(); // add registered cache file into job configuration for (Tuple2<String, DistributedCache.DistributedCacheEntry> e : streamGraph.getEnvironment().getCachedFiles()) { DistributedCache.writeFileInfoToConfig(e.f0, e.f1, jobGraph.getJobConfiguration()); } // set the ExecutionConfig last when it has been finalized try { jobGraph.setExecutionConfig(streamGraph.getExecutionConfig()); } catch (IOException e) { throw new IllegalConfigurationException("Could not serialize the ExecutionConfig." + "This indicates that non-serializable types (like custom serializers) were registered"); } return jobGraph; }
DistributedCache.readFileInfoFromConfig(jobConfiguration)) { LOG.info("Obtaining local cache file for '{}'.", entry.getKey()); Future<Path> cp = fileCache.createTmpFile(entry.getKey(), entry.getValue(), jobId, executionId);
public AbstractRuntimeUDFContext(TaskInfo taskInfo, ClassLoader userCodeClassLoader, ExecutionConfig executionConfig, Map<String, Accumulator<?, ?>> accumulators, Map<String, Future<Path>> cpTasks, MetricGroup metrics) { this.taskInfo = checkNotNull(taskInfo); this.userCodeClassLoader = userCodeClassLoader; this.executionConfig = executionConfig; this.distributedCache = new DistributedCache(checkNotNull(cpTasks)); this.accumulators = checkNotNull(accumulators); this.metrics = metrics; }
receiver.open(outputFile); String path = function.getRuntimeContext().getDistributedCache().getFile(FLINK_PYTHON_DC_ID).getAbsolutePath();
DistributedCache.writeFileInfoToConfig(e.getKey(), e.getValue(), graph.getJobConfiguration());
@Override public void open(Configuration parameters) throws Exception{ File file = getRuntimeContext().getDistributedCache().getFile(testName); BufferedReader reader = new BufferedReader(new FileReader(file)); String output = reader.readLine(); reader.close(); assertEquals(output, testString); }
if (!jythonInitialized) { String path = context.getDistributedCache().getFile(PythonConstants.FLINK_PYTHON_DC_ID).getAbsolutePath();
/** * Gets the local temporary file copy of a distributed cache files. * * @param name distributed cache file name * @return local temporary file copy of a distributed cache file. */ @Override public File getCachedFile(String name) { return context.getDistributedCache().getFile(name); }