/** Like {@link #match(String)}, but with a configurable {@link EmptyMatchTreatment}. */ public static MatchResult match(String spec, EmptyMatchTreatment emptyMatchTreatment) throws IOException { MatchResult res = match(spec); return maybeAdjustEmptyMatchResult(spec, res, emptyMatchTreatment); }
/** * Like {@link #match(List)}, but for a single resource specification. * * <p>The function {@link #match(List)} is preferred when matching multiple patterns, as it allows * for bulk API calls to remote filesystems. */ public static MatchResult match(String spec) throws IOException { List<MatchResult> matches = match(Collections.singletonList(spec)); verify( matches.size() == 1, "FileSystem implementation for %s did not return exactly one MatchResult: %s", spec, matches); return matches.get(0); }
Collection<Metadata> files = FileSystems.match(filePattern).metadata(); LOG.debug( "Found file(s) {} by matching the path: {}",
@ProcessElement public void process(ProcessContext c) throws Exception { String filepattern = c.element(); MatchResult match = FileSystems.match(filepattern, emptyMatchTreatment); LOG.info("Matched {} files for pattern {}", match.metadata().size(), filepattern); for (MatchResult.Metadata metadata : match.metadata()) { c.output(metadata); } } }
/** * Returns the {@link Metadata} for a single file resource. Expects a resource specification * {@code spec} that matches a single result. * * @param spec a resource specification that matches exactly one result. * @return the {@link Metadata} for the specified resource. * @throws FileNotFoundException if the file resource is not found. * @throws IOException in the event of an error in the inner call to {@link #match}, or if the * given spec does not match exactly 1 result. */ public static Metadata matchSingleFileSpec(String spec) throws IOException { List<MatchResult> matches = FileSystems.match(Collections.singletonList(spec)); MatchResult matchResult = Iterables.getOnlyElement(matches); if (matchResult.status() == Status.NOT_FOUND) { throw new FileNotFoundException(String.format("File spec %s not found", spec)); } else if (matchResult.status() != Status.OK) { throw new IOException( String.format("Error matching file spec %s: status %s", spec, matchResult.status())); } else { List<Metadata> metadata = matchResult.metadata(); if (metadata.size() != 1) { throw new IOException( String.format( "Expecting spec %s to match exactly one file, but matched %s: %s", spec, metadata.size(), metadata)); } return metadata.get(0); } }
/** * Returns {@link MatchResult MatchResults} for the given {@link ResourceId resourceIds}. * * @param resourceIds {@link ResourceId resourceIds} that might be derived from {@link #match}, * {@link ResourceId#resolve}, or {@link ResourceId#getCurrentDirectory()}. * @throws IOException if all {@code resourceIds} failed to match due to issues like: network * connection, authorization. Exception for individual {@link ResourceId} need to be deferred * until callers retrieve metadata with {@link MatchResult#metadata()}. */ public static List<MatchResult> matchResources(List<ResourceId> resourceIds) throws IOException { return match(FluentIterable.from(resourceIds).transform(ResourceId::toString).toList()); }
MatchResult result = FileSystems.match(path); checkArgument( result.status() == Status.OK && !result.metadata().isEmpty(),
Iterables.getOnlyElement(FileSystems.match(Collections.singletonList(filePattern))) .metadata();
@Override public final BoundedReader<T> createReader(PipelineOptions options) throws IOException { // Validate the current source prior to creating a reader for it. this.validate(); String fileOrPattern = fileOrPatternSpec.get(); if (mode == Mode.FILEPATTERN) { long startTime = System.currentTimeMillis(); List<Metadata> fileMetadata = FileSystems.match(fileOrPattern, emptyMatchTreatment).metadata(); LOG.info("Matched {} files for pattern {}", fileMetadata.size(), fileOrPattern); List<FileBasedReader<T>> fileReaders = new ArrayList<>(); for (Metadata metadata : fileMetadata) { long endOffset = metadata.sizeBytes(); fileReaders.add( createForSubrangeOfFile(metadata, 0, endOffset).createSingleFileReader(options)); } LOG.debug( "Creating a reader for file pattern {} took {} ms", fileOrPattern, System.currentTimeMillis() - startTime); if (fileReaders.size() == 1) { return fileReaders.get(0); } return new FilePatternReader(this, fileReaders); } else { return createSingleFileReader(options); } }
@Override public Watch.Growth.PollResult<MatchResult.Metadata> apply(String element, Context c) throws Exception { Instant now = Instant.now(); return Watch.Growth.PollResult.incomplete( now, FileSystems.match(element, EmptyMatchTreatment.ALLOW).metadata()) .withWatermark(now); } }
@Override public final long getEstimatedSizeBytes(PipelineOptions options) throws IOException { // This implementation of method getEstimatedSizeBytes is provided to simplify subclasses. Here // we perform the size estimation of files and file patterns using the interface provided by // FileSystem. String fileOrPattern = fileOrPatternSpec.get(); if (mode == Mode.FILEPATTERN) { long totalSize = 0; List<Metadata> allMatches = FileSystems.match(fileOrPattern, emptyMatchTreatment).metadata(); for (Metadata metadata : allMatches) { totalSize += metadata.sizeBytes(); } LOG.info( "Filepattern {} matched {} files with total size {}", fileOrPattern, allMatches.size(), totalSize); return totalSize; } else { long start = getStartOffset(); long end = Math.min(getEndOffset(), getMaxEndOffset(options)); return end - start; } }
MatchResult singleMatch = Iterables.getOnlyElement( FileSystems.match(Collections.singletonList(tempDir.toString() + "*"))); for (Metadata matchResult : singleMatch.metadata()) { if (allMatches.add(matchResult.resourceId())) {
long startTime = System.currentTimeMillis(); List<Metadata> expandedFiles = FileSystems.match(fileOrPattern, emptyMatchTreatment).metadata(); List<FileBasedSource<T>> splitResults = new ArrayList<>(expandedFiles.size()); for (Metadata metadata : expandedFiles) {
@RequiresStableInput @ProcessElement public void processElement(ProcessContext c) throws Exception { MatchResult matchResult = FileSystems.match(outputPrefix + "*"); boolean firstTime = matchResult.metadata().isEmpty(); KV<String, String> kv = c.element(); writeTextToFileSideEffect(kv.getValue(), outputPrefix + kv.getKey()); if (firstTime) { throw new Exception( "Deliberate failure: should happen only once for each application of the DoFn" + "within the transform graph."); } }
@Override public PCollection<byte[]> expand(PBegin input) { if (getFilepattern() == null) { throw new IllegalStateException( "Need to set the filepattern of a TFRecordIO.Read transform"); } if (getValidate()) { checkState(getFilepattern().isAccessible(), "Cannot validate with a RVP."); try { MatchResult matches = FileSystems.match(getFilepattern().get()); checkState( !matches.metadata().isEmpty(), "Unable to find any files matching %s", getFilepattern().get()); } catch (IOException e) { throw new IllegalStateException( String.format("Failed to validate %s", getFilepattern().get()), e); } } return input.apply("Read", org.apache.beam.sdk.io.Read.from(getSource())); }
p.run(); List<Metadata> matches = FileSystems.match(targetLocationGlob).metadata(); List<String> actuals = new ArrayList<>(strs.size()); List<String> files = new ArrayList<>(strs.size());
@ProcessElement public void processElement(ProcessContext c) { try { KV<String, String> kv = c.element(); String filePath = GcsUtil.joinPath(importDirectory.get(), kv.getValue()); MatchResult match = FileSystems.match(filePath, EmptyMatchTreatment.DISALLOW); ResourceId resourceId = match.metadata().get(0).resourceId(); TableManifest.Builder builder = TableManifest.newBuilder(); try (InputStream stream = Channels.newInputStream(FileSystems.open(resourceId))) { Reader reader = new InputStreamReader(stream); JsonFormat.parser().merge(reader, builder); } c.output(KV.of(kv.getKey(), builder.build())); } catch (IOException e) { throw new RuntimeException(e); } } }));
final String pattern = baseName + "*"; List<Metadata> metadata = FileSystems.match(Collections.singletonList(pattern)).get(0).metadata(); for (Metadata meta : metadata) { outputFiles.add(new File(meta.resourceId().toString()));
if (numShards == 0) { String pattern = outputPrefix.toString() + "*"; List<MatchResult> matches = FileSystems.match(Collections.singletonList(pattern)); for (Metadata expectedFile : Iterables.getOnlyElement(matches).metadata()) { expectedFiles.add(new File(expectedFile.resourceId().toString()));