public String filenamePrefixForWindow(IntervalWindow window) { String filePrefix = prefix.isDirectory() ? "" : prefix.getFilename(); return String.format( "%s-%s-%s", filePrefix, formatter.print(window.start()), formatter.print(window.end())); }
private static String extractFilename(ResourceId input) { if (input.isDirectory()) { return ""; } else { return firstNonNull(input.getFilename(), ""); } } }
/** * @param window interval window * @return file name prefix. */ String filenamePrefixForWindow(final IntervalWindow window) { final String prefix = baseFilename.isDirectory() ? "" : firstNonNull(baseFilename.getFilename(), ""); return String.format( "%s-%s-%s", prefix, FORMATTER.print(window.start()), FORMATTER.print(window.end())); } @Override
public String filenamePrefixForWindow(IntervalWindow window) { String prefix = baseFilename.isDirectory() ? "" : firstNonNull(baseFilename.getFilename(), ""); return String.format( "%s-%s-%s", prefix, FORMATTER.print(window.start()), FORMATTER.print(window.end())); }
@Override public void encode(ResourceId value, OutputStream os) throws IOException { STRING_CODER.encode(value.toString(), os); BOOL_CODER.encode(value.isDirectory(), os); }
/** * Enforces that the {@link ResourceId} implementation of {@code baseDirectory} meets the {@link * ResourceId} spec. */ public static void runResourceIdBattery(ResourceId baseDirectory) { checkArgument( baseDirectory.isDirectory(), "baseDirectory %s is not a directory", baseDirectory); List<ResourceId> allResourceIds = new ArrayList<>(); allResourceIds.add(baseDirectory); // Validate that individual resources meet the fairly restrictive spec we have. validateResourceIds(allResourceIds); // Validate operations with resolving child resources. validateResolvingIds(baseDirectory, allResourceIds); // Validate safeguards against resolving bad paths. validateFailureResolvingIds(baseDirectory); }
private static void validateResourceIds(List<ResourceId> resourceIds) { for (ResourceId resourceId : resourceIds) { // ResourceIds should equal themselves. assertThat("ResourceId equal to itself", resourceId, equalTo(resourceId)); // ResourceIds should be clonable via FileSystems#matchNewResource. ResourceId cloned; if (resourceId.isDirectory()) { cloned = FileSystems.matchNewResource(resourceId.toString(), true /* isDirectory */); } else { cloned = FileSystems.matchNewResource(resourceId.toString(), false /* isDirectory */); } assertThat("ResourceId equals clone of itself", cloned, equalTo(resourceId)); // .. and clones have consistent toString. assertThat( "ResourceId toString consistency", cloned.toString(), equalTo(resourceId.toString())); // .. and have consistent isDirectory. assertThat( "ResourceId isDirectory consistency", cloned.isDirectory(), equalTo(resourceId.isDirectory())); } }
public String filenamePrefixForWindow(IntervalWindow window) { String prefix = baseFilename.isDirectory() ? "" : firstNonNull(baseFilename.getFilename(), ""); return String.format( "%s%s-%s", prefix, FORMATTER.print(window.start()), FORMATTER.print(window.end())); }
@Override public ResourceId unwindowedFilename( int shardNumber, int numShards, OutputFileHints outputFileHints) { DecimalFormat df = new DecimalFormat("0000"); String prefix = baseFilename.isDirectory() ? "" : firstNonNull(baseFilename.getFilename(), ""); String filename = String.format( "%s-%s-of-%s%s%s", prefix, df.format(shardNumber), df.format(numShards), outputFileHints.getSuggestedFilenameSuffix(), suffix); return baseFilename .getCurrentDirectory() .resolve(filename, StandardResolveOptions.RESOLVE_FILE); } }
allResourceIds.add(file1); allResourceIds.add(file2); assertThat("Resolved file isDirectory()", file1.isDirectory(), is(false)); assertThat("Resolved file isDirectory()", file2.isDirectory(), is(false)); assertThat("Resolved file isDirectory()", file2a.isDirectory(), is(false)); assertThat("Resolved directory isDirectory()", dir1.isDirectory(), is(true)); assertThat("Resolved directory isDirectory()", dir2.isDirectory(), is(true)); assertThat("Resolved directory isDirectory()", dir2a.isDirectory(), is(true)); allResourceIds.add(dir1); allResourceIds.add(dir2);
@ProcessElement public void process(ProcessContext c) { MatchResult.Metadata metadata = c.element(); if (metadata.resourceId().isDirectory()) { switch (spec.getDirectoryTreatment()) { case SKIP: return; case PROHIBIT: throw new IllegalArgumentException( "Trying to read " + metadata.resourceId() + " which is a directory"); default: throw new UnsupportedOperationException( "Unknown DirectoryTreatment: " + spec.getDirectoryTreatment()); } } Compression compression = (spec.getCompression() == Compression.AUTO) ? Compression.detect(metadata.resourceId().getFilename()) : spec.getCompression(); c.output( new ReadableFile( MatchResult.Metadata.builder() .setResourceId(metadata.resourceId()) .setSizeBytes(metadata.sizeBytes()) .setIsReadSeekEfficient( metadata.isReadSeekEfficient() && compression == Compression.UNCOMPRESSED) .build(), compression)); } }
@Override public ResourceId windowedFilename( int shardNumber, int numShards, BoundedWindow window, PaneInfo paneInfo, OutputFileHints outputFileHints) { String filenamePrefix = outputFilePrefix.isDirectory() ? "" : firstNonNull(outputFilePrefix.getFilename(), ""); IntervalWindow interval = (IntervalWindow) window; String windowStr = String.format("%s-%s", interval.start().toString(), interval.end().toString()); String filename = String.format( "%s-%s-%s-of-%s-pane-%s%s%s.avro", filenamePrefix, windowStr, shardNumber, numShards, paneInfo.getIndex(), paneInfo.isLast() ? "-last" : "", outputFileHints.getSuggestedFilenameSuffix()); return outputFilePrefix.getCurrentDirectory().resolve(filename, RESOLVE_FILE); }