/** * Create the bucket filename in Acid format * @param subdir the subdirectory for the bucket. * @param bucket the bucket number * @return the filename */ public static Path createBucketFile(Path subdir, int bucket) { return createBucketFile(subdir, bucket, true); }
/** * This determines the set of {@link ReaderPairAcid} to create for a given delta/. * For unbucketed tables {@code bucket} can be thought of as a write tranche. */ static Path[] getDeltaFiles(Path deltaDirectory, int bucket, Options mergerOptions) { assert (!mergerOptions.isCompacting && deltaDirectory.getName().startsWith(AcidUtils.DELETE_DELTA_PREFIX) ) || mergerOptions.isCompacting : "Unexpected delta: " + deltaDirectory + "(isCompacting=" + mergerOptions.isCompacting() + ")"; return new Path[] {AcidUtils.createBucketFile(deltaDirectory, bucket)}; }
@Override public RawReader<Text> getRawReader(Configuration conf, boolean collapseEvents, int bucket, ValidWriteIdList validWriteIdList, Path baseDirectory, Path... deltaDirectory) throws IOException { List<Path> filesToRead = new ArrayList<Path>(); if (baseDirectory != null) { if (baseDirectory.getName().startsWith(AcidUtils.BASE_PREFIX)) { Path p = AcidUtils.createBucketFile(baseDirectory, bucket); FileSystem fs = p.getFileSystem(conf); if (fs.exists(p)) filesToRead.add(p); } else { filesToRead.add(new Path(baseDirectory, "000000_0")); } } for (int i = 0; i < deltaDirectory.length; i++) { Path p = AcidUtils.createBucketFile(deltaDirectory[i], bucket); FileSystem fs = p.getFileSystem(conf); if (fs.exists(p)) filesToRead.add(p); } return new MockRawReader(conf, filesToRead); }
@Override public RawReader<OrcStruct> getRawReader(Configuration conf, boolean collapseEvents, int bucket, ValidTxnList validTxnList, Path baseDirectory, Path[] deltaDirectory ) throws IOException { Reader reader = null; boolean isOriginal = false; if (baseDirectory != null) { Path bucketFile; if (baseDirectory.getName().startsWith(AcidUtils.BASE_PREFIX)) { bucketFile = AcidUtils.createBucketFile(baseDirectory, bucket); } else { isOriginal = true; bucketFile = findOriginalBucket(baseDirectory.getFileSystem(conf), baseDirectory, bucket); } reader = OrcFile.createReader(bucketFile, OrcFile.readerOptions(conf)); } return new OrcRawRecordMerger(conf, collapseEvents, reader, isOriginal, bucket, validTxnList, new Reader.Options(), deltaDirectory); }
Path dir = new Path(location, filename); fs.mkdirs(dir); partFile = AcidUtils.createBucketFile(dir, bucket); if (type == FileType.LENGTH_FILE) { partFile = new Path(partFile.toString() + AcidUtils.DELTA_SIDE_FILE_SUFFIX);
int totalDeleteEventCount = 0; for (Path deleteDeltaDir : deleteDeltaDirs) { Path deleteDeltaFile = AcidUtils.createBucketFile(deleteDeltaDir, bucket); FileSystem fs = deleteDeltaFile.getFileSystem(conf);
return createBucketFile(new Path(directory, subdir), options.getBucketId());
options.getStatementId()); return createBucketFile(new Path(directory, subdir), options.getBucket());
for(Path delta: deltaDirectory) { ReaderKey key = new ReaderKey(); Path deltaFile = AcidUtils.createBucketFile(delta, bucket); AcidUtils.ParsedDelta deltaDir = AcidUtils.parsedDelta(delta); FileSystem fs = deltaFile.getFileSystem(conf);
Path delta1_1_0 = new Path(root, AcidUtils.deltaSubdir( options.getMinimumWriteId(), options.getMaximumWriteId(), options.getStatementId())); Path bucket0 = AcidUtils.createBucketFile(delta1_1_0, BUCKET); Path bucket0SideFile = OrcAcidUtils.getSideFile(bucket0);
wo.inspector(inspector); wo.callback(new OrcRecordUpdater.KeyIndexBuilder("testEmpty")); Writer w = OrcFile.createWriter(AcidUtils.createBucketFile(new Path(root, AcidUtils.baseDir(100)), BUCKET), wo); w.close(); AcidUtils.Directory directory = AcidUtils.getAcidState(root, conf, writeIdList); Path basePath = AcidUtils.createBucketFile(directory.getBaseDirectory(), BUCKET); Reader baseReader = OrcFile.createReader(basePath,
Path bucketPath = AcidUtils.createBucketFile(mergerOptions.getBaseDir(), bucket); if(fs.exists(bucketPath) && fs.getFileStatus(bucketPath).getLen() > 0) {
directory.getCurrentDirectories().get(1).getPath()); Path basePath = AcidUtils.createBucketFile(directory.getBaseDirectory(), BUCKET); Path deltaPath = AcidUtils.createBucketFile(directory.getCurrentDirectories().get(1).getPath(), BUCKET); Path deleteDeltaDir = directory.getCurrentDirectories().get(0).getPath();
@Override public RawReader<OrcStruct> getRawReader(Configuration conf, boolean collapseEvents, int bucket, ValidTxnList validTxnList, Path baseDirectory, Path[] deltaDirectory ) throws IOException { Reader reader = null; boolean isOriginal = false; if (baseDirectory != null) { Path bucketFile; if (baseDirectory.getName().startsWith(AcidUtils.BASE_PREFIX)) { bucketFile = AcidUtils.createBucketFile(baseDirectory, bucket); } else { isOriginal = true; bucketFile = findOriginalBucket(baseDirectory.getFileSystem(conf), baseDirectory, bucket); } reader = OrcFile.createReader(bucketFile, OrcFile.readerOptions(conf)); } return new OrcRawRecordMerger(conf, collapseEvents, reader, isOriginal, bucket, validTxnList, new Reader.Options(), deltaDirectory); }
/** * Create a filename for a bucket file. * @param directory the partition directory * @param options the options for writing the bucket * @return the filename that should store the bucket */ public static Path createFilename(Path directory, AcidOutputFormat.Options options) { String subdir; if (options.getOldStyle()) { return new Path(directory, String.format(BUCKET_DIGITS, options.getBucket()) + "_0"); } else if (options.isWritingBase()) { subdir = BASE_PREFIX + String.format(DELTA_DIGITS, options.getMaximumTransactionId()); } else { subdir = deltaSubdir(options.getMinimumTransactionId(), options.getMaximumTransactionId()); } return createBucketFile(new Path(directory, subdir), options.getBucket()); }
final Path deltaFile = AcidUtils.createBucketFile(delta, bucket); final FileSystem fs = deltaFile.getFileSystem(conf); final FileStatus fileStatus = fs.getFileStatus(deltaFile);
for(Path delta: deltaDirectory) { ReaderKey key = new ReaderKey(); Path deltaFile = AcidUtils.createBucketFile(delta, bucket); FileSystem fs = deltaFile.getFileSystem(conf); long length = getLastFlushLength(fs, deltaFile);