boolean sawBase = dir.getName().startsWith(AcidUtils.BASE_PREFIX); boolean isRawFormat = !dir.getName().startsWith(AcidUtils.DELETE_DELTA_PREFIX) && AcidUtils.MetaDataFile.isRawFormat(dir, fs);//deltes can't be raw format
private static void validateAcidFiles(FileStatus[] srcs, FileSystem fs) throws SemanticException { try { for (FileStatus oneSrc : srcs) { if (!AcidUtils.MetaDataFile.isRawFormatFile(oneSrc.getPath(), fs)) { throw new SemanticException(ErrorMsg.LOAD_DATA_ACID_FILE, oneSrc.getPath().toString()); } } } catch (IOException ex) { throw new SemanticException(ex); } }
final boolean isBaseInRawFormat = base != null && MetaDataFile.isRawFormat(base, fs); return new DirectoryImpl(abortedDirectories, isBaseInRawFormat, original, obsolete, deltas, base);
isOriginal = AcidUtils.MetaDataFile.isRawFormat(baseDirectory, baseDirectory.getFileSystem(conf)); mergerOptions.rootPath(baseDirectory.getParent()); } else {
/** * Checks if the files in base/delta dir are a result of Load Data statement and thus do not * have ROW_IDs embedded in the data. * @param baseOrDeltaDir base or delta file. */ public static boolean isRawFormat(Path baseOrDeltaDir, FileSystem fs) throws IOException { Path dataFile = chooseFile(baseOrDeltaDir, fs); if (dataFile == null) { //directory is empty or doesn't have any that could have been produced by load data return false; } return isRawFormatFile(dataFile, fs); } public static boolean isRawFormatFile(Path dataFile, FileSystem fs) throws IOException {
public static ParsedDelta parsedDelta(Path deltaDir, String deltaPrefix, FileSystem fs) throws IOException { String filename = deltaDir.getName(); boolean isDeleteDelta = deltaPrefix.equals(DELETE_DELTA_PREFIX); if (filename.startsWith(deltaPrefix)) { //small optimization - delete delta can't be in raw format boolean isRawFormat = !isDeleteDelta && MetaDataFile.isRawFormat(deltaDir, fs); return parsedDelta(deltaDir, isRawFormat); } throw new IllegalArgumentException(deltaDir + " does not start with " + deltaPrefix); }
/** * Returns {@code true} if {@code parsedBase} was created by compaction. * As of Hive 4.0 we can tell if a directory is a result of compaction based on the * presence of {@link AcidUtils#VISIBILITY_PATTERN} suffix. Base directories written prior to * that, have to rely on the {@link MetaDataFile} in the directory. So look at the filename first * since that is the cheaper test.*/ private static boolean isCompactedBase(ParsedBase parsedBase, FileSystem fs) throws IOException { return parsedBase.getVisibilityTxnId() > 0 || MetaDataFile.isCompacted(parsedBase.getBaseDirPath(), fs); } private static void getChildState(FileStatus child, HdfsFileStatusWithId childWithId,