@Override public Path getPath() { if (inputSplit instanceof FileSplit) { return ((FileSplit) inputSplit).getPath(); } throw new RuntimeException(inputSplit + " is not a FileSplit"); }
@Override public Path getPath() { if (inputSplits != null && inputSplits.length > 0 && inputSplits[0] instanceof FileSplit) { return ((FileSplit) inputSplits[0]).getPath(); } return new Path(""); }
@Override public Path getPath() { if (inputSplit instanceof FileSplit) { return ((FileSplit) inputSplit).getPath(); } throw new RuntimeException(inputSplit + " is not a FileSplit"); }
private static Object determineFileId(FileSystem fs, FileSplit split, boolean allowSynthetic, boolean checkDefaultFs, boolean forceSynthetic) throws IOException { /* TODO: support this optionally? this is not OrcSplit, but we could add a custom split. Object fileKey = ((OrcSplit)split).getFileKey(); if (fileKey != null) return fileKey; */ LlapIoImpl.LOG.warn("Split for " + split.getPath() + " (" + split.getClass() + ") does not have file ID"); return HdfsUtils.getFileId(fs, split.getPath(), allowSynthetic, checkDefaultFs, forceSynthetic); }
public long get(FileSplit split) { Path path = split.getPath(); long start = split.getStart(); String key = path.toString()+"+"+String.format("%d",start); RCFileSyncEntry entry = cache.get(key); if(entry != null) { return entry.endSync; } return -1; } }
public static int parseSplitBucket(InputSplit split) { if (split instanceof FileSplit) { return getBucketIdFromFile(((FileSplit) split).getPath().getName()); } // cannot get this for combined splits return -1; }
private static Object determineFileId(FileSystem fs, FileSplit split, boolean allowSynthetic, boolean checkDefaultFs, boolean forceSynthetic) throws IOException { if (split instanceof OrcSplit) { Object fileKey = ((OrcSplit)split).getFileKey(); if (fileKey != null) { return fileKey; } } LOG.warn("Split for " + split.getPath() + " (" + split.getClass() + ") does not have file ID"); return HdfsUtils.getFileId(fs, split.getPath(), allowSynthetic, checkDefaultFs, forceSynthetic); }
private String oomErrorMessage(Reporter reporter) { return "Got OOM while mapping record #: " + recordCounter + " from input: " + ((FileSplit) reporter.getInputSplit()).getPath().getName(); }
public Optional<InternalHiveSplit> createInternalHiveSplit(FileSplit split) throws IOException { FileStatus file = fileSystem.getFileStatus(split.getPath()); return createInternalHiveSplit( split.getPath(), fileSystem.getFileBlockLocations(file, split.getStart(), split.getLength()), split.getStart(), split.getLength(), file.getLen(), OptionalInt.empty(), false); }
@Override public String getInputSplitSignature(InputSplit inputSplit) { FileSplit baseSplit = (FileSplit) inputSplit; return baseSplit.getPath().getName() + "_" + baseSplit.getStart() + "_" + baseSplit.getLength(); } }
public void put(FileSplit split, long endSync) { Path path = split.getPath(); long end = split.getStart() + split.getLength(); String key = path.toString()+"+"+String.format("%d",end); RCFileSyncEntry entry = new RCFileSyncEntry(); entry.end = end; entry.endSync = endSync; if(entry.endSync >= entry.end) { cache.put(key, entry); } }
@Override public String getInputSplitSignature(InputSplit inputSplit) { FileSplit baseSplit = (FileSplit) ((HCatSplit) inputSplit).getBaseSplit(); //file name(for intermediate table) + start pos + length return baseSplit.getPath().getName() + "_" + baseSplit.getStart() + "_" + baseSplit.getLength(); } }
@Override public int compare(InputSplit inp1, InputSplit inp2) { FileSplit fs1 = (FileSplit) inp1; FileSplit fs2 = (FileSplit) inp2; int retval = fs1.getPath().compareTo(fs2.getPath()); if (retval != 0) { return retval; } if (fs1.getStart() != fs2.getStart()) { return (int) (fs1.getStart() - fs2.getStart()); } return 0; } }
public OrcFileStripeMergeRecordReader(Configuration conf, FileSplit split) throws IOException { path = split.getPath(); start = split.getStart(); end = start + split.getLength(); FileSystem fs = path.getFileSystem(conf); this.reader = OrcFile.createReader(path, OrcFile.readerOptions(conf).filesystem(fs)); this.iter = reader.getStripes().iterator(); this.stripeIdx = 0; this.stripeStatistics = ((ReaderImpl) reader).getOrcProtoStripeStatistics(); }
@Override public int hashCode() { if (fs == null) return 0; final int prime = 31; int result = prime * 1 + fs.getPath().hashCode(); result = prime * result + Long.valueOf(fs.getStart()).hashCode(); return prime * result + Long.valueOf(fs.getLength()).hashCode(); }
public static void getPartitionValues(VectorizedRowBatchCtx vrbCtx, MapWork mapWork, FileSplit split, Object[] partitionValues) throws IOException { Map<Path, PartitionDesc> pathToPartitionInfo = mapWork.getPathToPartitionInfo(); PartitionDesc partDesc = HiveFileFormatUtils .getFromPathRecursively(pathToPartitionInfo, split.getPath(), IOPrepareCache.get().getPartitionDescMap()); getPartitionValues(vrbCtx, partDesc, partitionValues); }
public ConnectorPageSource newPageSource(FileFormatDataSourceStats stats, ConnectorSession session) { OrcPageSourceFactory orcPageSourceFactory = new OrcPageSourceFactory(TYPE_MANAGER, false, HDFS_ENVIRONMENT, stats); return HivePageSourceProvider.createHivePageSource( ImmutableSet.of(), ImmutableSet.of(orcPageSourceFactory), new Configuration(), session, fileSplit.getPath(), OptionalInt.empty(), fileSplit.getStart(), fileSplit.getLength(), fileSplit.getLength(), schema, TupleDomain.all(), columns, partitionKeys, DateTimeZone.UTC, TYPE_MANAGER, ImmutableMap.of(), Optional.empty(), false) .get(); }
private static String toString(FileSplit fs) { return "{" + fs.getPath() + ", " + fs.getStart() + ", " + fs.getLength() + "}"; }
public RCFileBlockMergeRecordReader(Configuration conf, FileSplit split) throws IOException { path = split.getPath(); FileSystem fs = path.getFileSystem(conf); this.in = new RCFile.Reader(fs, path, conf); this.end = split.getStart() + split.getLength(); this.conf = conf; if (split.getStart() > in.getPosition()) { in.sync(split.getStart()); // sync to start } this.start = in.getPosition(); more = start < end; }
@Override public boolean equals(Object obj) { if (this == obj) return true; if (!(obj instanceof FsWithHash)) return false; FsWithHash other = (FsWithHash)obj; if ((fs == null) != (other.fs == null)) return false; if (fs == null && other.fs == null) return true; return fs.getStart() == other.fs.getStart() && fs.getLength() == other.fs.getLength() && fs.getPath().equals(other.fs.getPath()); } }