Refine search
public Optional<InternalHiveSplit> createInternalHiveSplit(FileSplit split) throws IOException { FileStatus file = fileSystem.getFileStatus(split.getPath()); return createInternalHiveSplit( split.getPath(), fileSystem.getFileBlockLocations(file, split.getStart(), split.getLength()), split.getStart(), split.getLength(), file.getLen(), OptionalInt.empty(), false); }
public void put(FileSplit split, long endSync) { Path path = split.getPath(); long end = split.getStart() + split.getLength(); String key = path.toString()+"+"+String.format("%d",end); RCFileSyncEntry entry = new RCFileSyncEntry(); entry.end = end; entry.endSync = endSync; if(entry.endSync >= entry.end) { cache.put(key, entry); } }
public void put(FileSplit split, long endSync) { Path path = split.getPath(); long end = split.getStart() + split.getLength(); String key = path.toString()+"+"+String.format("%d",end); RCFileSyncEntry entry = new RCFileSyncEntry(); entry.end = end; entry.endSync = endSync; if(entry.endSync >= entry.end) { cache.put(key, entry); } }
public RCFileBlockMergeRecordReader(Configuration conf, FileSplit split) throws IOException { path = split.getPath(); FileSystem fs = path.getFileSystem(conf); this.in = new RCFile.Reader(fs, path, conf); this.end = split.getStart() + split.getLength(); this.conf = conf; if (split.getStart() > in.getPosition()) { in.sync(split.getStart()); // sync to start } this.start = in.getPosition(); more = start < end; }
@Override public String getInputSplitSignature(InputSplit inputSplit) { FileSplit baseSplit = (FileSplit) ((HCatSplit) inputSplit).getBaseSplit(); //file name(for intermediate table) + start pos + length return baseSplit.getPath().getName() + "_" + baseSplit.getStart() + "_" + baseSplit.getLength(); } }
/** * Returns a string representation of a Hadoop {@link FileSplit}. * * @param fs Hadoop {@link FileSplit} * @return its string representation */ public static String toStringHadoopFileSplit(FileSplit fs) { StringBuilder sb = new StringBuilder(); sb.append("HadoopFileSplit: Path: ").append(fs.getPath()); sb.append(" , Start: ").append(fs.getStart()); sb.append(" , Length: ").append(fs.getLength()); sb.append(" , Hosts: "); String[] locs; try { locs = fs.getLocations(); } catch (IOException e) { LOG.error(e.getMessage()); locs = new String[] {}; } for (String loc : locs) { sb.append(loc).append("; "); } return sb.toString(); }
@Override public String[] getLocations(InputSplit split) throws IOException { if (!(split instanceof FileSplit)) { if (LOG.isDebugEnabled()) { LOG.debug("Split: " + split + " is not a FileSplit. Using default locations"); } return split.getLocations(); } FileSplit fsplit = (FileSplit) split; String splitDesc = "Split at " + fsplit.getPath() + " with offset= " + fsplit.getStart() + ", length=" + fsplit.getLength(); String location = locations.get(determineLocation( locations, fsplit.getPath().toString(), fsplit.getStart(), splitDesc)); return (location != null) ? new String[] { location } : null; }
public RCFileBlockMergeRecordReader(Configuration conf, FileSplit split) throws IOException { path = split.getPath(); FileSystem fs = path.getFileSystem(conf); this.in = new RCFile.Reader(fs, path, conf); this.end = split.getStart() + split.getLength(); this.conf = conf; if (split.getStart() > in.getPosition()) { in.sync(split.getStart()); // sync to start } this.start = in.getPosition(); more = start < end; }
@Override public String[] getLocations(InputSplit split) throws IOException { if (!(split instanceof FileSplit)) { if (isDebugEnabled) { LOG.debug("Split: " + split + " is not a FileSplit. Using default locations"); } return split.getLocations(); } FileSplit fsplit = (FileSplit) split; String splitDesc = "Split at " + fsplit.getPath() + " with offset= " + fsplit.getStart() + ", length=" + fsplit.getLength(); String location = locations.get(determineLocation( locations, fsplit.getPath().toString(), fsplit.getStart(), splitDesc)); return (location != null) ? new String[] { location } : null; }
OrcRecordReader(Reader file, Configuration conf, FileSplit split) throws IOException { List<OrcProto.Type> types = file.getTypes(); this.file = file; numColumns = (types.size() == 0) ? 0 : types.get(0).getSubtypesCount(); this.offset = split.getStart(); this.length = split.getLength(); this.reader = createReaderFromFile(file, conf, offset, length); this.stats = new SerDeStats(); }
public TetherRecordReader(JobConf job, FileSplit split) throws IOException { this.in = new FsInput(split.getPath(), job); this.reader = new DataFileReader<>(in, new GenericDatumReader<>()); reader.sync(split.getStart()); // sync to start this.start = in.tell(); this.end = split.getStart() + split.getLength(); job.set(AvroJob.INPUT_SCHEMA, reader.getSchema().toString()); }
@Override public boolean contains(FileSplit split) throws HiveException { if (buckets == null) { return false; } String bucketName = split.getPath().toString(); IBucket bucket = buckets.get(bucketName); if (bucket == null) { bucketName = split.getPath().toUri().getPath(); bucket = buckets.get(bucketName); if (bucket == null) { return false; } } for (Long offset : bucket.getOffsets()) { if ((offset >= split.getStart()) && (offset <= split.getStart() + split.getLength())) { return true; } } return false; } }
OrcRecordReader(Reader file, Configuration conf, FileSplit split) throws IOException { List<OrcProto.Type> types = file.getTypes(); this.file = file; numColumns = (types.size() == 0) ? 0 : types.get(0).getSubtypesCount(); this.offset = split.getStart(); this.length = split.getLength(); this.reader = createReaderFromFile(file, conf, offset, length); this.stats = new SerDeStats(); }
long offset = split.getStart(), maxOffset = offset + split.getLength(); stripeIxFrom = -1; int stripeIxTo = -1; if (LlapIoImpl.ORC_LOGGER.isDebugEnabled()) { String tmp = "FileSplit {" + split.getStart() + ", " + split.getLength() + "}; stripes "; for (StripeInformation stripe : stripes) { tmp += "{" + stripe.getOffset() + ", " + stripe.getLength() + "}, ";
/** * @param clsName Input split class name. * @param in Input stream. * @param hosts Optional hosts. * @return File block or {@code null} if it is not a {@link FileSplit} instance. * @throws IgniteCheckedException If failed. */ @Nullable public static HadoopFileBlock readFileBlock(String clsName, FSDataInputStream in, @Nullable String[] hosts) throws IgniteCheckedException { if (!FileSplit.class.getName().equals(clsName)) return null; FileSplit split = U.newInstance(FileSplit.class); try { split.readFields(in); } catch (IOException e) { throw new IgniteCheckedException(e); } if (hosts == null) hosts = EMPTY_HOSTS; return new HadoopFileBlock(hosts, split.getPath().toUri(), split.getStart(), split.getLength()); } }
public RCFileRecordReader(Configuration conf, FileSplit split) throws IOException { Path path = split.getPath(); FileSystem fs = path.getFileSystem(conf); this.in = new RCFile.Reader(fs, path, conf); this.end = split.getStart() + split.getLength(); this.conf = conf; this.split = split; useCache = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEUSERCFILESYNCCACHE); if (split.getStart() > in.getPosition()) { long oldSync = useCache ? syncCache.get(split) : -1; if(oldSync == -1) { in.sync(split.getStart()); // sync to start } else { in.seek(oldSync); } } this.start = in.getPosition(); more = start < end; }
public OrcFileStripeMergeRecordReader(Configuration conf, FileSplit split) throws IOException { path = split.getPath(); start = split.getStart(); end = start + split.getLength(); FileSystem fs = path.getFileSystem(conf); this.reader = OrcFile.createReader(path, OrcFile.readerOptions(conf).filesystem(fs)); this.iter = reader.getStripes().iterator(); this.stripeIdx = 0; this.stripeStatistics = ((ReaderImpl) reader).getOrcProtoStripeStatistics(); }