@Override public long getLen() { return myFs.getLen(); }
private Optional<InternalHiveSplit> createInternalHiveSplit(LocatedFileStatus status, OptionalInt bucketNumber, boolean splittable) { splittable = splittable && isSplittable(inputFormat, fileSystem, status.getPath()); return createInternalHiveSplit( status.getPath(), status.getBlockLocations(), 0, status.getLen(), status.getLen(), bucketNumber, splittable); }
private long getPathLength(JobConf conf, Path path, Class<? extends InputFormat> clazz, long threshold) throws IOException { if (ContentSummaryInputFormat.class.isAssignableFrom(clazz)) { InputFormat input = HiveInputFormat.getInputFormatFromCache(clazz, conf); return ((ContentSummaryInputFormat)input).getContentSummary(path, conf).getLength(); } else { FileSystem fs = path.getFileSystem(conf); try { long length = 0; RemoteIterator<LocatedFileStatus> results = fs.listFiles(path, true); // No need to iterate more, when threshold is reached // (beneficial especially for object stores) while (length <= threshold && results.hasNext()) { length += results.next().getLen(); } LOG.trace("length=" + length + ", threshold=" + threshold); return length; } catch (FileNotFoundException e) { return 0; } } } }
private long getPathLength(JobConf conf, Path path, Class<? extends InputFormat> clazz, long threshold) throws IOException { if (ContentSummaryInputFormat.class.isAssignableFrom(clazz)) { InputFormat input = HiveInputFormat.getInputFormatFromCache(clazz, conf); return ((ContentSummaryInputFormat)input).getContentSummary(path, conf).getLength(); } else { FileSystem fs = path.getFileSystem(conf); try { long length = 0; RemoteIterator<LocatedFileStatus> results = fs.listFiles(path, true); // No need to iterate more, when threshold is reached // (beneficial especially for object stores) while (length <= threshold && results.hasNext()) { length += results.next().getLen(); } LOG.trace("length=" + length + ", threshold=" + threshold); return length; } catch (FileNotFoundException e) { return 0; } } } }
private static SortedSet<byte[]> readFileToSearch(final Configuration conf, final FileSystem fs, final LocatedFileStatus keyFileStatus) throws IOException, InterruptedException { SortedSet<byte []> result = new TreeSet<>(Bytes.BYTES_COMPARATOR); // Return entries that are flagged Counts.UNDEFINED in the value. Return the row. This is // what is missing. TaskAttemptContext context = new TaskAttemptContextImpl(conf, new TaskAttemptID()); try (SequenceFileAsBinaryInputFormat.SequenceFileAsBinaryRecordReader rr = new SequenceFileAsBinaryInputFormat.SequenceFileAsBinaryRecordReader()) { InputSplit is = new FileSplit(keyFileStatus.getPath(), 0, keyFileStatus.getLen(), new String [] {}); rr.initialize(is, context); while (rr.nextKeyValue()) { rr.getCurrentKey(); BytesWritable bw = rr.getCurrentValue(); if (Verify.VerifyReducer.whichType(bw.getBytes()) == Verify.Counts.UNDEFINED) { byte[] key = new byte[rr.getCurrentKey().getLength()]; System.arraycopy(rr.getCurrentKey().getBytes(), 0, key, 0, rr.getCurrentKey() .getLength()); result.add(key); } } } return result; } }
@Override public long getLen() { return myFs.getLen(); }
@Override public long getLen() { return myFs.getLen(); }
@Override public long getLen() { return myFs.getLen(); }
@Override public long getLen() { return myFs.getLen(); }
protected FileMetadata toMetadata(LocatedFileStatus fileStatus) { List<FileMetadata.BlockInfo> blocks = new ArrayList<>(); blocks.addAll(Arrays.stream(fileStatus.getBlockLocations()) .map(block -> new FileMetadata.BlockInfo(block.getOffset(), block.getLength(), block.isCorrupt())) .collect(Collectors.toList())); return new FileMetadata(fileStatus.getPath().toString(), fileStatus.getLen(), blocks); }
private Optional<InternalHiveSplit> createInternalHiveSplit(LocatedFileStatus status, OptionalInt bucketNumber, boolean splittable) { splittable = splittable && isSplittable(inputFormat, fileSystem, status.getPath()); return createInternalHiveSplit( status.getPath(), status.getBlockLocations(), 0, status.getLen(), status.getLen(), bucketNumber, splittable); }
@Override public CarbonFile[] locationAwareListFiles(PathFilter pathFilter) throws IOException { if (null != fileStatus && fileStatus.isDirectory()) { List<FileStatus> listStatus = new ArrayList<>(); Path path = fileStatus.getPath(); RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(path); while (iter.hasNext()) { LocatedFileStatus fileStatus = iter.next(); if (pathFilter.accept(fileStatus.getPath()) && fileStatus.getLen() > 0) { listStatus.add(fileStatus); } } return getFiles(listStatus.toArray(new FileStatus[listStatus.size()])); } return new CarbonFile[0]; }
while (files.hasNext()) { LocatedFileStatus file = files.next(); Path filePath = file.getPath(); // Get only the files with created date = current date if (DateUtils.truncate(new Date(file.getModificationTime()), java.util.Calendar.DAY_OF_MONTH).equals(dateNow)) { if (pattern.matcher(filePath.getName()).matches()) { fileInfo.put(file.getPath().getName(), file.getLen()); } } }
private Long getBytesUsed(CocosUser user, String... filesToOmmit) throws IOException { Long used = 0l; Path path = getUserDirectory(user); RemoteIterator<LocatedFileStatus> filesIt = fileSystem.listFiles(path, true); if (!filesIt.hasNext()) { return used; } List<String> ommit = filesToOmmit != null ? Arrays.asList(filesToOmmit) : new ArrayList<>(); for (LocatedFileStatus status = filesIt.next(); filesIt.hasNext(); status = filesIt.next()) { if (ommit.contains(status.getPath().getName())) { continue; } used += status.getLen(); } return used; }
private void verifyLocatedFileStatus( JobConf conf, List<LocatedFileStatus> stats) throws IOException { if (!conf.getBoolean("mapred.fileinputformat.verifysplits", true)) { return; } for (LocatedFileStatus stat: stats) { long fileLen = stat.getLen(); long blockLenTotal = 0; for (BlockLocation loc: stat.getBlockLocations()) { blockLenTotal += loc.getLength(); } if (blockLenTotal != fileLen) { throw new IOException("Error while getting located status, " + stat.getPath() + " has length " + fileLen + " but blocks total is " + blockLenTotal); } } }
@Override public List<SplitPath> getInputSplits(Configuration config, Path path, int splitSize) throws IOException { ImmutableList.Builder<SplitPath> splits = ImmutableList.builder(); RemoteIterator<LocatedFileStatus> files = listFiles(path, false); if (!files.hasNext()) { // No splits. Don't return nothing, return a single empty split String table = getTableName(_rootPath, path); return ImmutableList.of(new SplitPath(getSplitPath(_rootPath, table, getEmptySplitFileName()), 1)); } while (files.hasNext()) { LocatedFileStatus file = files.next(); splits.add(new SplitPath(file.getPath(), file.getLen())); } return splits.build(); }
@Override public List<SplitPath> getInputSplits(Configuration config, Path path, int splitSize) throws IOException { ImmutableList.Builder<SplitPath> splits = ImmutableList.builder(); RemoteIterator<LocatedFileStatus> files = listFiles(path, false); if (!files.hasNext()) { // No splits. Don't return nothing, return a single empty split String table = getTableName(_rootPath, path); return ImmutableList.of(new SplitPath(getSplitPath(_rootPath, table, getEmptySplitFileName()), 1)); } while (files.hasNext()) { LocatedFileStatus file = files.next(); splits.add(new SplitPath(file.getPath(), file.getLen())); } return splits.build(); }
public void lsR(final String description, final FileSystem fs, final Path dir) throws IOException { RemoteIterator<LocatedFileStatus> files = fs.listFiles(dir, true); LOG.info("{}: {}:", description, dir); StringBuilder sb = new StringBuilder(); while(files.hasNext()) { LocatedFileStatus status = files.next(); sb.append(String.format(" %s; type=%s; length=%d", status.getPath(), status.isDirectory()? "dir" : "file", status.getLen())); } LOG.info("{}", sb); }
private static void checkEquals(RemoteIterator<LocatedFileStatus> i1, RemoteIterator<LocatedFileStatus> i2) throws IOException { while (i1.hasNext()) { assertTrue(i2.hasNext()); // Compare all the fields but the path name, which is relative // to the original path from listFiles. LocatedFileStatus l1 = i1.next(); LocatedFileStatus l2 = i2.next(); assertEquals(l1.getAccessTime(), l2.getAccessTime()); assertEquals(l1.getBlockSize(), l2.getBlockSize()); assertEquals(l1.getGroup(), l2.getGroup()); assertEquals(l1.getLen(), l2.getLen()); assertEquals(l1.getModificationTime(), l2.getModificationTime()); assertEquals(l1.getOwner(), l2.getOwner()); assertEquals(l1.getPermission(), l2.getPermission()); assertEquals(l1.getReplication(), l2.getReplication()); } assertFalse(i2.hasNext()); }