public static Map<KeyExtent,Long> estimateSizes(ServerContext context, Path mapFile, long fileSize, List<KeyExtent> extents) throws IOException { FileSystem ns = context.getVolumeManager().getVolumeByPath(mapFile).getFileSystem(); return BulkImport.estimateSizes(context.getConfiguration(), mapFile, fileSize, extents, ns, null, context.getCryptoService()); }
public FileSKVIterator openReader(FileRef ref) throws IOException { Preconditions.checkState(volumeManager != null, "Opening files is not" + " supported at this time. It's only supported when" + " CompactionStrategy.gatherInformation() is called."); // @TODO verify the file isn't some random file in HDFS // @TODO ensure these files are always closed? FileOperations fileFactory = FileOperations.getInstance(); FileSystem ns = volumeManager.getVolumeByPath(ref.path()).getFileSystem(); return fileFactory.newReaderBuilder() .forFile(ref.path().toString(), ns, ns.getConf(), context.getCryptoService()) .withTableConfiguration(tableConfig).seekToBeginning().build(); }
.forFile(path.toString(), ns, ns.getConf(), context.getCryptoService()) .withTableConfiguration( context.getServerConfFactory().getTableConfiguration(tablet.getTableId()))
.forFile(tmpFile, fs, newConf, context.getCryptoService()).withTableConfiguration(aconf) .build();
try { reader = FileOperations.getInstance().newReaderBuilder() .forFile(mapfile.toString(), ns, ns.getConf(), context.getCryptoService()) .withTableConfiguration(context.getConfiguration()).build();
private Map<FileRef,Pair<Key,Key>> getFirstAndLastKeys(SortedMap<FileRef,DataFileValue> allFiles) throws IOException { Map<FileRef,Pair<Key,Key>> result = new HashMap<>(); FileOperations fileFactory = FileOperations.getInstance(); VolumeManager fs = getTabletServer().getFileSystem(); for (Entry<FileRef,DataFileValue> entry : allFiles.entrySet()) { FileRef file = entry.getKey(); FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem(); try (FileSKVIterator openReader = fileFactory.newReaderBuilder() .forFile(file.path().toString(), ns, ns.getConf(), context.getCryptoService()) .withTableConfiguration(this.getTableConfiguration()).seekToBeginning().build()) { Key first = openReader.getFirstKey(); Key last = openReader.getLastKey(); result.put(file, new Pair<>(first, last)); } } return result; }
.forFile(mapFile.path().toString(), fs, fs.getConf(), context.getCryptoService()) .withTableConfiguration(acuTableConf).withRateLimiter(env.getReadLimiter()).build();
.getFileSystem(); FileSKVWriter writer = new RFileOperations().newWriterBuilder() .forFile(newMapFile.toString(), ns, ns.getConf(), context.getCryptoService()) .withTableConfiguration(acuConf).build(); writer.startDefaultLocalityGroup(); ns = context.getVolumeManager().getVolumeByPath(new Path(s)).getFileSystem(); reader = FileOperations.getInstance().newIndexReaderBuilder() .forFile(s, ns, ns.getConf(), context.getCryptoService()) .withTableConfiguration(acuConf).build(); iters.add(reader);
if (useIndex) reader = FileOperations.getInstance().newIndexReaderBuilder() .forFile(path.toString(), ns, ns.getConf(), context.getCryptoService()) .withTableConfiguration(acuConf).build(); else reader = FileOperations.getInstance().newScanReaderBuilder() .forFile(path.toString(), ns, ns.getConf(), context.getCryptoService()) .withTableConfiguration(acuConf).overRange(new Range(prevEndRow, false, null, true), LocalityGroupUtil.EMPTY_CF_SET, false) .forFile(path.toString(), ns, ns.getConf(), context.getCryptoService()) .withTableConfiguration(acuConf).build()); else readers.add(FileOperations.getInstance().newScanReaderBuilder() .forFile(path.toString(), ns, ns.getConf(), context.getCryptoService()) .withTableConfiguration(acuConf).overRange(new Range(prevEndRow, false, null, true), LocalityGroupUtil.EMPTY_CF_SET, false)
FileSystem ns = this.fs.getVolumeByPath(outputFilePath).getFileSystem(); mfw = fileFactory.newWriterBuilder() .forFile(outputFilePathName, ns, ns.getConf(), context.getCryptoService()) .withTableConfiguration(acuTableConf).withRateLimiter(env.getWriteLimiter()).build();
CryptoService cryptoService = context.getCryptoService(); logFile.write(LOG_FILE_HEADER_V4.getBytes(UTF_8));
long maxTime = -1; try (FileSKVIterator reader = FileOperations.getInstance().newReaderBuilder() .forFile(path.toString(), ns, ns.getConf(), context.getCryptoService()) .withTableConfiguration(conf).seekToBeginning().build()) { while (reader.hasTop()) {
public static WritableComparable<Key> findLastKey(ServerContext context, Collection<FileRef> mapFiles) throws IOException { Key lastKey = null; for (FileRef ref : mapFiles) { Path path = ref.path(); FileSystem ns = context.getVolumeManager().getVolumeByPath(path).getFileSystem(); FileSKVIterator reader = FileOperations.getInstance().newReaderBuilder() .forFile(path.toString(), ns, ns.getConf(), context.getCryptoService()) .withTableConfiguration(context.getConfiguration()).seekToBeginning().build(); try { if (!reader.hasTop()) // file is empty, so there is no last key continue; Key key = reader.getLastKey(); if (lastKey == null || key.compareTo(lastKey) > 0) lastKey = key; } finally { try { if (reader != null) reader.close(); } catch (IOException e) { log.error("{}", e.getMessage(), e); } } } return lastKey; }
.forFile(filename, fs, fs.getConf(), context.getCryptoService()) .withTableConfiguration(context.getConfiguration()).seekToBeginning().build()) { Text row = startRow;
private synchronized FileSKVIterator getReader() throws IOException { if (reader == null) { Configuration conf = context.getHadoopConf(); FileSystem fs = FileSystem.getLocal(conf); reader = new RFileOperations().newReaderBuilder() .forFile(memDumpFile, fs, conf, context.getCryptoService()) .withTableConfiguration(context.getConfiguration()).seekToBeginning().build(); if (iflag != null) reader.setInterruptFlag(iflag); if (getSamplerConfig() != null) { reader = reader.getSample(getSamplerConfig()); } } return reader; }
SummaryCollection fsc = SummaryReader .load(fs, conf, factory, file.path(), summarySelector, summaryCache, indexCache, fileLenCache, context.getCryptoService()) .getSummaries(Collections.singletonList(new Gatherer.RowRange(extent))); sc.merge(fsc, factory);
@Override public TSummaries startGetSummariesForPartition(TInfo tinfo, TCredentials credentials, TSummaryRequest request, int modulus, int remainder) throws ThriftSecurityException, TException { // do not expect users to call this directly, expect other tservers to call this method if (!security.canPerformSystemActions(credentials)) { throw new AccumuloSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED).asThriftException(); } ServerConfigurationFactory factory = context.getServerConfFactory(); ExecutorService spe = resourceManager.getSummaryRemoteExecutor(); Future<SummaryCollection> future = new Gatherer(context, request, factory.getTableConfiguration(Table.ID.of(request.getTableId())), context.getCryptoService()).processPartition(spe, modulus, remainder); return startSummaryOperation(credentials, future); }
@Override public TSummaries startGetSummaries(TInfo tinfo, TCredentials credentials, TSummaryRequest request) throws ThriftSecurityException, ThriftTableOperationException, TException { Namespace.ID namespaceId; Table.ID tableId = Table.ID.of(request.getTableId()); try { namespaceId = Tables.getNamespaceId(context, tableId); } catch (TableNotFoundException e1) { throw new ThriftTableOperationException(tableId.canonicalID(), null, null, TableOperationExceptionType.NOTFOUND, null); } if (!security.canGetSummaries(credentials, tableId, namespaceId)) { throw new AccumuloSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED).asThriftException(); } ServerConfigurationFactory factory = context.getServerConfFactory(); ExecutorService es = resourceManager.getSummaryPartitionExecutor(); Future<SummaryCollection> future = new Gatherer(context, request, factory.getTableConfiguration(tableId), context.getCryptoService()).gather(es); return startSummaryOperation(credentials, future); }
@Override public TSummaries startGetSummariesFromFiles(TInfo tinfo, TCredentials credentials, TSummaryRequest request, Map<String,List<TRowRange>> files) throws ThriftSecurityException, TException { // do not expect users to call this directly, expect other tservers to call this method if (!security.canPerformSystemActions(credentials)) { throw new AccumuloSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED).asThriftException(); } ExecutorService srp = resourceManager.getSummaryRetrievalExecutor(); TableConfiguration tableCfg = confFactory .getTableConfiguration(Table.ID.of(request.getTableId())); BlockCache summaryCache = resourceManager.getSummaryCache(); BlockCache indexCache = resourceManager.getIndexCache(); Cache<String,Long> fileLenCache = resourceManager.getFileLenCache(); FileSystemResolver volMgr = p -> fs.getVolumeByPath(p).getFileSystem(); Future<SummaryCollection> future = new Gatherer(context, request, tableCfg, context.getCryptoService()).processFiles(volMgr, files, summaryCache, indexCache, fileLenCache, srp); return startSummaryOperation(credentials, future); }