private Path getDataFile(ExecutionBlockId ebId, int partId) throws IOException { try { // the base dir for an output dir String executionBlockBaseDir = ebId.getQueryId().toString() + "/output" + "/" + ebId.getId() + "/hash-shuffle"; Path baseDirPath = localFS.makeQualified(lDirAllocator.getLocalPathForWrite(executionBlockBaseDir, systemConf)); //LOG.info(ebId + "'s basedir is created (" + baseDirPath + ")"); // If EB has many partition, too many shuffle file are in single directory. return StorageUtil.concatPath(baseDirPath, "" + getPartParentId(partId, systemConf), "" + partId); } catch (Exception e) { LOG.error(e.getMessage(), e); throw new IOException(e); } }
Path dataFile = getDataFile(ebId, partId); FileSystem fs = dataFile.getFileSystem(systemConf); if (fs.exists(dataFile)) { partitionAppenderMeta.dataFile = dataFile; partitionAppenderMeta.appender = new HashShuffleAppenderWrapper(ebId, partId, pageSize, appender, getVolumeId(dataFile)); partitionAppenderMeta.appender.init(); partitionAppenderMap.put(partId, partitionAppenderMeta);
public HashShuffleAppenderManager getHashShuffleAppenderManager() { if(hashShuffleAppenderManager == null) { if (workerContext != null) { this.hashShuffleAppenderManager = workerContext.getHashShuffleAppenderManager(); } else { try { this.hashShuffleAppenderManager = new HashShuffleAppenderManager(queryContext.getConf()); } catch (IOException e) { LOG.error(e.getMessage(), e); } } } return hashShuffleAppenderManager; }
List<IntermediateEntryProto> intermediateEntries = Lists.newArrayList(); List<HashShuffleAppenderManager.HashShuffleIntermediate> shuffles = getWorkerContext().getHashShuffleAppenderManager().close(ebId); if (shuffles == null) { reporterBuilder.addAllIntermediateEntries(intermediateEntries);
Path dataFile = getDataFile(ebId, partId); FileSystem fs = dataFile.getFileSystem(systemConf); if (fs.exists(dataFile)) {
/** * Asynchronously write partitions. */ public Future<MemoryRowBlock> writePartitions(TableMeta meta, Schema schema, final TaskAttemptId taskId, int partId, final MemoryRowBlock rowBlock, final boolean release) throws IOException { HashShuffleAppenderWrapper appender = getAppender(rowBlock, taskId.getTaskId().getExecutionBlockId(), partId, meta, schema); ExecutorService executor = executors.get(appender.getVolumeId()); return executor.submit(new Callable<MemoryRowBlock>() { @Override public MemoryRowBlock call() throws Exception { appender.writeRowBlock(taskId, rowBlock); if (release) rowBlock.release(); else rowBlock.clear(); return rowBlock; } }); }
context.getHashShuffleAppenderManager().finalizeTask(getId());
List<IntermediateEntryProto> intermediateEntries = Lists.newArrayList(); List<HashShuffleAppenderManager.HashShuffleIntermediate> shuffles = getWorkerContext().getHashShuffleAppenderManager().close(ebId); if (shuffles == null) { reporterBuilder.addAllIntermediateEntries(intermediateEntries);
/** * Asynchronously write partitions. */ public Future<MemoryRowBlock> writePartitions(TableMeta meta, Schema schema, final TaskAttemptId taskId, int partId, final MemoryRowBlock rowBlock, final boolean release) throws IOException { final HashShuffleAppenderWrapper appender = getAppender(rowBlock, taskId.getTaskId().getExecutionBlockId(), partId, meta, schema); ExecutorService executor = executors.get(appender.getVolumeId()); return executor.submit(new Callable<MemoryRowBlock>() { @Override public MemoryRowBlock call() throws Exception { appender.writeRowBlock(taskId, rowBlock); if (release) rowBlock.release(); else rowBlock.clear(); return rowBlock; } }); }
context.getHashShuffleAppenderManager().finalizeTask(getId());
private Path getDataFile(ExecutionBlockId ebId, int partId) throws IOException { try { // the base dir for an output dir String executionBlockBaseDir = ebId.getQueryId().toString() + "/output" + "/" + ebId.getId() + "/hash-shuffle"; Path baseDirPath = lDirAllocator.getLocalPathForWrite(executionBlockBaseDir, systemConf); //LOG.info(ebId + "'s basedir is created (" + baseDirPath + ")"); // If EB has many partition, too many shuffle file are in single directory. return localFS.makeQualified( StorageUtil.concatPath(baseDirPath, "" + getPartParentId(partId, systemConf), "" + partId)); } catch (Exception e) { LOG.error(e.getMessage(), e); throw new IOException(e); } }
Path dataFile = getDataFile(ebId, partId); FileSystem fs = dataFile.getFileSystem(systemConf); if (fs.exists(dataFile)) { partitionAppenderMeta.dataFile = dataFile; partitionAppenderMeta.appender = new HashShuffleAppenderWrapper(ebId, partId, pageSize, appender, getVolumeId(dataFile)); partitionAppenderMeta.appender.init(); partitionAppenderMap.put(partId, partitionAppenderMeta);
exec.next(); exec.close(); ctx.getHashShuffleAppenderManager().close(ebId);
public HashShuffleAppenderManager getHashShuffleAppenderManager() { if(hashShuffleAppenderManager == null) { if (workerContext != null) { this.hashShuffleAppenderManager = workerContext.getHashShuffleAppenderManager(); } else { try { this.hashShuffleAppenderManager = new HashShuffleAppenderManager(queryContext.getConf()); } catch (IOException e) { LOG.error(e.getMessage(), e); } } } return hashShuffleAppenderManager; }
private Path getDataFile(ExecutionBlockId ebId, int partId) throws IOException { try { // the base dir for an output dir String executionBlockBaseDir = ebId.getQueryId().toString() + "/output" + "/" + ebId.getId() + "/hash-shuffle"; Path baseDirPath = lDirAllocator.getLocalPathForWrite(executionBlockBaseDir, systemConf); //LOG.info(ebId + "'s basedir is created (" + baseDirPath + ")"); // If EB has many partition, too many shuffle file are in single directory. return localFS.makeQualified( StorageUtil.concatPath(baseDirPath, "" + getPartParentId(partId, systemConf), "" + partId)); } catch (Exception e) { LOG.error(e.getMessage(), e); throw new IOException(e); } }
exec.next(); exec.close(); ctx.getHashShuffleAppenderManager().close(ebId);
hashShuffleAppenderManager = new HashShuffleAppenderManager(systemConf); } catch (IOException e) { LOG.fatal(e.getMessage(), e);
int partParentId = HashShuffleAppenderManager.getPartParentId(Integer.parseInt(partId), conf); Path partPath = StorageUtil.concatPath(queryBaseDir, "hash-shuffle", String.valueOf(partParentId), partId);
hashShuffleAppenderManager = new HashShuffleAppenderManager(systemConf); } catch (IOException e) { LOG.fatal(e.getMessage(), e);
final long offset = params.offset(); final long length = params.length(); final int partParentId = HashShuffleAppenderManager.getPartParentId(Integer.parseInt(partId), conf); final Path partPath = StorageUtil.concatPath(queryBaseDir, "hash-shuffle", String.valueOf(partParentId), partId);