@Override public void close() throws Exception { spillService.deleteSpillSubdirs(id); }
public SpillFile getSpillFile(String fileName) throws RuntimeException { try { final SpillDirectory spillDirectory = spillService.getSpillSubdir(id); return new SpillFile(spillDirectory.getFileSystem(), new Path(spillDirectory.getSpillDirPath(), fileName)); } catch (UserException e) { throw UserException.dataWriteError(e) .addContext("for %s spill id %s", caller, id) .addContext("Caller", caller) .build(logger); } }
@Override public void run() { ArrayList<String> newHealthySpillDirs = Lists.newArrayList(); // Local copy of spillDirs, to protect against changes ArrayList<String> currentSpillDirs = spillDirs; for (String spillDir : currentSpillDirs) { final Path spillDirPath = new Path(spillDir); if (isHealthy(spillDirPath)) { newHealthySpillDirs.add(spillDir); } } healthySpillDirs = newHealthySpillDirs; long timeNow = System.currentTimeMillis(); if (lastSpillSweep + spillSweepInterval < timeNow) { long targetTime = timeNow > spillSweepThreshold ? (timeNow - spillSweepThreshold) : 0; for (String spillDir : newHealthySpillDirs) { sweep(spillDir, targetTime); } lastSpillSweep = timeNow; } }
final SpillService spillService = new SpillServiceImpl(DremioConfig.create(null, config), new DefaultSpillServiceOptions(), new Provider<SchedulerService>() { @Override
final File spillDir = spillParentDir.newFolder(); when(config.getStringList(DremioConfig.SPILLING_PATH_STRING)).thenReturn(ImmutableList.of(spillDir.getPath())); final NoopScheduler schedulerService = new NoopScheduler(); final SpillService spillService = new SpillServiceImpl(config, new TestSpillServiceOptions(), new Provider<SchedulerService>() { @Override spillService.start(); spillService.close();
minDiskSpace = options.minDiskSpace(); minDiskSpacePercentage = options.minDiskSpacePercentage(); healthCheckInterval = options.healthCheckInterval(); healthCheckEnabled = healthCheckEnabled && options.enableHealthCheck(); spillSweepInterval = options.spillSweepInterval(); spillSweepThreshold = options.spillSweepThreshold(); .startingAt(Instant.now()) .build(), new SpillHealthCheckTask() );
final DremioConfig dremioConfig = DremioConfig.create(null, config); final SchedulerService schedulerService = Mockito.mock(SchedulerService.class); final SpillService spillService = new SpillServiceImpl(dremioConfig, new SpillServiceOptionsImpl(options), new Provider<SchedulerService>() { @Override
final Path path = new Path(tempDir.getAbsolutePath(), "dremioSerializable"); tempDir.deleteOnExit(); final SpillDirectory spillDirectory = new SpillDirectory(path, fs); return spillDirectory; }).when(spillService).getSpillSubdir(any(String.class));
@Override public SpillDirectory getSpillSubdir(String id) throws UserException { ArrayList<String> currentSpillDirs = Lists.newArrayList(spillDirs); while (!currentSpillDirs.isEmpty()) { // pick a random spill directory final int index = ThreadLocalRandom.current().nextInt(currentSpillDirs.size()); final String spillDir = currentSpillDirs.get(index); final Path spillDirPath = new Path(spillDir); if (isHealthy(spillDirPath)) { try { //TODO: track number of spills created in 'spillDir' FileSystem fileSystem = spillDirPath.getFileSystem(SPILLING_CONFIG); final Path spillSubdir = new Path(spillDirPath, id); return new SpillDirectory(spillSubdir, fileSystem); } catch (IOException e) { // Ignore this 'spillDir'. Still consider the others } } // Hm... 'spillDir' didn't work out. Let's consider the others currentSpillDirs.remove(index); } // TODO: withContextParameters() throw UserException.dataWriteError() .message("Failed to spill to disk. Please check space availability") .addContext("spill id", id) .addContext("all spill locations", spillDirs.toString()) .build(logger); }
/** * Create the spill service * @param config Configuration for the spill service, containing items such as the spill path(s), * number of I/O completion threads, etc. */ public SpillServiceImpl(DremioConfig config, SpillServiceOptions options, final Provider<SchedulerService> schedulerService) { this.spillDirs = new ArrayList<>(config.getStringList(DremioConfig.SPILLING_PATH_STRING)); this.spillStreams = new HashMap<>(); this.options = options; this.schedulerService = schedulerService; // Option values set at start minDiskSpace = 0; minDiskSpacePercentage = 0; healthCheckInterval = 0; spillSweepInterval = 0; spillSweepThreshold = 0; healthCheckEnabled = false; for (String spillDir : this.spillDirs) { try { final Path spillDirPath = new Path(spillDir); final FileSystem fileSystem = spillDirPath.getFileSystem(SPILLING_CONFIG); healthCheckEnabled = healthCheckEnabled || isHealthCheckEnabled(fileSystem.getUri().getScheme()); } catch (Exception e) {} } // healthySpillDirs set at start() this.healthySpillDirs = Lists.newArrayList(); }
public SpillManager(SabotConfig sabotConfig, OptionManager optionManager, String id, Configuration hadoopConf, SpillService spillService, String caller) { final List<String> directories = new ArrayList<>(sabotConfig.getStringList(ExecConstants.SPILL_DIRS)); if (directories.isEmpty()) { throw UserException.dataWriteError().message("No spill locations specified.").build(logger); } this.id = id; this.caller = caller; this.hadoopConf = hadoopConf; this.spillService = spillService; // load options if (optionManager != null) { this.minDiskSpacePercentage = optionManager.getOption(ExecConstants.SPILL_DISK_SPACE_LIMIT_PERCENTAGE); this.minDiskSpace = optionManager.getOption(ExecConstants.SPILL_DISK_SPACE_LIMIT_BYTES); this.healthCheckInterval = optionManager.getOption(ExecConstants.SPILL_DISK_SPACE_CHECK_INTERVAL); } else { this.minDiskSpacePercentage = ExecConstants.SPILL_DISK_SPACE_LIMIT_PERCENTAGE.getDefault().getFloatVal(); this.minDiskSpace = ExecConstants.SPILL_DISK_SPACE_LIMIT_BYTES.getDefault().getNumVal(); this.healthCheckInterval = ExecConstants.SPILL_DISK_SPACE_CHECK_INTERVAL.getDefault().getNumVal(); } try { spillService.makeSpillSubdirs(id); } catch (UserException e) { throw UserException.dataWriteError(e) .addContext("Caller", caller) .build(logger); } }
final SpillService spillService = new SpillServiceImpl(DremioConfig.create(null, config), new DefaultSpillServiceOptions(), new Provider<SchedulerService>() { @Override
)); registry.bind(SpillService.class, new SpillServiceImpl( dremioConfig, new SpillServiceOptionsImpl(registry.provider(SabotContext.class)),
final Path path = new Path(tempDir.getAbsolutePath(), "dremioSerializable"); tempDir.deleteOnExit(); final SpillDirectory spillDirectory = new SpillDirectory(path, fs); return spillDirectory; }).when(spillService).getSpillSubdir(any(String.class));
boolean outOfMemory = false; final SchedulerService schedulerService = Mockito.mock(SchedulerService.class); final SpillService spillService = new SpillServiceImpl(DremioConfig.create(null, config), new DefaultSpillServiceOptions(), new Provider<SchedulerService>() { @Override
registry.bind(SpillService.class, new SpillServiceImpl( config, new SpillServiceOptionsImpl(registry.provider(SabotContext.class)),