private boolean writeS3MetadataFile(String filenameHint, Path pathToS3Directory, String globForS3Files, Optional<String> s3Bucket, Optional<String> s3KeyPattern, boolean finished, Optional<String> s3StorageClass, Optional<Long> applyS3StorageClassAfterBytes, boolean checkSubdirectories) { final String s3UploaderBucket = s3Bucket.or(taskDefinition.getExecutorData().getDefaultS3Bucket()); if (Strings.isNullOrEmpty(s3UploaderBucket)) { log.warn("No s3 bucket specified, not writing s3 metadata for file matcher {}", globForS3Files); return false; } S3UploadMetadata s3UploadMetadata = new S3UploadMetadata(pathToS3Directory.toString(), globForS3Files, s3UploaderBucket, getS3KeyPattern(s3KeyPattern.or(taskDefinition.getExecutorData().getS3UploaderKeyPattern())), finished, Optional.<String> absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional. absent(), s3StorageClass, applyS3StorageClassAfterBytes, Optional.of(finished), Optional.of(checkSubdirectories), Optional.absent(), Collections.emptyMap(), Optional.absent(), Optional.absent(), Optional.absent()); String s3UploadMetadataFileName = String.format("%s-%s%s", taskDefinition.getTaskId(), filenameHint, baseConfiguration.getS3UploaderMetadataSuffix()); Path s3UploadMetadataPath = Paths.get(baseConfiguration.getS3UploaderMetadataDirectory()).resolve(s3UploadMetadataFileName); return jsonObjectFileHelper.writeObject(s3UploadMetadata, s3UploadMetadataPath, log); }
private void readInitialFiles() throws IOException { final long start = System.currentTimeMillis(); LOG.info("Scanning for metadata files (*{}) in {}", baseConfiguration.getS3UploaderMetadataSuffix(), baseConfiguration.getS3UploaderMetadataDirectory()); int foundFiles = 0; for (Path file : JavaUtils.iterable(Paths.get(baseConfiguration.getS3UploaderMetadataDirectory()))) { if (!isS3MetadataFile(file)) { continue; } if (handleNewOrModifiedS3Metadata(file)) { foundFiles++; } } LOG.info("Found {} file(s) in {}", foundFiles, JavaUtils.duration(start)); }
@Inject public SingularityS3UploaderDriver(SingularityRunnerBaseConfiguration baseConfiguration, SingularityS3UploaderConfiguration configuration, SingularityS3Configuration s3Configuration, SingularityS3UploaderMetrics metrics, JsonObjectFileHelper jsonObjectFileHelper, @Named(SingularityRunnerBaseModule.HOST_NAME_PROPERTY) String hostname, SingularityRunnerExceptionNotifier exceptionNotifier) { super(configuration.getPollForShutDownMillis(), Paths.get(baseConfiguration.getS3UploaderMetadataDirectory()), ImmutableList.of(StandardWatchEventKinds.ENTRY_CREATE, StandardWatchEventKinds.ENTRY_MODIFY, StandardWatchEventKinds.ENTRY_DELETE)); this.baseConfiguration = baseConfiguration; this.s3Configuration = s3Configuration; this.metrics = metrics; this.fileSystem = FileSystems.getDefault(); this.jsonObjectFileHelper = jsonObjectFileHelper; this.configuration = configuration; this.metadataToUploader = Maps.newHashMap(); this.uploaderLastHadFilesAt = Maps.newHashMap(); this.expiring = Sets.newHashSet(); this.metrics.setExpiringCollection(expiring); this.runLock = new ReentrantLock(); this.checkFileOpenLock = new ReentrantLock(); this.processUtils = new ProcessUtils(LOG); this.executorService = JavaUtils.newFixedTimingOutThreadPool(configuration.getExecutorMaxUploadThreads(), TimeUnit.SECONDS.toMillis(30), "SingularityS3Uploader-%d"); this.scheduler = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat("SingularityS3Driver-%d").build()); this.hostname = hostname; this.exceptionNotifier = exceptionNotifier; this.immediateUploadersFutures = new ConcurrentHashMap<>(); this.metadataToImmediateUploader = new ConcurrentHashMap<>(); }
@Override protected boolean processEvent(Kind<?> kind, final Path filename) throws IOException { metrics.getFilesystemEventsMeter().mark(); if (!isS3MetadataFile(filename)) { return false; } runLock.lock(); try { if (isStopped()) { LOG.warn("Driver is stopped, ignoring file watch event for {}", filename); return false; } final Path fullPath = Paths.get(baseConfiguration.getS3UploaderMetadataDirectory()).resolve(filename); if (kind.equals(StandardWatchEventKinds.ENTRY_DELETE)) { Optional<SingularityUploader> found = Iterables.tryFind(metadataToUploader.values(), input -> input != null && input.getMetadataPath().equals(fullPath)) .or(Iterables.tryFind(metadataToImmediateUploader.values(), input -> input != null && input.getMetadataPath().equals(fullPath))); LOG.trace("Found {} to match deleted path {}", found, filename); if (found.isPresent()) { expiring.add(found.get()); } } else { return handleNewOrModifiedS3Metadata(fullPath); } return false; } finally { runLock.unlock(); } }