private void readInitialFiles() throws IOException { final long start = System.currentTimeMillis(); LOG.info("Scanning for metadata files (*{}) in {}", baseConfiguration.getS3UploaderMetadataSuffix(), baseConfiguration.getS3UploaderMetadataDirectory()); int foundFiles = 0; for (Path file : JavaUtils.iterable(Paths.get(baseConfiguration.getS3UploaderMetadataDirectory()))) { if (!isS3MetadataFile(file)) { continue; } if (handleNewOrModifiedS3Metadata(file)) { foundFiles++; } } LOG.info("Found {} file(s) in {}", foundFiles, JavaUtils.duration(start)); }
@Provides @Singleton @Named(HOST_NAME_PROPERTY) public String getHostname(SingularityRunnerBaseConfiguration baseConfiguration) { if (baseConfiguration.getHostname().isPresent()) { return baseConfiguration.getHostname().get(); } try { InetAddress addr = InetAddress.getLocalHost(); return addr.getHostName(); } catch (UnknownHostException e) { throw new RuntimeException("No local hostname/address found, unable to start without functioning local networking - alternatively, hostname can be configured", e); } }
public Optional<String> getRootLogPath() { if (primaryConfiguration.getLoggingFilename().isPresent()) { return Optional.of(Paths.get(primaryConfiguration.getLoggingDirectory().or(baseConfiguration.getLoggingDirectory()).or(BaseRunnerConfiguration.DEFAULT_DIRECTORY)).resolve(primaryConfiguration.getLoggingFilename().get()).toString()); } else { return Optional.absent(); } }
@Inject public SingularityS3UploaderDriver(SingularityRunnerBaseConfiguration baseConfiguration, SingularityS3UploaderConfiguration configuration, SingularityS3Configuration s3Configuration, SingularityS3UploaderMetrics metrics, JsonObjectFileHelper jsonObjectFileHelper, @Named(SingularityRunnerBaseModule.HOST_NAME_PROPERTY) String hostname, SingularityRunnerExceptionNotifier exceptionNotifier) { super(configuration.getPollForShutDownMillis(), Paths.get(baseConfiguration.getS3UploaderMetadataDirectory()), ImmutableList.of(StandardWatchEventKinds.ENTRY_CREATE, StandardWatchEventKinds.ENTRY_MODIFY, StandardWatchEventKinds.ENTRY_DELETE)); this.baseConfiguration = baseConfiguration; this.s3Configuration = s3Configuration; this.metrics = metrics; this.fileSystem = FileSystems.getDefault(); this.jsonObjectFileHelper = jsonObjectFileHelper; this.configuration = configuration; this.metadataToUploader = Maps.newHashMap(); this.uploaderLastHadFilesAt = Maps.newHashMap(); this.expiring = Sets.newHashSet(); this.metrics.setExpiringCollection(expiring); this.runLock = new ReentrantLock(); this.checkFileOpenLock = new ReentrantLock(); this.processUtils = new ProcessUtils(LOG); this.executorService = JavaUtils.newFixedTimingOutThreadPool(configuration.getExecutorMaxUploadThreads(), TimeUnit.SECONDS.toMillis(30), "SingularityS3Uploader-%d"); this.scheduler = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat("SingularityS3Driver-%d").build()); this.hostname = hostname; this.exceptionNotifier = exceptionNotifier; this.immediateUploadersFutures = new ConcurrentHashMap<>(); this.metadataToImmediateUploader = new ConcurrentHashMap<>(); }
private boolean isS3MetadataFile(Path filename) { if (!filename.toString().endsWith(baseConfiguration.getS3UploaderMetadataSuffix())) { LOG.trace("Ignoring a file {} without {} suffix", filename, baseConfiguration.getS3UploaderMetadataSuffix()); return false; } return true; } }
private boolean writeTailMetadata(boolean finished) { if (!taskDefinition.getExecutorData().getLoggingTag().isPresent()) { if (!finished) { log.warn("Not writing logging metadata because logging tag is absent"); } return true; } final TailMetadata tailMetadata = new TailMetadata(taskDefinition.getServiceLogOut(), taskDefinition.getExecutorData().getLoggingTag().get(), taskDefinition.getExecutorData().getLoggingExtraFields(), finished); final Path path = TailMetadata.getTailMetadataPath(Paths.get(baseConfiguration.getLogWatcherMetadataDirectory()), baseConfiguration.getLogWatcherMetadataSuffix(), tailMetadata); return jsonObjectFileHelper.writeObject(tailMetadata, path, log); }
@Inject public SingularityRunnerExceptionNotifier(SingularityRunnerBaseConfiguration configuration) { this.configuration = configuration; if (configuration.getSentryDsn().isPresent()) { this.raven = Optional.of(RavenFactory.ravenInstance(configuration.getSentryDsn().get())); } else { this.raven = Optional.absent(); } }
public FileAppender<ILoggingEvent> buildFileAppender(LoggerContext context, String file) { FileAppender<ILoggingEvent> fileAppender = new FileAppender<>(); fileAppender.setFile(file); fileAppender.setContext(context); fileAppender.setPrudent(true); PatternLayoutEncoder encoder = new PatternLayoutEncoder(); encoder.setContext(context); encoder.setPattern(primaryConfiguration.getLoggingPattern().or(baseConfiguration.getLoggingPattern()).or(JavaUtils.LOGBACK_LOGGING_PATTERN)); encoder.start(); fileAppender.setEncoder(encoder); fileAppender.start(); return fileAppender; } }
public Logger configureRootLogger() { LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory(); Logger rootLogger = prepareRootLogger(context); context.setName(executorPid); context.getLogger("ROOT").setLevel(Level.toLevel(BaseRunnerConfiguration.DEFAULT_ROOT_LOG_LEVEL)); context.getLogger("com.hubspot").setLevel(Level.toLevel(BaseRunnerConfiguration.DEFAULT_HUBSPOT_LOG_LEVEL)); for (Map.Entry<String, String> entry : baseConfiguration.getLoggingLevel().entrySet()) { context.getLogger(entry.getKey()).setLevel(Level.toLevel(entry.getValue())); } for (Map.Entry<String, String> entry : primaryConfiguration.getLoggingLevel().entrySet()) { context.getLogger(entry.getKey()).setLevel(Level.toLevel(entry.getValue())); } if (getRootLogPath().isPresent()) { rootLogger.addAppender(buildFileAppender(context, getRootLogPath().get())); } return rootLogger; }
@Before public void setup() { SingularityRunnerBaseConfiguration baseConfig = new SingularityRunnerBaseConfiguration(); SingularityS3Configuration s3Config = new SingularityS3Configuration(); s3Config.setArtifactCacheDirectory(cacheDir.toString()); artifactManager = new ArtifactManager( baseConfig, s3Config, LoggerFactory.getLogger(ArtifactManagerTest.class), new SingularityRunnerExceptionNotifier(baseConfig) ); }
@Override protected boolean processEvent(Kind<?> kind, final Path filename) throws IOException { metrics.getFilesystemEventsMeter().mark(); if (!isS3MetadataFile(filename)) { return false; } runLock.lock(); try { if (isStopped()) { LOG.warn("Driver is stopped, ignoring file watch event for {}", filename); return false; } final Path fullPath = Paths.get(baseConfiguration.getS3UploaderMetadataDirectory()).resolve(filename); if (kind.equals(StandardWatchEventKinds.ENTRY_DELETE)) { Optional<SingularityUploader> found = Iterables.tryFind(metadataToUploader.values(), input -> input != null && input.getMetadataPath().equals(fullPath)) .or(Iterables.tryFind(metadataToImmediateUploader.values(), input -> input != null && input.getMetadataPath().equals(fullPath))); LOG.trace("Found {} to match deleted path {}", found, filename); if (found.isPresent()) { expiring.add(found.get()); } } else { return handleNewOrModifiedS3Metadata(fullPath); } return false; } finally { runLock.unlock(); } }
@Inject public SingularityRunnerExceptionNotifier(SingularityRunnerBaseConfiguration configuration) { this.configuration = configuration; if (configuration.getSentryDsn().isPresent()) { this.raven = Optional.of(RavenFactory.ravenInstance(configuration.getSentryDsn().get())); } else { this.raven = Optional.absent(); } }
public FileAppender<ILoggingEvent> buildFileAppender(LoggerContext context, String file) { FileAppender<ILoggingEvent> fileAppender = new FileAppender<>(); fileAppender.setFile(file); fileAppender.setContext(context); fileAppender.setPrudent(true); PatternLayoutEncoder encoder = new PatternLayoutEncoder(); encoder.setContext(context); encoder.setPattern(primaryConfiguration.getLoggingPattern().or(baseConfiguration.getLoggingPattern()).or(JavaUtils.LOGBACK_LOGGING_PATTERN)); encoder.start(); fileAppender.setEncoder(encoder); fileAppender.start(); return fileAppender; } }
public Logger configureRootLogger() { LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory(); Logger rootLogger = prepareRootLogger(context); context.setName(executorPid); context.getLogger("ROOT").setLevel(Level.toLevel(BaseRunnerConfiguration.DEFAULT_ROOT_LOG_LEVEL)); context.getLogger("com.hubspot").setLevel(Level.toLevel(BaseRunnerConfiguration.DEFAULT_HUBSPOT_LOG_LEVEL)); for (Map.Entry<String, String> entry : baseConfiguration.getLoggingLevel().entrySet()) { context.getLogger(entry.getKey()).setLevel(Level.toLevel(entry.getValue())); } for (Map.Entry<String, String> entry : primaryConfiguration.getLoggingLevel().entrySet()) { context.getLogger(entry.getKey()).setLevel(Level.toLevel(entry.getValue())); } if (getRootLogPath().isPresent()) { rootLogger.addAppender(buildFileAppender(context, getRootLogPath().get())); } return rootLogger; }
private boolean writeS3MetadataFile(String filenameHint, Path pathToS3Directory, String globForS3Files, Optional<String> s3Bucket, Optional<String> s3KeyPattern, boolean finished, Optional<String> s3StorageClass, Optional<Long> applyS3StorageClassAfterBytes, boolean checkSubdirectories) { final String s3UploaderBucket = s3Bucket.or(taskDefinition.getExecutorData().getDefaultS3Bucket()); if (Strings.isNullOrEmpty(s3UploaderBucket)) { log.warn("No s3 bucket specified, not writing s3 metadata for file matcher {}", globForS3Files); return false; } S3UploadMetadata s3UploadMetadata = new S3UploadMetadata(pathToS3Directory.toString(), globForS3Files, s3UploaderBucket, getS3KeyPattern(s3KeyPattern.or(taskDefinition.getExecutorData().getS3UploaderKeyPattern())), finished, Optional.<String> absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional. absent(), s3StorageClass, applyS3StorageClassAfterBytes, Optional.of(finished), Optional.of(checkSubdirectories), Optional.absent(), Collections.emptyMap(), Optional.absent(), Optional.absent(), Optional.absent()); String s3UploadMetadataFileName = String.format("%s-%s%s", taskDefinition.getTaskId(), filenameHint, baseConfiguration.getS3UploaderMetadataSuffix()); Path s3UploadMetadataPath = Paths.get(baseConfiguration.getS3UploaderMetadataDirectory()).resolve(s3UploadMetadataFileName); return jsonObjectFileHelper.writeObject(s3UploadMetadata, s3UploadMetadataPath, log); }
public Optional<String> getRootLogPath() { if (primaryConfiguration.getLoggingFilename().isPresent()) { return Optional.of(Paths.get(primaryConfiguration.getLoggingDirectory().or(baseConfiguration.getLoggingDirectory()).or(BaseRunnerConfiguration.DEFAULT_DIRECTORY)).resolve(primaryConfiguration.getLoggingFilename().get()).toString()); } else { return Optional.absent(); } }
@Provides @Singleton @Named(HOST_NAME_PROPERTY) public String getHostname(SingularityRunnerBaseConfiguration baseConfiguration) { if (baseConfiguration.getHostname().isPresent()) { return baseConfiguration.getHostname().get(); } try { InetAddress addr = InetAddress.getLocalHost(); return addr.getHostName(); } catch (UnknownHostException e) { throw new RuntimeException("No local hostname/address found, unable to start without functioning local networking - alternatively, hostname can be configured", e); } }