private LogCopier buildLogCopier(Config config, Path sinkLogDir, Path appWorkDir) throws IOException { FileSystem rawLocalFs = this.closer.register(new RawLocalFileSystem()); rawLocalFs.initialize(URI.create(ConfigurationKeys.LOCAL_FS_URI), new Configuration()); LogCopier.Builder builder = LogCopier.newBuilder() .useSrcFileSystem(this.fs) .useDestFileSystem(rawLocalFs) .readFrom(getHdfsLogDir(appWorkDir)) .writeTo(sinkLogDir) .acceptsLogFileExtensions(ImmutableSet.of(ApplicationConstants.STDOUT, ApplicationConstants.STDERR)); if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE)) { builder.useMaxBytesPerLogFile(config.getBytes(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE)); } if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER)) { builder.useScheduler(config.getString(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER)); } return builder.build(); }
/** * Build a {@link LogCopier} instance used to copy the logs out from this {@link GobblinYarnLogSource}. * * @param config the {@link Config} use to create the {@link LogCopier} * @param containerId the {@link ContainerId} of the container the {@link LogCopier} runs in * @param destFs the destination {@link FileSystem} * @param appWorkDir the Gobblin Yarn application working directory on HDFS * @return a {@link LogCopier} instance * @throws IOException if it fails on any IO operation */ protected LogCopier buildLogCopier(Config config, ContainerId containerId, FileSystem destFs, Path appWorkDir) throws IOException { LogCopier.Builder builder = LogCopier.newBuilder() .useSrcFileSystem(FileSystem.getLocal(new Configuration())) .useDestFileSystem(destFs) .readFrom(getLocalLogDir()) .writeTo(getHdfsLogDir(containerId, destFs, appWorkDir)) .acceptsLogFileExtensions(ImmutableSet.of(ApplicationConstants.STDOUT, ApplicationConstants.STDERR)) .useLogFileNamePrefix(containerId.toString()); if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE)) { builder.useMaxBytesPerLogFile(config.getBytes(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE)); } if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER)) { builder.useScheduler(config.getString(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER)); } return builder.build(); }
private LogCopier buildLogCopier(Config config, Path sinkLogDir, Path appWorkDir) throws IOException { FileSystem rawLocalFs = this.closer.register(new RawLocalFileSystem()); rawLocalFs.initialize(URI.create(ConfigurationKeys.LOCAL_FS_URI), new Configuration()); LogCopier.Builder builder = LogCopier.newBuilder() .useSrcFileSystem(this.fs) .useDestFileSystem(rawLocalFs) .readFrom(getHdfsLogDir(appWorkDir)) .writeTo(sinkLogDir) .acceptsLogFileExtensions(ImmutableSet.of(ApplicationConstants.STDOUT, ApplicationConstants.STDERR)); if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE)) { builder.useMaxBytesPerLogFile(config.getBytes(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE)); } if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER)) { builder.useScheduler(config.getString(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER)); } return builder.build(); }
/** * Build a {@link LogCopier} instance used to copy the logs out from this {@link GobblinYarnLogSource}. * * @param config the {@link Config} use to create the {@link LogCopier} * @param containerId the {@link ContainerId} of the container the {@link LogCopier} runs in * @param destFs the destination {@link FileSystem} * @param appWorkDir the Gobblin Yarn application working directory on HDFS * @return a {@link LogCopier} instance * @throws IOException if it fails on any IO operation */ protected LogCopier buildLogCopier(Config config, ContainerId containerId, FileSystem destFs, Path appWorkDir) throws IOException { LogCopier.Builder builder = LogCopier.newBuilder() .useSrcFileSystem(FileSystem.getLocal(new Configuration())) .useDestFileSystem(destFs) .readFrom(getLocalLogDir()) .writeTo(getHdfsLogDir(containerId, destFs, appWorkDir)) .acceptsLogFileExtensions(ImmutableSet.of(ApplicationConstants.STDOUT, ApplicationConstants.STDERR)) .useLogFileNamePrefix(containerId.toString()); if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE)) { builder.useMaxBytesPerLogFile(config.getBytes(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE)); } if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER)) { builder.useScheduler(config.getString(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER)); } return builder.build(); }