How to use

useSrcFileSystem

method

in

org.apache.gobblin.util.logs.LogCopier$Builder

private LogCopier buildLogCopier(Config config, Path sinkLogDir, Path appWorkDir) throws IOException {
 FileSystem rawLocalFs = this.closer.register(new RawLocalFileSystem());
 rawLocalFs.initialize(URI.create(ConfigurationKeys.LOCAL_FS_URI), new Configuration());
 LogCopier.Builder builder = LogCopier.newBuilder()
     .useSrcFileSystem(this.fs)
     .useDestFileSystem(rawLocalFs)
     .readFrom(getHdfsLogDir(appWorkDir))
     .writeTo(sinkLogDir)
     .acceptsLogFileExtensions(ImmutableSet.of(ApplicationConstants.STDOUT, ApplicationConstants.STDERR));
 if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE)) {
  builder.useMaxBytesPerLogFile(config.getBytes(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE));
 }
 if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER)) {
  builder.useScheduler(config.getString(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER));
 }
 return builder.build();
}

/**
 * Build a {@link LogCopier} instance used to copy the logs out from this {@link GobblinYarnLogSource}.
 *
 * @param config the {@link Config} use to create the {@link LogCopier}
 * @param containerId the {@link ContainerId} of the container the {@link LogCopier} runs in
 * @param destFs the destination {@link FileSystem}
 * @param appWorkDir the Gobblin Yarn application working directory on HDFS
 * @return a {@link LogCopier} instance
 * @throws IOException if it fails on any IO operation
 */
protected LogCopier buildLogCopier(Config config, ContainerId containerId, FileSystem destFs, Path appWorkDir)
  throws IOException {
 LogCopier.Builder builder = LogCopier.newBuilder()
     .useSrcFileSystem(FileSystem.getLocal(new Configuration()))
     .useDestFileSystem(destFs)
     .readFrom(getLocalLogDir())
     .writeTo(getHdfsLogDir(containerId, destFs, appWorkDir))
     .acceptsLogFileExtensions(ImmutableSet.of(ApplicationConstants.STDOUT, ApplicationConstants.STDERR))
     .useLogFileNamePrefix(containerId.toString());
 if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE)) {
  builder.useMaxBytesPerLogFile(config.getBytes(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE));
 }
 if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER)) {
  builder.useScheduler(config.getString(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER));
 }
 return builder.build();
}

private LogCopier buildLogCopier(Config config, Path sinkLogDir, Path appWorkDir) throws IOException {
 FileSystem rawLocalFs = this.closer.register(new RawLocalFileSystem());
 rawLocalFs.initialize(URI.create(ConfigurationKeys.LOCAL_FS_URI), new Configuration());
 LogCopier.Builder builder = LogCopier.newBuilder()
     .useSrcFileSystem(this.fs)
     .useDestFileSystem(rawLocalFs)
     .readFrom(getHdfsLogDir(appWorkDir))
     .writeTo(sinkLogDir)
     .acceptsLogFileExtensions(ImmutableSet.of(ApplicationConstants.STDOUT, ApplicationConstants.STDERR));
 if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE)) {
  builder.useMaxBytesPerLogFile(config.getBytes(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE));
 }
 if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER)) {
  builder.useScheduler(config.getString(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER));
 }
 return builder.build();
}

/**
 * Build a {@link LogCopier} instance used to copy the logs out from this {@link GobblinYarnLogSource}.
 *
 * @param config the {@link Config} use to create the {@link LogCopier}
 * @param containerId the {@link ContainerId} of the container the {@link LogCopier} runs in
 * @param destFs the destination {@link FileSystem}
 * @param appWorkDir the Gobblin Yarn application working directory on HDFS
 * @return a {@link LogCopier} instance
 * @throws IOException if it fails on any IO operation
 */
protected LogCopier buildLogCopier(Config config, ContainerId containerId, FileSystem destFs, Path appWorkDir)
  throws IOException {
 LogCopier.Builder builder = LogCopier.newBuilder()
     .useSrcFileSystem(FileSystem.getLocal(new Configuration()))
     .useDestFileSystem(destFs)
     .readFrom(getLocalLogDir())
     .writeTo(getHdfsLogDir(containerId, destFs, appWorkDir))
     .acceptsLogFileExtensions(ImmutableSet.of(ApplicationConstants.STDOUT, ApplicationConstants.STDERR))
     .useLogFileNamePrefix(containerId.toString());
 if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE)) {
  builder.useMaxBytesPerLogFile(config.getBytes(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE));
 }
 if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER)) {
  builder.useScheduler(config.getString(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER));
 }
 return builder.build();
}