public static Configuration getFsConfiguration(State state) { return HadoopUtils.getConfFromState(state, Optional.of(WRITER_ENCRYPTED_CONFIG_PATH)); } }
public static Configuration getConfFromState(State state) { return getConfFromState(state, Optional.<String> absent()); }
private FileSystem getFileSystem(State state) throws IOException { Configuration conf = HadoopUtils.getConfFromState(state); String uri = state.getProp(ConfigurationKeys.SOURCE_FILEBASED_FS_URI, ConfigurationKeys.LOCAL_FS_URI); FileSystem fs = FileSystem.get(URI.create(uri), conf); return fs; }
private static FileSystem getFileSystem(State state) { try { if (state.contains(MRCompactor.COMPACTION_FILE_SYSTEM_URI)) { URI uri = URI.create(state.getProp(MRCompactor.COMPACTION_FILE_SYSTEM_URI)); return FileSystem.get(uri, HadoopUtils.getConfFromState(state)); } return FileSystem.get(HadoopUtils.getConfFromState(state)); } catch (IOException e) { throw new RuntimeException("Failed to get filesystem for datasetsFinder.", e); } }
private static FileSystem getSourceFs(State state) throws IOException { if (state.contains(HIVE_SOURCE_FS_URI)) { return FileSystem.get(URI.create(state.getProp(HIVE_SOURCE_FS_URI)), HadoopUtils.getConfFromState(state)); } return FileSystem.get(HadoopUtils.getConfFromState(state)); }
public DeletingCopyDataPublisher(State state) throws IOException { super(state); Configuration conf = HadoopUtils.getConfFromState(state); String uri = state.getProp(ConfigurationKeys.SOURCE_FILEBASED_FS_URI, ConfigurationKeys.LOCAL_FS_URI); this.sourceFs = FileSystem.get(URI.create(uri), conf); }
public static HiveUnitUpdateProvider create(State state) { try { return (HiveUnitUpdateProvider) GobblinConstructorUtils.invokeFirstConstructor(Class.forName(state.getProp( OPTIONAL_HIVE_UNIT_UPDATE_PROVIDER_CLASS_KEY, DEFAULT_HIVE_UNIT_UPDATE_PROVIDER_CLASS)), ImmutableList.<Object>of(FileSystem.get(HadoopUtils.getConfFromState(state))), ImmutableList.of()); } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException | ClassNotFoundException | IOException e) { throw new RuntimeException(e); } }
protected FileSystem getSourceFileSystem(State state) throws IOException { Configuration conf = HadoopUtils.getConfFromState(state); String uri = state.getProp(ConfigurationKeys.SOURCE_FILEBASED_FS_URI, ConfigurationKeys.LOCAL_FS_URI); return HadoopUtils.getOptionallyThrottledFileSystem(FileSystem.get(URI.create(uri), conf), state); }
protected FileSystem getSourceFileSystem (State state) throws IOException { Configuration conf = HadoopUtils.getConfFromState(state); String uri = state.getProp(ConfigurationKeys.SOURCE_FILEBASED_FS_URI, ConfigurationKeys.LOCAL_FS_URI); return HadoopUtils.getOptionallyThrottledFileSystem(FileSystem.get(URI.create(uri), conf), state); } }
protected FileSystem getSourceFileSystem(State state) throws IOException { Configuration conf = HadoopUtils.getConfFromState(state, Optional.of(ConfigurationKeys.SOURCE_FILEBASED_ENCRYPTED_CONFIG_PATH)); String uri = state.getProp(ConfigurationKeys.SOURCE_FILEBASED_FS_URI, ConfigurationKeys.LOCAL_FS_URI); return HadoopUtils.getOptionallyThrottledFileSystem(FileSystem.get(URI.create(uri), conf), state); }
protected Optional<CommitSequenceStore> createCommitSequenceStore() throws IOException { if (this.semantics != DeliverySemantics.EXACTLY_ONCE) { return Optional.<CommitSequenceStore>absent(); } Preconditions.checkState(this.jobState.contains(FsCommitSequenceStore.GOBBLIN_RUNTIME_COMMIT_SEQUENCE_STORE_DIR)); try (FileSystem fs = FileSystem.get(URI.create(this.jobState .getProp(FsCommitSequenceStore.GOBBLIN_RUNTIME_COMMIT_SEQUENCE_STORE_FS_URI, ConfigurationKeys.LOCAL_FS_URI)), HadoopUtils.getConfFromState(this.jobState))) { return Optional.<CommitSequenceStore>of(new FsCommitSequenceStore(fs, new Path(this.jobState.getProp(FsCommitSequenceStore.GOBBLIN_RUNTIME_COMMIT_SEQUENCE_STORE_DIR)))); } }
/** * Copy dependent jars to a temporary job directory on HDFS */ private void copyJarDependencies (State state) throws IOException { if (this.tmpJobDir == null) { throw new RuntimeException("Job directory is not created"); } if (!state.contains(ConfigurationKeys.JOB_JAR_FILES_KEY)) { return; } // create sub-dir to save jar files LocalFileSystem lfs = FileSystem.getLocal(HadoopUtils.getConfFromState(state)); Path tmpJarFileDir = new Path(this.tmpJobDir, MRCompactor.COMPACTION_JAR_SUBDIR); this.fs.mkdirs(tmpJarFileDir); state.setProp (MRCompactor.COMPACTION_JARS, tmpJarFileDir.toString()); // copy jar files to hdfs for (String jarFile : state.getPropAsList(ConfigurationKeys.JOB_JAR_FILES_KEY)) { for (FileStatus status : lfs.globStatus(new Path(jarFile))) { Path tmpJarFile = new Path(this.fs.makeQualified(tmpJarFileDir), status.getPath().getName()); this.fs.copyFromLocalFile(status.getPath(), tmpJarFile); log.info(String.format("%s will be added to classpath", tmpJarFile)); } } } }
@Override public FileAwareInputStream readRecord(@Deprecated FileAwareInputStream reuse) throws DataRecordException, IOException { if (!this.recordRead) { Configuration conf = this.state == null ? HadoopUtils.newConfiguration() : HadoopUtils.getConfFromState(this.state); FileSystem fsFromFile = this.file.getOrigin().getPath().getFileSystem(conf); this.recordRead = true; return new FileAwareInputStream(this.file, MeteredInputStream.builder().in(fsFromFile.open(this.file.getFileStatus().getPath())).build()); } return null; }
public HiveAvroSerDeManager(State props) throws IOException { super(props); this.fs = FileSystem.get(HadoopUtils.getConfFromState(props)); this.useSchemaFile = props.getPropAsBoolean(USE_SCHEMA_FILE, DEFAULT_USE_SCHEMA_FILE); this.schemaFileName = props.getProp(SCHEMA_FILE_NAME, DEFAULT_SCHEMA_FILE_NAME); this.schemaLiteralLengthLimit = props.getPropAsInt(SCHEMA_LITERAL_LENGTH_LIMIT, DEFAULT_SCHEMA_LITERAL_LENGTH_LIMIT); this.metricContext = Instrumented.getMetricContext(props, HiveAvroSerDeManager.class); }
Configuration conf = HadoopUtils.getConfFromState(state);
@VisibleForTesting DatasetsFinder(State state, FileSystem fs) { this.state = state; this.conf = HadoopUtils.getConfFromState(state); this.fs = fs; this.inputDir = getInputDir(); this.destDir = getDestDir(); this.tmpOutputDir = getTmpOutputDir(); this.blacklist = DatasetFilterUtils.getPatternList(state, MRCompactor.COMPACTION_BLACKLIST); this.whitelist = DatasetFilterUtils.getPatternList(state, MRCompactor.COMPACTION_WHITELIST); setTopicsFromConfigStore(state); this.highPriority = getHighPriorityPatterns(); this.normalPriority = getNormalPriorityPatterns(); this.recompactDatasets = getRecompactDatasets(); }
@Override public void run() { Configuration conf = HadoopUtils.getConfFromState(this.dataset.jobProps());
public MRCompactor(Properties props, List<? extends Tag<?>> tags, Optional<CompactorListener> compactorListener) throws IOException { this.state = new State(); this.state.addAll(props); this.initilizeTime = getCurrentTime(); this.tags = tags; this.conf = HadoopUtils.getConfFromState(this.state); this.tmpOutputDir = getTmpOutputDir(); this.fs = getFileSystem(); this.datasets = getDatasetsFinder().findDistinctDatasets(); this.jobExecutor = createJobExecutor(); this.jobRunnables = Maps.newConcurrentMap(); this.closer = Closer.create(); this.stopwatch = Stopwatch.createStarted(); this.gobblinMetrics = initializeMetrics(); this.eventSubmitter = new EventSubmitter.Builder( GobblinMetrics.get(this.state.getProp(ConfigurationKeys.JOB_NAME_KEY)).getMetricContext(), MRCompactor.COMPACTION_TRACKING_EVENTS_NAMESPACE).build(); this.compactorListener = compactorListener; this.dataVerifTimeoutMinutes = getDataVerifTimeoutMinutes(); this.compactionTimeoutMinutes = getCompactionTimeoutMinutes(); this.shouldVerifDataCompl = shouldVerifyDataCompleteness(); this.compactionCompleteListener = getCompactionCompleteListener(); this.verifier = this.shouldVerifDataCompl ? Optional.of(this.closer.register(new DataCompletenessVerifier(this.state))) : Optional.<DataCompletenessVerifier> absent(); this.shouldPublishDataIfCannotVerifyCompl = shouldPublishDataIfCannotVerifyCompl(); }