public HadoopFsHelper(State state) { this(state, HadoopUtils.getConfFromState(state)); }
public static Configuration getFsConfiguration(State state) { return HadoopUtils.getConfFromState(state, Optional.of(WRITER_ENCRYPTED_CONFIG_PATH)); } }
public static Configuration getConfFromState(State state) { return getConfFromState(state, Optional.<String> absent()); }
private FileSystem getFileSystem(String fsUri) throws IOException { return FileSystem.get(URI.create(fsUri), HadoopUtils.getConfFromState(this.props)); }
private static FileSystem getFileSystem(State state) { try { if (state.contains(MRCompactor.COMPACTION_FILE_SYSTEM_URI)) { URI uri = URI.create(state.getProp(MRCompactor.COMPACTION_FILE_SYSTEM_URI)); return FileSystem.get(uri, HadoopUtils.getConfFromState(state)); } return FileSystem.get(HadoopUtils.getConfFromState(state)); } catch (IOException e) { throw new RuntimeException("Failed to get filesystem for datasetsFinder.", e); } }
private FileSystem getFileSystem(State state) throws IOException { Configuration conf = HadoopUtils.getConfFromState(state); String uri = state.getProp(ConfigurationKeys.SOURCE_FILEBASED_FS_URI, ConfigurationKeys.LOCAL_FS_URI); FileSystem fs = FileSystem.get(URI.create(uri), conf); return fs; }
public DeletingCopyDataPublisher(State state) throws IOException { super(state); Configuration conf = HadoopUtils.getConfFromState(state); String uri = state.getProp(ConfigurationKeys.SOURCE_FILEBASED_FS_URI, ConfigurationKeys.LOCAL_FS_URI); this.sourceFs = FileSystem.get(URI.create(uri), conf); }
public static FileSystem getSourceFs(State state) throws IOException { if (state.contains(HIVE_SOURCE_FS_URI)) { return FileSystem.get(URI.create(state.getProp(HIVE_SOURCE_FS_URI)), HadoopUtils.getConfFromState(state)); } return FileSystem.get(HadoopUtils.getConfFromState(state)); }
/** * Get a {@link FileSystem} object for the uri specified at {@link ConfigurationKeys#SOURCE_FILEBASED_FS_URI}. * @throws IOException */ public static FileSystem getSourceFileSystem(State state) throws IOException { Configuration conf = HadoopUtils.getConfFromState(state, Optional.of(ConfigurationKeys.SOURCE_FILEBASED_ENCRYPTED_CONFIG_PATH)); String uri = state.getProp(ConfigurationKeys.SOURCE_FILEBASED_FS_URI, ConfigurationKeys.LOCAL_FS_URI); return HadoopUtils.getOptionallyThrottledFileSystem(FileSystem.get(URI.create(uri), conf), state); }
/** * @deprecated use {@link HadoopUtils#getSourceFileSystem(State)}. */ @Deprecated protected FileSystem getSourceFileSystem(State state) throws IOException { Configuration conf = HadoopUtils.getConfFromState(state, Optional.of(ConfigurationKeys.SOURCE_FILEBASED_ENCRYPTED_CONFIG_PATH)); String uri = state.getProp(ConfigurationKeys.SOURCE_FILEBASED_FS_URI, ConfigurationKeys.LOCAL_FS_URI); return HadoopUtils.getOptionallyThrottledFileSystem(FileSystem.get(URI.create(uri), conf), state); }
protected FileSystem getSourceFileSystem (State state) throws IOException { Configuration conf = HadoopUtils.getConfFromState(state); String uri = state.getProp(ConfigurationKeys.SOURCE_FILEBASED_FS_URI, ConfigurationKeys.LOCAL_FS_URI); return HadoopUtils.getOptionallyThrottledFileSystem(FileSystem.get(URI.create(uri), conf), state); } }
@Override public String retrieve(Config config, String targetDir) throws IOException { URI uri = URI.create(this.fsUri); FileSystem fs = FileSystem.get(uri, HadoopUtils.getConfFromState(ConfigUtils.configToState(config))); final Path sourceFile = new Path(path); final String zipFile = appendSlash(targetDir) + StringUtils.substringAfterLast(this.path, File.separator); LOGGER.debug("Downloading to zip: " + zipFile + " from uri: " + sourceFile); fs.copyToLocalFile(sourceFile, new Path(zipFile)); return zipFile; } }
public static FileSystem getSourceFileSystem(State state) throws IOException { Configuration conf = HadoopUtils.getConfFromState(state); String uri = state.getProp(ConfigurationKeys.SOURCE_FILEBASED_FS_URI, ConfigurationKeys.LOCAL_FS_URI); return HadoopUtils.getOptionallyThrottledFileSystem(FileSystem.get(URI.create(uri), conf), state); }
protected Optional<CommitSequenceStore> createCommitSequenceStore() throws IOException { if (this.semantics != DeliverySemantics.EXACTLY_ONCE) { return Optional.<CommitSequenceStore>absent(); } Preconditions.checkState(this.jobState.contains(FsCommitSequenceStore.GOBBLIN_RUNTIME_COMMIT_SEQUENCE_STORE_DIR)); try (FileSystem fs = FileSystem.get(URI.create(this.jobState .getProp(FsCommitSequenceStore.GOBBLIN_RUNTIME_COMMIT_SEQUENCE_STORE_FS_URI, ConfigurationKeys.LOCAL_FS_URI)), HadoopUtils.getConfFromState(this.jobState))) { return Optional.<CommitSequenceStore>of(new FsCommitSequenceStore(fs, new Path(this.jobState.getProp(FsCommitSequenceStore.GOBBLIN_RUNTIME_COMMIT_SEQUENCE_STORE_DIR)))); } }
/** * Copy dependent jars to a temporary job directory on HDFS */ private void copyJarDependencies (State state) throws IOException { if (this.tmpJobDir == null) { throw new RuntimeException("Job directory is not created"); } if (!state.contains(ConfigurationKeys.JOB_JAR_FILES_KEY)) { return; } // create sub-dir to save jar files LocalFileSystem lfs = FileSystem.getLocal(HadoopUtils.getConfFromState(state)); Path tmpJarFileDir = new Path(this.tmpJobDir, MRCompactor.COMPACTION_JAR_SUBDIR); this.fs.mkdirs(tmpJarFileDir); state.setProp (MRCompactor.COMPACTION_JARS, tmpJarFileDir.toString()); // copy jar files to hdfs for (String jarFile : state.getPropAsList(ConfigurationKeys.JOB_JAR_FILES_KEY)) { for (FileStatus status : lfs.globStatus(new Path(jarFile))) { Path tmpJarFile = new Path(this.fs.makeQualified(tmpJarFileDir), status.getPath().getName()); this.fs.copyFromLocalFile(status.getPath(), tmpJarFile); log.info(String.format("%s will be added to classpath", tmpJarFile)); } } } }
public HiveAvroSerDeManager(State props) throws IOException { super(props); if (props.contains(HiveRegistrationPolicyBase.HIVE_FS_URI)) { this.fs = FileSystem.get(URI.create(props.getProp(HiveRegistrationPolicyBase.HIVE_FS_URI)), HadoopUtils.getConfFromState(props)); } else { this.fs = FileSystem.get(HadoopUtils.getConfFromState(props)); } this.useSchemaFile = props.getPropAsBoolean(USE_SCHEMA_FILE, DEFAULT_USE_SCHEMA_FILE); this.useSchemaTempFile = props.getPropAsBoolean(USE_SCHEMA_TEMP_FILE, DEFAULT_USE_SCHEMA_TEMP_FILE); this.schemaFileName = props.getProp(SCHEMA_FILE_NAME, DEFAULT_SCHEMA_FILE_NAME); this.schemaTempFileName = props.getProp(SCHEMA_TEMP_FILE_NAME, DEFAULT_SCHEMA_TEMP_FILE_NAME); this.schemaLiteralLengthLimit = props.getPropAsInt(SCHEMA_LITERAL_LENGTH_LIMIT, DEFAULT_SCHEMA_LITERAL_LENGTH_LIMIT); this.metricContext = Instrumented.getMetricContext(props, HiveAvroSerDeManager.class); }
@Test public void testStateToConfiguration() throws IOException { Map<String, String> vals = Maps.newHashMap(); vals.put("test_key1", "test_val1"); vals.put("test_key2", "test_val2"); Configuration expected = HadoopUtils.newConfiguration(); State state = new State(); for (Map.Entry<String, String> entry : vals.entrySet()) { state.setProp(entry.getKey(), entry.getValue()); expected.set(entry.getKey(), entry.getValue()); } Assert.assertEquals(HadoopUtils.getConfFromState(state), expected); Assert.assertEquals(HadoopUtils.getConfFromState(state, Optional.<String>absent()), expected); Assert.assertEquals(HadoopUtils.getConfFromState(state, Optional.of("dummy")), expected); }
@Override public HiveSerDeConverter init(WorkUnitState state) { super.init(state); Configuration conf = HadoopUtils.getConfFromState(state); try { this.serializer = HiveSerDeWrapper.getSerializer(state).getSerDe(); this.deserializer = HiveSerDeWrapper.getDeserializer(state).getSerDe(); this.deserializer.initialize(conf, state.getProperties()); setColumnsIfPossible(state); this.serializer.initialize(conf, state.getProperties()); } catch (IOException e) { log.error("Failed to instantiate serializer and deserializer", e); throw Throwables.propagate(e); } catch (SerDeException e) { log.error("Failed to initialize serializer and deserializer", e); throw Throwables.propagate(e); } return this; }
@Override public FileAwareInputStream readRecord(@Deprecated FileAwareInputStream reuse) throws DataRecordException, IOException { if (!this.recordRead) { Configuration conf = this.state == null ? HadoopUtils.newConfiguration() : HadoopUtils.getConfFromState(this.state); FileSystem fsFromFile = this.file.getOrigin().getPath().getFileSystem(conf); this.recordRead = true; FileAwareInputStream.FileAwareInputStreamBuilder builder = FileAwareInputStream.builder().file(this.file); if (this.file.getFileStatus().isDirectory()) { return builder.inputStream(EmptyInputStream.instance).build(); } FSDataInputStream dataInputStream = fsFromFile.open(this.file.getFileStatus().getPath()); if (this.state != null && DistcpFileSplitter.isSplitWorkUnit(this.state)) { Optional<DistcpFileSplitter.Split> split = DistcpFileSplitter.getSplit(this.state); builder.split(split); if (split.isPresent()) { dataInputStream.seek(split.get().getLowPosition()); } } builder.inputStream(MeteredInputStream.builder().in(dataInputStream).build()); return builder.build(); } return null; }
@VisibleForTesting DatasetsFinder(State state, FileSystem fs) { this.state = state; this.conf = HadoopUtils.getConfFromState(state); this.fs = fs; this.inputDir = getInputDir(); this.destDir = getDestDir(); this.tmpOutputDir = getTmpOutputDir(); this.blacklist = DatasetFilterUtils.getPatternList(state, MRCompactor.COMPACTION_BLACKLIST); this.whitelist = DatasetFilterUtils.getPatternList(state, MRCompactor.COMPACTION_WHITELIST); setTopicsFromConfigStore(state); this.highPriority = getHighPriorityPatterns(); this.normalPriority = getNormalPriorityPatterns(); this.recompactDatasets = getRecompactDatasets(); }