/** * Given a {@link Path} path, it returns the path component of its URI, which has the form * scheme://authority/path. * * @param path an HDFS {@link Path} * @return the path component of the {@link Path} URI */ public static String getPathWithoutScheme(Path path) { return path.toUri().getPath(); }
private String pathToObjectName(final Path path) { org.apache.hadoop.fs.Path hadoopPath = HadoopFileSystem.toHadoopPath(path); if (!hadoopPath.isAbsolute()) { hadoopPath = new org.apache.hadoop.fs.Path(fs.getWorkingDirectory(), hadoopPath); } return hadoopPath.toUri().getScheme() != null && hadoopPath.toUri().getPath().isEmpty() ? "" : hadoopPath.toUri().getPath().substring(1); }
/** * Uses the scheme and authority of the object's current location and the path constructed * using the object's new name to construct a path for the object's new location. */ private Path constructRenamedPath(Path defaultNewPath, Path currentPath) { URI currentUri = currentPath.toUri(); return new Path(currentUri.getScheme(), currentUri.getAuthority(), defaultNewPath.toUri().getPath()); }
private URI initializeFromURI(String fromPath, boolean isLocal) throws IOException, URISyntaxException, SemanticException { URI fromURI = new Path(fromPath).toUri(); String path = fromURI.getPath(); try { path = new String(URLCodec.decodeUrl( new Path(System.getProperty("user.dir"), fromPath).toUri().toString() .getBytes("US-ASCII")), "US-ASCII"); } catch (DecoderException de) { path = new Path(new Path("/user/" + System.getProperty("user.name")), path).toString(); } else { URI defaultURI = FileSystem.get(conf).getUri(); fromScheme = defaultURI.getScheme(); fromAuthority = defaultURI.getAuthority(); URI defaultURI = FileSystem.get(conf).getUri(); fromAuthority = defaultURI.getAuthority();
|| conf.getBoolVar(HiveConf.ConfVars.HIVEEXIMTESTMODE);; if (testMode) { URI uri = new Path(location).toUri(); FileSystem fs = FileSystem.get(uri, conf); String scheme = fs.getScheme(); String authority = uri.getAuthority(); String path = uri.getPath(); if (!path.startsWith("/")) { path = (new Path(System.getProperty("test.tmp.dir"), path)).toUri().getPath(); throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(), e); return uri.toString(); } else { Path path = new Path(location); if (path.isAbsolute()) { return location; return path.getFileSystem(conf).makeQualified(path).toString();
public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) { for (Map.Entry<Path, HbckInfo> entry: regions.entrySet()) { TableName tableName = entry.getValue().getTableName(); Path path = entry.getKey(); errors.print("This sidelined region dir should be bulk loaded: " + path.toString()); errors.print("Bulk load command looks like: " + "hbase org.apache.hadoop.hbase.tool.LoadIncrementalHFiles " + path.toUri().getPath() + " "+ tableName); } }
/** * Uploads a local file to HDFS * This method is not thread safe * * @param source * @param conf * @return * @throws IOException */ public static URI uploadToHDFS(URI source, HiveConf conf) throws IOException { Path localFile = new Path(source.getPath()); Path remoteFile = new Path(SessionState.get().getSparkSession().getHDFSSessionDir(), getFileName(source)); FileSystem fileSystem = FileSystem.get(remoteFile.toUri(), conf); // Overwrite if the remote file already exists. Whether the file can be added // on executor is up to spark, i.e. spark.files.overwrite fileSystem.copyFromLocalFile(false, true, localFile, remoteFile); Path fullPath = fileSystem.getFileStatus(remoteFile).getPath(); return fullPath.toUri(); }
@Before public void setUpBase() throws Exception { if (!peerExist(PEER_ID2)) { ReplicationPeerConfigBuilder builder = ReplicationPeerConfig.newBuilder() .setClusterKey(utility2.getClusterKey()).setSerial(isSerialPeer()); if (isSyncPeer()) { FileSystem fs2 = utility2.getTestFileSystem(); // The remote wal dir is not important as we do not use it in DA state, here we only need to // confirm that a sync peer in DA state can still replicate data to remote cluster // asynchronously. builder.setReplicateAllUserTables(false) .setTableCFsMap(ImmutableMap.of(tableName, ImmutableList.of())) .setRemoteWALDir(new Path("/RemoteWAL") .makeQualified(fs2.getUri(), fs2.getWorkingDirectory()).toUri().toString()); } hbaseAdmin.addReplicationPeer(PEER_ID2, builder.build()); } }
@BeforeClass public void setUp() throws Exception { this.localFs = FileSystem.getLocal(new Configuration()); this.localFs.mkdirs(this.outputTaskStateDir); this.taskStateStore = new FsStateStore<>(this.localFs, this.outputTaskStateDir.toUri().getPath(), TaskState.class); this.taskStateCollectorService = new TaskStateCollectorService(new Properties(), this.jobState, this.eventBus, this.taskStateStore, new Path(this.outputTaskStateDir, JOB_ID)); this.eventBus.register(this); }
/** * Shamelessly cloned from GenericOptionsParser. */ public static String realFile(String newFile, Configuration conf) throws IOException { Path path = new Path(newFile); URI pathURI = path.toUri(); FileSystem fs; if (pathURI.getScheme() == null) { fs = FileSystem.getLocal(conf); } else { fs = path.getFileSystem(conf); } if (!fs.exists(path)) { return null; } String file = path.makeQualified(fs).toString(); return file; }
public int processFile(String fileName) throws IOException { Path path = new Path(fileName); FileSystem fs; if (!path.toUri().isAbsolute()) { fs = FileSystem.getLocal(conf); path = fs.makeQualified(path); } else { fs = FileSystem.get(path.toUri(), conf); } BufferedReader bufferReader = null; int rc = 0; try { bufferReader = new BufferedReader(new InputStreamReader(fs.open(path))); rc = processReader(bufferReader); } finally { IOUtils.closeStream(bufferReader); } return rc; }
/** * * @param entity, name of the entity to be changed, e.g. hive table or partition * @param sd, StorageDescriptor of the entity */ public static void updateAvroSchemaURL(String entity, StorageDescriptor sd, HiveCopyEntityHelper hiveHelper) { String oldAvroSchemaURL = sd.getSerdeInfo().getParameters().get(HIVE_TABLE_AVRO_SCHEMA_URL); if (oldAvroSchemaURL != null) { Path oldAvroSchemaPath = new Path(oldAvroSchemaURL); URI sourceFileSystemURI = hiveHelper.getDataset().getFs().getUri(); if (PathUtils.isAbsoluteAndSchemeAuthorityNull(oldAvroSchemaPath) || (oldAvroSchemaPath.toUri().getScheme().equals(sourceFileSystemURI.getScheme()) && oldAvroSchemaPath.toUri().getAuthority().equals(sourceFileSystemURI.getAuthority()))) { String newAvroSchemaURL = hiveHelper.getTargetPathHelper().getTargetPath(oldAvroSchemaPath, hiveHelper.getTargetFileSystem(), Optional.<Partition>absent(), true).toString(); sd.getSerdeInfo().getParameters().put(HIVE_TABLE_AVRO_SCHEMA_URL, newAvroSchemaURL); log.info(String.format("For entity %s, change %s from %s to %s", entity, HIVE_TABLE_AVRO_SCHEMA_URL, oldAvroSchemaURL, newAvroSchemaURL)); } } }
@VisibleForTesting static void serializeJobState(FileSystem fs, Path mrJobDir, Configuration conf, JobState jobState, Job job) throws IOException { Path jobStateFilePath = new Path(mrJobDir, JOB_STATE_FILE_NAME); // Write the job state with an empty task set (work units are read by the mapper from a different file) try (DataOutputStream dataOutputStream = new DataOutputStream(fs.create(jobStateFilePath))) { jobState.write(dataOutputStream, false, conf.getBoolean(SERIALIZE_PREVIOUS_WORKUNIT_STATES_KEY, DEFAULT_SERIALIZE_PREVIOUS_WORKUNIT_STATES)); } job.getConfiguration().set(ConfigurationKeys.JOB_STATE_FILE_PATH_KEY, jobStateFilePath.toString()); DistributedCache.addCacheFile(jobStateFilePath.toUri(), job.getConfiguration()); job.getConfiguration().set(ConfigurationKeys.JOB_STATE_DISTRIBUTED_CACHE_NAME, jobStateFilePath.getName()); }
private void addGoodFilesToOutputPath (List<Path> goodPaths) throws IOException { for (Path path: goodPaths) { String fileName = path.getName(); LOG.info(String.format("Adding %s to %s", path.toString(), this.dataset.outputPath())); Path outPath = MRCompactorJobRunner.this.lateOutputRecordCountProvider.constructLateFilePath(fileName, MRCompactorJobRunner.this.fs, this.dataset.outputPath()); HadoopUtils.movePath(MRCompactorJobRunner.this.tmpFs, path, FileSystem.get(this.dataset.outputPath().getParent().toUri(), this.fs.getConf()), outPath, false, this.fs.getConf()); } }