private void setupFileSystem(HadoopShims shims) throws IOException { if (fsType == FsType.local) { fs = FileSystem.getLocal(conf); } else if (fsType == FsType.hdfs || fsType == FsType.encrypted_hdfs) { int numDataNodes = 4; if (fsType == FsType.encrypted_hdfs) { // Set the security key provider so that the MiniDFS cluster is initialized // with encryption conf.set(SECURITY_KEY_PROVIDER_URI_NAME, getKeyProviderURI()); conf.setInt("fs.trash.interval", 50); dfs = shims.getMiniDfs(conf, numDataNodes, true, null); fs = dfs.getFileSystem(); // set up the java key provider for encrypted hdfs cluster hes = shims.createHdfsEncryptionShim(fs, conf); LOG.info("key provider is initialized"); } else { dfs = shims.getMiniDfs(conf, numDataNodes, true, null); fs = dfs.getFileSystem(); } } else { throw new IllegalArgumentException("Unknown or unhandled fsType [" + fsType + "]"); } }
@Override public void setupConfiguration(Configuration conf) { Configuration config = mr.getConfig(); for (Map.Entry<String, String> pair : config) { conf.set(pair.getKey(), pair.getValue()); } Path jarPath = new Path("hdfs:///user/hive"); Path hdfsPath = new Path("hdfs:///user/"); try { FileSystem fs = cluster.getFileSystem(); jarPath = fs.makeQualified(jarPath); conf.set("hive.jar.directory", jarPath.toString()); fs.mkdirs(jarPath); hdfsPath = fs.makeQualified(hdfsPath); conf.set("hive.user.install.directory", hdfsPath.toString()); fs.mkdirs(hdfsPath); } catch (Exception e) { e.printStackTrace(); } } }
@Override public void setupConfiguration(Configuration conf) { Configuration config = mr.getConfig(); for (Map.Entry<String, String> pair: config) { conf.set(pair.getKey(), pair.getValue()); } Path jarPath = new Path("hdfs:///user/hive"); Path hdfsPath = new Path("hdfs:///user/"); try { FileSystem fs = cluster.getFileSystem(); jarPath = fs.makeQualified(jarPath); conf.set("hive.jar.directory", jarPath.toString()); fs.mkdirs(jarPath); hdfsPath = fs.makeQualified(hdfsPath); conf.set("hive.user.install.directory", hdfsPath.toString()); fs.mkdirs(hdfsPath); } catch (Exception e) { e.printStackTrace(); } } }
@Override public void setupConfiguration(Configuration conf) { Configuration config = mr.getConfig(); for (Map.Entry<String, String> pair : config) { conf.set(pair.getKey(), pair.getValue()); } Path jarPath = new Path("hdfs:///user/hive"); Path hdfsPath = new Path("hdfs:///user/"); try { FileSystem fs = cluster.getFileSystem(); jarPath = fs.makeQualified(jarPath); conf.set("hive.jar.directory", jarPath.toString()); fs.mkdirs(jarPath); hdfsPath = fs.makeQualified(hdfsPath); conf.set("hive.user.install.directory", hdfsPath.toString()); fs.mkdirs(hdfsPath); } catch (Exception e) { e.printStackTrace(); } } }
@Override public void setupConfiguration(Configuration conf) { Configuration config = mr.getConfig(); for (Map.Entry<String, String> pair: config) { conf.set(pair.getKey(), pair.getValue()); } Path jarPath = new Path("hdfs:///user/hive"); Path hdfsPath = new Path("hdfs:///user/"); try { FileSystem fs = cluster.getFileSystem(); jarPath = fs.makeQualified(jarPath); conf.set("hive.jar.directory", jarPath.toString()); fs.mkdirs(jarPath); hdfsPath = fs.makeQualified(hdfsPath); conf.set("hive.user.install.directory", hdfsPath.toString()); fs.mkdirs(hdfsPath); } catch (Exception e) { e.printStackTrace(); } } }
@Override public void setupConfiguration(Configuration conf) { Configuration config = mr.getConfig(); for (Map.Entry<String, String> pair : config) { conf.set(pair.getKey(), pair.getValue()); } Path jarPath = new Path("hdfs:///user/hive"); Path hdfsPath = new Path("hdfs:///user/"); try { FileSystem fs = cluster.getFileSystem(); jarPath = fs.makeQualified(jarPath); conf.set("hive.jar.directory", jarPath.toString()); fs.mkdirs(jarPath); hdfsPath = fs.makeQualified(hdfsPath); conf.set("hive.user.install.directory", hdfsPath.toString()); fs.mkdirs(hdfsPath); } catch (Exception e) { e.printStackTrace(); } } }
@Override public void setupConfiguration(Configuration conf) { Configuration config = mr.getConfig(); for (Map.Entry<String, String> pair: config) { conf.set(pair.getKey(), pair.getValue()); } Path jarPath = new Path("hdfs:///user/hive"); Path hdfsPath = new Path("hdfs:///user/"); try { FileSystem fs = cluster.getFileSystem(); jarPath = fs.makeQualified(jarPath); conf.set("hive.jar.directory", jarPath.toString()); fs.mkdirs(jarPath); hdfsPath = fs.makeQualified(hdfsPath); conf.set("hive.user.install.directory", hdfsPath.toString()); fs.mkdirs(hdfsPath); } catch (Exception e) { e.printStackTrace(); } } }