@Override public Configuration createConfiguration() { return delegate.createConfiguration(); }
@Override public void configureConnectionInformation( String namenodeHost, String namenodePort, String jobtrackerHost, String jobtrackerPort, Configuration conf, List<String> logMessages ) throws Exception { delegate.configureConnectionInformation( namenodeHost, namenodePort, jobtrackerHost, jobtrackerPort, conf, logMessages ); }
@Override public FileSystem getFileSystem( Configuration conf ) throws IOException { return delegate.getFileSystem( conf ); }
@Override public HadoopFileSystem create( NamedCluster namedCluster, URI uri ) throws IOException { final URI finalUri = uri != null ? uri : URI.create( "" ); final HadoopShim hadoopShim = hadoopConfiguration.getHadoopShim(); final Configuration configuration = hadoopShim.createConfiguration(); FileSystem fileSystem = (FileSystem) hadoopShim.getFileSystem( configuration ).getDelegate(); if ( fileSystem instanceof LocalFileSystem ) { LOGGER.error( "Got a local filesystem, was expecting an hdfs connection" ); throw new IOException( "Got a local filesystem, was expecting an hdfs connection" ); } return new HadoopFileSystemImpl( new HadoopFileSystemCallable() { @Override public FileSystem getFileSystem() { try { return (FileSystem) hadoopShim.getFileSystem( finalUri, configuration, null ).getDelegate(); } catch ( IOException e ) { LOGGER.debug( "Error looking up/creating the file system ", e ); return null; } catch ( InterruptedException e ) { LOGGER.debug( "Error looking up/creating the file system ", e ); return null; } } } ); } }
name ) ) { appenderFile = appenderManager.getFile(); Configuration configuration = hadoopShim.createConfiguration(); if ( executionMode != ExecutionMode.LOCAL ) { List<String> configMessages = new ArrayList<String>(); hadoopShim.configureConnectionInformation( variableSpace.environmentSubstitute( namedCluster.getHdfsHost() ), variableSpace.environmentSubstitute( namedCluster.getHdfsPort() ), variableSpace.environmentSubstitute( namedCluster.getJobTrackerHost() ),
protected void configure( Configuration conf ) throws Exception { FileSystem fs = hadoopShim.getFileSystem( conf ); URL[] urls = new URL[] { resolvedJarUrl }; URLClassLoader loader = new URLClassLoader( urls, hadoopShim.getClass().getClassLoader() ); hadoopShim.configureConnectionInformation( hdfsHostnameS, hdfsPortS, jobTrackerHostnameS, jobTrackerPortS, conf, configMessages ); for ( String m : configMessages ) { log.logBasic( m );
@Override protected void configure( Configuration conf ) throws Exception { callVisitors(); setMapRunnerClass( hadoopShim.getPentahoMapReduceMapRunnerClass().getCanonicalName() ); conf.set( TRANSFORMATION_MAP_XML, mapperTransformationXml ); conf.set( TRANSFORMATION_MAP_INPUT_STEPNAME, mapperInputStep ); conf.set( TRANSFORMATION_MAP_OUTPUT_STEPNAME, mapperOutputStep ); if ( combinerTransformationXml != null ) { conf.set( TRANSFORMATION_COMBINER_XML, combinerTransformationXml ); conf.set( TRANSFORMATION_COMBINER_INPUT_STEPNAME, combinerInputStep ); conf.set( TRANSFORMATION_COMBINER_OUTPUT_STEPNAME, combinerOutputStep ); setCombinerClass( hadoopShim.getPentahoMapReduceCombinerClass().getCanonicalName() ); } if ( reducerTransformationXml != null ) { conf.set( TRANSFORMATION_REDUCE_XML, reducerTransformationXml ); conf.set( TRANSFORMATION_REDUCE_INPUT_STEPNAME, reducerInputStep ); conf.set( TRANSFORMATION_REDUCE_OUTPUT_STEPNAME, reducerOutputStep ); setReducerClass( hadoopShim.getPentahoMapReduceReducerClass().getCanonicalName() ); } conf.setJarByClass( hadoopShim.getPentahoMapReduceMapRunnerClass() ); conf.set( LOG_LEVEL, logLevel.toString() ); configureVariableSpace( conf ); super.configure( conf ); }
cleanOutputPath( conf ); FileSystem fs = hadoopShim.getFileSystem( conf ); if ( hadoopShim.getDistributedCacheUtil().isKettleEnvironmentInstalledAt( fs, kettleEnvInstallDir ) ) { log.logDetailed( BaseMessages.getString( PKG, "JobEntryHadoopTransJobExecutor.UsingKettleInstallationFrom", kettleEnvInstallDir.toUri().getPath() ) ); hadoopShim.getDistributedCacheUtil() .installKettleEnvironment( pmrLibArchive, fs, kettleEnvInstallDir, bigDataPluginFolder, additionalPluginNames ); if ( !hadoopShim.getDistributedCacheUtil().isKettleEnvironmentInstalledAt( fs, kettleEnvInstallDir ) ) { throw new KettleException( BaseMessages.getString( PKG, JOB_ENTRY_HADOOP_TRANS_JOB_EXECUTOR_KETTLE_INSTALLATION_MISSING_FROM, conf.set( MAPREDUCE_APPLICATION_CLASSPATH, CLASSES + mapreduceClasspath ); hadoopShim.getDistributedCacheUtil().configureWithKettleEnvironment( conf, fs, kettleEnvInstallDir ); log.logBasic( MAPREDUCE_APPLICATION_CLASSPATH + ": " + conf.get( MAPREDUCE_APPLICATION_CLASSPATH ) ); } catch ( Exception ex ) {
@Test public void testProcess() throws Exception { HadoopShim hadoopShim = new MockHadoopShim(); HadoopConfiguration c = spy( new HadoopConfiguration( VFS.getManager().resolveFile( "ram:///" ), "id", "name", hadoopShim ) ); Configuration conf = mock( Configuration.class ); c.getHadoopShim().submitJob( conf ); verify( c ).process( conf ); assertEquals( hadoopShim.getHadoopVersion(), c.getHadoopShim().getHadoopVersion() ); } }
@Override public RunningJob submitJob( Configuration c ) throws IOException { return delegate.submitJob( c ); }
@SuppressWarnings( "deprecation" ) @Override public Class<?> getHadoopWritableCompatibleClass( ValueMetaInterface kettleType ) { return delegate.getHadoopWritableCompatibleClass( kettleType ); }
@Override public String getHadoopVersion() { return delegate.getHadoopVersion(); }
@Override public DistributedCacheUtil getDistributedCacheUtil() throws ConfigurationException { return delegate.getDistributedCacheUtil(); }
@Override public Driver getJdbcDriver( String driverType ) { return delegate.getJdbcDriver( driverType ); }
@SuppressWarnings( "deprecation" ) @Override public Class<?> getPentahoMapReduceCombinerClass() { return delegate.getPentahoMapReduceCombinerClass(); }
protected MapReduceJobAdvanced submit( Configuration conf ) throws IOException { return new RunningJobMapReduceJobAdvancedImpl( hadoopShim.submitJob( conf ) ); }
@Override public String getHadoopWritableCompatibleClassName( ValueMetaInterface valueMetaInterface ) { Class<?> hadoopWritableCompatibleClass = hadoopShim.getHadoopWritableCompatibleClass( valueMetaInterface ); if ( hadoopWritableCompatibleClass == null ) { return null; } return hadoopWritableCompatibleClass.getCanonicalName(); }
@Test public void getHadoopShim() throws Exception { HadoopShim hadoopShim = new MockHadoopShim(); HadoopConfiguration c = new HadoopConfiguration( VFS.getManager().resolveFile( "ram:///" ), "id", "name", hadoopShim ); assertEquals( hadoopShim.getHadoopVersion(), c.getHadoopShim().getHadoopVersion() ); }
protected void stageMetaStoreForHadoop( Configuration conf, FileSystem fs, String installPath ) throws Exception { java.nio.file.Path localMetaStoreSnapshotDirPath; Path hdfsMetaStoreDirForCurrentJobPath; FileObject localMetaStoreSnapshotDirObject; localMetaStoreSnapshotDirPath = Files.createTempDirectory( XmlUtil.META_FOLDER_NAME ); localMetaStoreSnapshotDirObject = KettleVFS.getFileObject( localMetaStoreSnapshotDirPath.toString() ); hdfsMetaStoreDirForCurrentJobPath = fs.asPath( installPath, XmlUtil.META_FOLDER_NAME ); //fill local metastore snapshot by the existing named cluster snapshotMetaStore( localMetaStoreSnapshotDirPath.toString() ); hadoopShim.getDistributedCacheUtil().stageForCache( localMetaStoreSnapshotDirObject, fs, hdfsMetaStoreDirForCurrentJobPath, true, true ); hadoopShim.getDistributedCacheUtil().addCachedFiles( conf, fs, hdfsMetaStoreDirForCurrentJobPath, null ); }
Driver jdbcDriver; try { jdbcDriver = hadoopConfiguration.getHadoopShim().getJdbcDriver( pair.getKey() ); } catch ( Throwable e ) { return null;