public void refreshLocalResourcesFromConf(HiveConf conf) throws IOException, LoginException, IllegalArgumentException, URISyntaxException, TezException { String dir = tezScratchDir.toString(); localizedResources.clear(); // these are local resources set through add file, jar, etc List<LocalResource> lrs = utils.localizeTempFilesFromConf(dir, conf); if (lrs != null) { localizedResources.addAll(lrs); } // these are local resources that are set through the mr "tmpjars" property List<LocalResource> handlerLr = utils.localizeTempFiles(dir, conf, additionalFilesNotFromConf.toArray(new String[additionalFilesNotFromConf.size()])); if (handlerLr != null) { localizedResources.addAll(handlerLr); } }
/** * Converted the list of jars into local resources */ Map<String,LocalResource> getExtraLocalResources(JobConf jobConf, Path scratchDir, String[] inputOutputJars) throws Exception { final Map<String,LocalResource> resources = new HashMap<String,LocalResource>(); final List<LocalResource> localResources = utils.localizeTempFiles( scratchDir.toString(), jobConf, inputOutputJars); if (null != localResources) { for (LocalResource lr : localResources) { resources.put(utils.getBaseName(lr), lr); } } return resources; }
newResources = utils.localizeTempFiles(dir, conf, newFilesNotFromConf, skipFilesFromConf); if (newResources != null) { resources.localizedResources.addAll(newResources);
@Test public void testExistingSessionGetsStorageHandlerResources() throws Exception { final String[] inputOutputJars = new String[] {"file:///tmp/foo.jar"}; LocalResource res = createResource(inputOutputJars[0]); final List<LocalResource> resources = Collections.singletonList(res); when(utils.localizeTempFiles(anyString(), any(Configuration.class), eq(inputOutputJars), any(String[].class))).thenReturn(resources); when(sessionState.isOpen()).thenReturn(true); when(sessionState.isOpening()).thenReturn(false); task.ensureSessionHasResources(sessionState, inputOutputJars); // TODO: ideally we should have a test for session itself. verify(sessionState).ensureLocalResources(any(Configuration.class), eq(inputOutputJars)); }
public void refreshLocalResourcesFromConf(HiveConf conf) throws IOException, LoginException, IllegalArgumentException, URISyntaxException, TezException { String dir = tezScratchDir.toString(); localizedResources.clear(); // these are local resources set through add file, jar, etc List<LocalResource> lrs = utils.localizeTempFilesFromConf(dir, conf); if (lrs != null) { localizedResources.addAll(lrs); } // these are local resources that are set through the mr "tmpjars" property List<LocalResource> handlerLr = utils.localizeTempFiles(dir, conf, additionalFilesNotFromConf.toArray(new String[additionalFilesNotFromConf.size()])); if (handlerLr != null) { localizedResources.addAll(handlerLr); } }
/** * Converted the list of jars into local resources */ Map<String,LocalResource> getExtraLocalResources(JobConf jobConf, Path scratchDir, String[] inputOutputJars) throws Exception { final Map<String,LocalResource> resources = new HashMap<String,LocalResource>(); final List<LocalResource> localResources = utils.localizeTempFiles( scratchDir.toString(), jobConf, inputOutputJars); if (null != localResources) { for (LocalResource lr : localResources) { resources.put(utils.getBaseName(lr), lr); } } return resources; }