private File unzipFile(File cachedFile) throws IOException { String filename = cachedFile.getName(); File destDir = new File(cachedFile.getParentFile(), filename + "_unzip"); File lockFile = new File(cachedFile.getParentFile(), filename + "_unzip.lock"); if (!destDir.exists()) { FileOutputStream out = new FileOutputStream(lockFile); try { java.nio.channels.FileLock lock = out.getChannel().lock(); try { // Recheck in case of concurrent processes if (!destDir.exists()) { File tempDir = pluginFiles.createTempDir(); ZipUtils.unzip(cachedFile, tempDir, newLibFilter()); FileUtils.moveDirectory(tempDir, destDir); } } finally { lock.release(); } } finally { out.close(); deleteQuietly(lockFile); } } return destDir; } }
private void enqueueNotFoundDownload(String pluginKey, String pluginHash) { doReturn(Optional.empty()).when(pluginFiles).get(argThat(p -> pluginKey.equals(p.key) && pluginHash.equals(p.hash))); } }
/** * Get the JAR file of specified plugin. If not present in user local cache, * then it's downloaded from server and added to cache. * * @return the file, or {@link Optional#empty()} if plugin not found (404 HTTP code) * @throws IllegalStateException if the plugin can't be downloaded (not 404 nor 2xx HTTP codes) * or can't be cached locally. */ public Optional<File> get(InstalledPlugin plugin) { // Does not fail if another process tries to create the directory at the same time. File jarInCache = jarInCache(plugin.key, plugin.hash); if (jarInCache.exists() && jarInCache.isFile()) { return Optional.of(jarInCache); } return download(plugin); }
public PluginFiles(ScannerWsClient wsClient, Configuration configuration) { this.wsClient = wsClient; File home = locateHomeDir(configuration); this.cacheDir = mkdir(new File(home, "cache"), "user cache"); this.tempDir = mkdir(new File(home, "_tmp"), "temp dir"); LOGGER.info("User cache: {}", cacheDir.getAbsolutePath()); }
private File unpack200(String pluginKey, File compressedFile) { LOGGER.debug("Unpacking plugin {}", pluginKey); File jar = newTempFile(); try (InputStream input = new GZIPInputStream(new BufferedInputStream(FileUtils.openInputStream(compressedFile))); JarOutputStream output = new JarOutputStream(new BufferedOutputStream(FileUtils.openOutputStream(jar)))) { Pack200.newUnpacker().unpack(input, output); } catch (IOException e) { throw new IllegalStateException(format("Fail to download plugin [%s]. Pack200 error.", pluginKey), e); } return jar; }
@Before public void setUp() throws Exception { HttpConnector connector = HttpConnector.newBuilder().url(server.url("/").toString()).build(); GlobalAnalysisMode analysisMode = new GlobalAnalysisMode(new ScannerProperties(Collections.emptyMap())); ScannerWsClient wsClient = new ScannerWsClient(WsClientFactories.getDefault().newClient(connector), false, analysisMode); userHome = temp.newFolder(); MapSettings settings = new MapSettings(); settings.setProperty("sonar.userHome", userHome.getAbsolutePath()); underTest = new PluginFiles(wsClient, settings.asConfig()); }
public PluginFiles(ScannerWsClient wsClient, Configuration configuration) { this.wsClient = wsClient; File home = locateHomeDir(configuration); this.cacheDir = mkdir(new File(home, "cache"), "user cache"); this.tempDir = mkdir(new File(home, "_tmp"), "temp dir"); LOGGER.info("User cache: {}", cacheDir.getAbsolutePath()); }
private File unpack200(String pluginKey, File compressedFile) { LOGGER.debug("Unpacking plugin {}", pluginKey); File jar = newTempFile(); try (InputStream input = new GZIPInputStream(new BufferedInputStream(FileUtils.openInputStream(compressedFile))); JarOutputStream output = new JarOutputStream(new BufferedOutputStream(FileUtils.openOutputStream(jar)))) { Pack200.newUnpacker().unpack(input, output); } catch (IOException e) { throw new IllegalStateException(format("Fail to download plugin [%s]. Pack200 error.", pluginKey), e); } return jar; }
private Loaded loadPlugins(Map<String, ScannerPlugin> result) { for (InstalledPlugin plugin : listInstalledPlugins()) { if (pluginPredicate.apply(plugin.key)) { Optional<File> jarFile = pluginFiles.get(plugin); if (!jarFile.isPresent()) { return new Loaded(false, plugin.key); } PluginInfo info = PluginInfo.create(jarFile.get()); result.put(info.getKey(), new ScannerPlugin(plugin.key, plugin.updatedAt, info)); } } return new Loaded(true, null); }
/** * Get the JAR file of specified plugin. If not present in user local cache, * then it's downloaded from server and added to cache. * * @return the file, or {@link Optional#empty()} if plugin not found (404 HTTP code) * @throws IllegalStateException if the plugin can't be downloaded (not 404 nor 2xx HTTP codes) * or can't be cached locally. */ public Optional<File> get(InstalledPlugin plugin) { // Does not fail if another process tries to create the directory at the same time. File jarInCache = jarInCache(plugin.key, plugin.hash); if (jarInCache.exists() && jarInCache.isFile()) { return Optional.of(jarInCache); } return download(plugin); }
@Before public void setUp() throws IOException { tempDir = temp.newFolder(); PluginFiles pluginFiles = mock(PluginFiles.class); when(pluginFiles.createTempDir()).thenReturn(tempDir); underTest = new ScannerPluginJarExploder(pluginFiles); }
private void enqueueDownload(String pluginKey, String pluginHash) throws IOException { File jar = temp.newFile(); Manifest manifest = new Manifest(); manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); manifest.getMainAttributes().putValue("Plugin-Key", pluginKey); try (JarOutputStream output = new JarOutputStream(FileUtils.openOutputStream(jar), manifest)) { } doReturn(Optional.of(jar)).when(pluginFiles).get(argThat(p -> pluginKey.equals(p.key) && pluginHash.equals(p.hash))); }
private File unzipFile(File cachedFile) throws IOException { String filename = cachedFile.getName(); File destDir = new File(cachedFile.getParentFile(), filename + "_unzip"); File lockFile = new File(cachedFile.getParentFile(), filename + "_unzip.lock"); if (!destDir.exists()) { FileOutputStream out = new FileOutputStream(lockFile); try { java.nio.channels.FileLock lock = out.getChannel().lock(); try { // Recheck in case of concurrent processes if (!destDir.exists()) { File tempDir = pluginFiles.createTempDir(); ZipUtils.unzip(cachedFile, tempDir, newLibFilter()); FileUtils.moveDirectory(tempDir, destDir); } } finally { lock.release(); } } finally { out.close(); deleteQuietly(lockFile); } } return destDir; } }
@Test public void filter_blacklisted_plugins() throws IOException { WsTestUtil.mockReader(wsClient, "api/plugins/installed", new InputStreamReader(getClass().getResourceAsStream("ScannerPluginInstallerTest/installed-plugins-ws.json"))); enqueueDownload("scmgit", "abc"); enqueueDownload("java", "def"); when(pluginPredicate.apply("scmgit")).thenReturn(true); when(pluginPredicate.apply("java")).thenReturn(false); Map<String, ScannerPlugin> result = underTest.installRemotes(); assertThat(result.keySet()).containsExactlyInAnyOrder("scmgit"); verify(pluginFiles, times(1)).get(any()); }
@Test public void get_jar_from_cache_if_present() throws Exception { FileAndMd5 jar = createFileInCache("foo"); File result = underTest.get(newInstalledPlugin("foo", jar.md5)).get(); verifySameContent(result, jar); // no requests to server assertThat(server.getRequestCount()).isEqualTo(0); }
@Test public void download_and_add_jar_to_cache_if_missing() throws Exception { FileAndMd5 tempJar = new FileAndMd5(); enqueueDownload(tempJar); InstalledPlugin plugin = newInstalledPlugin("foo", tempJar.md5); File result = underTest.get(plugin).get(); verifySameContent(result, tempJar); HttpUrl requestedUrl = server.takeRequest().getRequestUrl(); assertThat(requestedUrl.encodedPath()).isEqualTo("/api/plugins/download"); assertThat(requestedUrl.encodedQuery()).isEqualTo("plugin=foo&acceptCompressions=pack200"); // get from cache on second call result = underTest.get(plugin).get(); verifySameContent(result, tempJar); assertThat(server.getRequestCount()).isEqualTo(1); }
@Test public void download_a_new_version_of_plugin_during_blue_green_switch() throws IOException { FileAndMd5 tempJar = new FileAndMd5(); enqueueDownload(tempJar); // expecting to download plugin foo with checksum "abc" InstalledPlugin pluginV1 = newInstalledPlugin("foo", "abc"); File result = underTest.get(pluginV1).get(); verifySameContent(result, tempJar); // new version of downloaded jar is put in cache with the new md5 InstalledPlugin pluginV2 = newInstalledPlugin("foo", tempJar.md5); result = underTest.get(pluginV2).get(); verifySameContent(result, tempJar); assertThat(server.getRequestCount()).isEqualTo(1); // v1 still requests server and downloads v2 enqueueDownload(tempJar); result = underTest.get(pluginV1).get(); verifySameContent(result, tempJar); assertThat(server.getRequestCount()).isEqualTo(2); }
@Test public void download_compressed_and_add_uncompressed_to_cache_if_missing() throws Exception { FileAndMd5 jar = new FileAndMd5(); enqueueCompressedDownload(jar, true); InstalledPlugin plugin = newInstalledPlugin("foo", jar.md5); File result = underTest.get(plugin).get(); verifySameContentAfterCompression(jar.file, result); RecordedRequest recordedRequest = server.takeRequest(); assertThat(recordedRequest.getRequestUrl().queryParameter("acceptCompressions")).isEqualTo("pack200"); // get from cache on second call result = underTest.get(plugin).get(); verifySameContentAfterCompression(jar.file, result); assertThat(server.getRequestCount()).isEqualTo(1); }
@Test public void fail_if_integrity_of_compressed_download_is_not_valid() throws Exception { FileAndMd5 jar = new FileAndMd5(); enqueueCompressedDownload(jar, false); expectISE("foo", "was expected to have checksum invalid_hash but had "); InstalledPlugin plugin = newInstalledPlugin("foo", jar.md5); underTest.get(plugin).get(); }
@Test public void return_empty_if_plugin_not_found_on_server() { server.enqueue(new MockResponse().setResponseCode(404)); InstalledPlugin plugin = newInstalledPlugin("foo", "abc"); Optional<File> result = underTest.get(plugin); assertThat(result).isEmpty(); }