public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum) { super(url, destPool, templateId, checksum); parseUrl(); }
protected void createTemporaryDirectoryAndFile(String downloadDir) { createFolder(getDestPoolPath() + File.separator + downloadDir); File f = new File(getDestPoolPath() + File.separator + downloadDir + File.separator + getFileNameFromUrl()); setDownloadedFilePath(f.getAbsolutePath()); }
@Override public DirectTemplateInformation getTemplateInformation() { String sizeResult = Script.runSimpleBashScript("ls -als " + getInstallFullPath() + " | awk '{print $1}'"); long size = Long.parseLong(sizeResult); return new DirectTemplateInformation(installPath, size, checksum); }
@Override public boolean downloadTemplate() { if (StringUtils.isBlank(getUrl())) { throw new CloudRuntimeException("Download url has not been set, aborting"); String downloadDir = getDirectDownloadTempPath(getTemplateId()); boolean downloaded = false; int i = 0; do { if (!isRedownload()) { setUrl(metalinkUrls.get(i)); s_logger.info("Trying to download template from url: " + getUrl()); try { File f = new File(getDestPoolPath() + File.separator + downloadDir + File.separator + getFileNameFromUrl()); if (f.exists()) { f.delete(); f.createNewFile(); setDownloadedFilePath(f.getAbsolutePath()); request = createRequest(getUrl(), reqHeaders); downloaded = super.downloadTemplate(); if (downloaded) { s_logger.info("Successfully downloaded template from url: " + getUrl()); s_logger.error("Error downloading template: " + getTemplateId() + " from " + getUrl() + ": " + e.getMessage()); while (!downloaded && !isRedownload() && i < metalinkUrls.size()); return downloaded;
@Override public boolean downloadTemplate() { String mountSrcUuid = UUID.randomUUID().toString(); String mount = String.format(mountCommand, srcHost + ":" + srcPath, "/mnt/" + mountSrcUuid); Script.runSimpleBashScript(mount); String downloadDir = getDestPoolPath() + File.separator + getDirectDownloadTempPath(getTemplateId()); setDownloadedFilePath(downloadDir + File.separator + getFileNameFromUrl()); Script.runSimpleBashScript("cp /mnt/" + mountSrcUuid + srcPath + " " + getDownloadedFilePath()); Script.runSimpleBashScript("umount /mnt/" + mountSrcUuid); return true; } }
if (!downloader.downloadTemplate()) { s_logger.warn("Couldn't download template"); return new DirectDownloadAnswer(false, "Unable to download template", true); if (!downloader.validateChecksum()) { s_logger.warn("Couldn't validate template checksum"); return new DirectDownloadAnswer(false, "Checksum validation failed", false); if (!downloader.extractAndInstallDownloadedTemplate()) { s_logger.warn("Couldn't extract and install template"); return new DirectDownloadAnswer(false, "Extraction and installation failed", false); DirectTemplateInformation info = downloader.getTemplateInformation(); return new DirectDownloadAnswer(true, info.getSize(), info.getInstallPath());
/** * Get direct template downloader from direct download command and destination pool */ private DirectTemplateDownloader getDirectTemplateDownloaderFromCommand(DirectDownloadCommand cmd, KVMStoragePool destPool) { if (cmd instanceof HttpDirectDownloadCommand) { return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders()); } else if (cmd instanceof HttpsDirectDownloadCommand) { return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders()); } else if (cmd instanceof NfsDirectDownloadCommand) { return new NfsDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum()); } else if (cmd instanceof MetalinkDirectDownloadCommand) { return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders()); } else { throw new IllegalArgumentException("Unsupported protocol, please provide HTTP(S), NFS or a metalink"); } }
@Override public boolean extractAndInstallDownloadedTemplate() { installPath = UUID.randomUUID().toString(); if (isTemplateExtractable()) { extractDownloadedTemplate(); } else { Script.runSimpleBashScript("mv " + downloadedFilePath + " " + getInstallFullPath()); } return true; }
@Override public boolean validateChecksum() { if (StringUtils.isBlank(getChecksum()) && CollectionUtils.isNotEmpty(metalinkChecksums)) { String chk = metalinkChecksums.get(random.nextInt(metalinkChecksums.size())); setChecksum(chk); s_logger.info("Checksum not provided but " + metalinkChecksums.size() + " found on metalink file, performing checksum using one of them: " + chk); } return super.validateChecksum(); } }
protected boolean performDownload() { s_logger.info("Downloading template " + getTemplateId() + " from " + getUrl() + " to: " + getDownloadedFilePath()); try ( InputStream in = request.getResponseBodyAsStream(); OutputStream out = new FileOutputStream(getDownloadedFilePath()); ) { IOUtils.copy(in, out); } catch (IOException e) { s_logger.error("Error downloading template " + getTemplateId() + " due to: " + e.getMessage()); return false; } return true; } }
public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers) { super(url, destPoolPath, templateId, checksum); s_httpClientManager.getParams().setConnectionTimeout(5000); s_httpClientManager.getParams().setSoTimeout(5000); client = new HttpClient(s_httpClientManager); request = createRequest(url, headers); String downloadDir = getDirectDownloadTempPath(templateId); createTemporaryDirectoryAndFile(downloadDir); }
public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers) { super(url, templateId, destPoolPath, checksum, headers); SSLContext sslcontext = null; try { sslcontext = getSSLContext(); } catch (KeyStoreException | NoSuchAlgorithmException | CertificateException | IOException | KeyManagementException e) { throw new CloudRuntimeException("Failure getting SSL context for HTTPS downloader: " + e.getMessage()); } SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); RequestConfig config = RequestConfig.custom() .setConnectTimeout(5000) .setConnectionRequestTimeout(5000) .setSocketTimeout(5000).build(); httpsClient = HttpClients.custom().setSSLSocketFactory(factory).setDefaultRequestConfig(config).build(); createUriRequest(url, headers); }
@Override public boolean validateChecksum() { if (StringUtils.isNotBlank(checksum)) { int retry = 3; boolean valid = false; try { while (!valid && retry > 0) { retry--; s_logger.info("Performing checksum validation for downloaded template " + templateId + " using " + checksum + ", retries left: " + retry); valid = DigestHelper.check(checksum, new FileInputStream(downloadedFilePath)); if (!valid && retry > 0) { s_logger.info("Checksum validation failded, re-downloading template"); redownload = true; resetDownloadFile(); downloadTemplate(); } } s_logger.info("Checksum validation for template " + templateId + ": " + (valid ? "succeeded" : "failed")); return valid; } catch (IOException e) { throw new CloudRuntimeException("could not check sum for file: " + downloadedFilePath, e); } catch (NoSuchAlgorithmException e) { throw new CloudRuntimeException("Unknown checksum algorithm: " + checksum, e); } } s_logger.info("No checksum provided, skipping checksum validation"); return true; }
/** * Return extract command to execute given downloaded file */ private String getExtractCommandForDownloadedFile() { if (downloadedFilePath.endsWith(".zip")) { return "unzip -p " + downloadedFilePath + " | cat > " + getInstallFullPath(); } else if (downloadedFilePath.endsWith(".bz2")) { return "bunzip2 -c " + downloadedFilePath + " > " + getInstallFullPath(); } else if (downloadedFilePath.endsWith(".gz")) { return "gunzip -c " + downloadedFilePath + " > " + getInstallFullPath(); } else { throw new CloudRuntimeException("Unable to extract template " + templateId + " on " + downloadedFilePath); } }
@Override public boolean downloadTemplate() { CloseableHttpResponse response; try { response = httpsClient.execute(req); } catch (IOException e) { throw new CloudRuntimeException("Error on HTTPS request: " + e.getMessage()); } return consumeResponse(response); }
/** * Delete and create download file */ private void resetDownloadFile() { File f = new File(getDownloadedFilePath()); s_logger.info("Resetting download file: " + getDownloadedFilePath() + ", in order to re-download and persist template " + templateId + " on it"); try { if (f.exists()) { f.delete(); } f.createNewFile(); } catch (IOException e) { s_logger.error("Error creating file to download on: " + getDownloadedFilePath() + " due to: " + e.getMessage()); throw new CloudRuntimeException("Failed to create download file for direct download"); } }
/** * Extract downloaded template into installPath, remove compressed file */ private void extractDownloadedTemplate() { String extractCommand = getExtractCommandForDownloadedFile(); Script.runSimpleBashScript(extractCommand); Script.runSimpleBashScript("rm -f " + downloadedFilePath); }
/** * Parse url and set srcHost and srcPath */ private void parseUrl() { URI uri = null; String url = getUrl(); try { uri = new URI(UriUtils.encodeURIComponent(url)); if (uri.getScheme() != null && uri.getScheme().equalsIgnoreCase("nfs")) { srcHost = uri.getHost(); srcPath = uri.getPath(); } } catch (URISyntaxException e) { throw new CloudRuntimeException("Invalid NFS url " + url + " caused error: " + e.getMessage()); } }
public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum, Map<String, String> headers) { super(url, templateId, destPoolPath, checksum, headers); metalinkUrl = url; metalinkUrls = UriUtils.getMetalinkUrls(metalinkUrl); metalinkChecksums = UriUtils.getMetalinkChecksums(metalinkUrl); if (CollectionUtils.isEmpty(metalinkUrls)) { throw new CloudRuntimeException("No urls found on metalink file: " + metalinkUrl + ". Not possible to download template " + templateId); } setUrl(metalinkUrls.get(0)); s_logger.info("Metalink downloader created, metalink url: " + metalinkUrl + " parsed - " + metalinkUrls.size() + " urls and " + (CollectionUtils.isNotEmpty(metalinkChecksums) ? metalinkChecksums.size() : "0") + " checksums found"); }
@Override public boolean downloadTemplate() { try { int status = client.executeMethod(request); if (status != HttpStatus.SC_OK) { s_logger.warn("Not able to download template, status code: " + status); return false; } return performDownload(); } catch (IOException e) { throw new CloudRuntimeException("Error on HTTP request: " + e.getMessage()); } finally { request.releaseConnection(); } }