private String execute(String uri) throws IOException { String url = getNominatimUrl() + uri; Get get = new Get(url); String result = get.executeAsString(); if (get.isSuccessful()) return result; return null; }
private InputStream openStream(URL url) throws IOException { String urlString = url.toExternalForm(); // make sure HTTPS requests use HTTP Client with it's SSL tweaks if (urlString.contains("https://")) { Get get = new Get(urlString); return get.executeAsStream(); } return url.openStream(); }
public void setRange(long startIndex, Long endIndex) { // avoid GZIP'ed range disableContentCompression(); // Apache accepts just bytes=1234-1235 while the spec says bytes 1234-1235/1236 setHeader(RANGE, "bytes=" + startIndex + "-" + (endIndex != null ? endIndex : "")); } }
private Get get(String url) { Get get = new Get(url); get.setUserAgent(USER_AGENT); return get; }
public void run() throws IOException { Get request = new Get(getDownload().getUrl()); request.setRange(0L, RANGE_END_INDEX); if (getDownload().getETag() != null) request.setIfNoneMatch(getDownload().getETag()); InputStream inputStream = request.executeAsStream(); log.info(format("GET 0-%d for %s returned with status code %s and content length %d", RANGE_END_INDEX, getDownload().getUrl(), request.getStatusCode(), request.getContentLength())); if (request.isPartialContent()) { writePartialFile(inputStream, getDownload().getFile().getExpectedChecksum().getContentLength(), getDownload().getFile().getFile()); closeQuietly(inputStream); } else if (request.isOk()){ // HTTP Range not supported copyAndClose(inputStream, new FileOutputStream(getDownload().getFile().getFile())); setLastModified(getDownload().getFile().getFile(), request.getLastModified()); } request.release(); if (request.isNotModified()) { downloadExecutor.notModified(); } else if (request.isSuccessful()) { getDownload().setETag(request.getETag()); getDownload().getFile().setActualChecksum(extractChecksum(request)); downloadExecutor.succeeded(); } else downloadExecutor.downloadFailed(); }
private Result download() throws IOException { downloadExecutor.updateState(Downloading); Long contentLength = getDownload().getFile().getExpectedChecksum() != null ? getDownload().getFile().getExpectedChecksum().getContentLength() : null; log.info(format("Downloading %d bytes from %s with ETag %s", contentLength, getDownload().getUrl(), getDownload().getETag())); Get get = new Get(getDownload().getUrl()); get.setSocketTimeout(SOCKET_TIMEOUT); if (new Validator(getDownload()).isExistsTargets() && getDownload().getETag() != null) get.setIfNoneMatch(getDownload().getETag()); InputStream inputStream = get.executeAsStream(); log.info(format("Download from %s returned with status code %s and content length %d", getDownload().getUrl(), get.getStatusCode(), get.getContentLength())); if (get.isSuccessful() && inputStream != null) { if(contentLength == null) contentLength = get.getContentLength(); if (contentLength != null) getModelUpdater().expectingBytes(contentLength); new Copier(getModelUpdater()).copyAndClose(inputStream, new FileOutputStream(getDownload().getTempFile()), 0, contentLength); getDownload().setETag(get.getETag()); return new Result(true, get.getLastModified()); } return new Result(get.isSuccessful(), get.isNotModified()); }
public Double getElevationFor(double longitude, double latitude) throws IOException { String url = getElevationUrl("locations=" + latitude + "," + longitude); // could be up to 512 locations Get get = get(url); log.info("Getting elevation for " + longitude + "," + latitude); String result = get.executeAsString(); if (get.isSuccessful()) try { ElevationResponse elevationResponse = unmarshalElevation(result); if (elevationResponse != null) { String status = elevationResponse.getStatus(); checkForError(url, status); List<Double> elevations = extractElevations(elevationResponse.getResult()); return elevations != null && elevations.size() > 0 ? elevations.get(0) : null; } } catch (JAXBException e) { throw new IOException("Cannot unmarshall " + result + ": " + e, e); } return null; }
private void recursiveCollect(String uri, Set<String> uris, Set<String> visitedUris) throws IOException { if (visitedUris.contains(uri)) return; visitedUris.add(uri); log.info(format("Downloading %s", getUrl() + uri)); Get get = new Get(getUrl() + uri); String result = get.executeAsString(); List<String> anchors = new AnchorParser().parseAnchors(result.replaceAll("<area", "<a")); List<String> included = new AnchorFilter().filterAnchors(baseUrl, anchors, extensions, includes, excludes); for (String anchor : included) { // create the anchor relative to the current uri String nextUri = appendURIs(uri, anchor); uris.add(nextUri); } List<String> recurse = new AnchorFilter().filterAnchors(baseUrl, anchors, new HashSet<>(asList(".html", "/")), null, null); for (String anchor : recurse) { if((getUrl() + anchor).equals(baseUrl) || baseUrl.endsWith(anchor)) continue; // create the anchor relative to the current uri String nextUri = appendURIs(uri, anchor); recursiveCollect(nextUri, uris, visitedUris); } }
private Result resume() throws IOException { downloadExecutor.updateState(Resuming); long fileSize = getDownload().getTempFile().length(); Long contentLength = getDownload().getFile().getExpectedChecksum() != null ? getDownload().getFile().getExpectedChecksum().getContentLength() : null; log.info(format("Resuming bytes %d-%d from %s", fileSize, contentLength, getDownload().getUrl())); Get get = new Get(getDownload().getUrl()); get.setRange(fileSize, contentLength); InputStream inputStream = get.executeAsStream(); log.info(format("Resume from %s returned with status code %s", getDownload().getUrl(), get.getStatusCode())); if (get.isPartialContent()) { getModelUpdater().expectingBytes(contentLength != null ? contentLength : get.getContentLength() != null ? get.getContentLength() : 0); new Copier(getModelUpdater()).copyAndClose(inputStream, new FileOutputStream(getDownload().getTempFile(), true), fileSize, contentLength); return new Result(true); } return new Result(false); }
public List<NavigationPosition> getPositionsFor(String address) throws IOException { String url = getGeocodingUrl("address=" + encodeUri(address)); Get get = get(url); log.info("Getting positions for " + address); String result = get.executeAsString(); if (get.isSuccessful()) try { GeocodeResponse geocodeResponse = unmarshalGeocode(result); if (geocodeResponse != null) { String status = geocodeResponse.getStatus(); checkForError(url, status); return extractAdresses(geocodeResponse.getResult()); } } catch (JAXBException e) { throw new IOException("Cannot unmarshall " + result + ": " + e, e); } return null; }
private String execute(String uri) throws IOException { String url = getPhotonUrl() + uri; Get get = new Get(url); String result = get.executeAsString(); if (get.isSuccessful()) return result; return null; }
public String getAddressFor(NavigationPosition position) throws IOException { String url = getGeocodingUrl("latlng=" + position.getLatitude() + "," + position.getLongitude()); Get get = get(url); log.info("Getting location for " + position.getLongitude() + "," + position.getLatitude()); String result = get.executeAsString(); if (get.isSuccessful()) try { GeocodeResponse geocodeResponse = unmarshalGeocode(result); if (geocodeResponse != null) { String status = geocodeResponse.getStatus(); checkForError(url, status); return extractClosestLocation(geocodeResponse.getResult(), position.getLongitude(), position.getLatitude()); } } catch (JAXBException e) { throw new IOException("Cannot unmarshall " + result + ": " + e, e); } return null; }
CatalogType fetch(String url) throws IOException { long start = System.currentTimeMillis(); String urlWithXml = url + FORMAT_XML; try { Get get = new Get(urlWithXml); String result = get.executeAsString(); if (get.isSuccessful()) try { return unmarshal(result); } catch (JAXBException e) { throw new IOException("Cannot unmarshall " + result + ": " + e, e); } } finally { long end = System.currentTimeMillis(); log.info("Fetching from " + urlWithXml + " took " + (end - start) + " milliseconds"); } return null; }
private String execute(String uri, String apiType) throws IOException { String userName = trim(APIKeyRegistry.getInstance().getAPIKey("geonames", apiType)); if(userName == null) return null; String url = getGeoNamesApiUrl() + uri + "&username=" + userName; Get get = new Get(url); String result = get.executeAsString(); if (get.isSuccessful()) { checkCurrentlyOverloaded(url, result); return result; } return null; }