@Override public FetchedData apply(Resource resource) { try { return this.fetch(resource); } catch (Exception e) { int statusCode = DEFAULT_ERROR_CODE; if (e instanceof FileNotFoundException){ statusCode = 404; } LOG.warn("FETCH-ERROR {}", resource.getUrl()); LOG.debug(e.getMessage(), e); FetchedData fetchedData = new FetchedData(new byte[0], "", statusCode); resource.setStatus(ResourceStatus.ERROR.toString()); fetchedData.setResource(resource); return fetchedData; } } }
byte[] rawData = bufferOutStream.toByteArray(); IOUtils.closeQuietly(bufferOutStream); FetchedData fetchedData = new FetchedData(rawData, urlConn.getContentType(), responseCode); resource.setStatus(ResourceStatus.FETCHED.toString()); fetchedData.setResource(resource);
return super.fetch(resource); fetchedData = new FetchedData(html.getBytes(), "application/html", status); resource.setStatus(ResourceStatus.FETCHED.toString()); fetchedData.setResource(resource);
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { IOUtils.copy(boundedStream, out); fetchedData = new FetchedData(out.toByteArray(), response.getContentType(), response.getStatusCode()); long contentLength = page.getWebResponse().getContentLength(); if (contentLength > 0 && contentLength < Integer.MAX_VALUE) { fetchedData = new FetchedData(new byte[0], "unknown/unknown", 0); // fixme: use proper status code resource.setStatus(ResourceStatus.ERROR.toString());