public Resource(String url, String group, JobContext sparklerJob, Date fetchTimestamp, Integer numTries, Integer numFetches, ResourceStatus status) { this(url, group, sparklerJob, fetchTimestamp); //this.numFetches = numFetches; this.status = status.toString(); }
public Resource(String url, Integer discoverDepth, JobContext sparklerJob, ResourceStatus status) throws MalformedURLException { this(url, new URL(url).getHost(), sparklerJob); this.indexedAt = new Date(); this.id = resourceId(url, sparklerJob, this.indexedAt); this.discoverDepth = discoverDepth; this.status = status.toString(); }
public Resource(String url, Integer discoverDepth, JobContext sparklerJob, ResourceStatus status, String parent, Map<String, Double> score) throws MalformedURLException { this(url, new URL(url).getHost(), sparklerJob); this.indexedAt = new Date(); this.id = resourceId(url, sparklerJob, this.indexedAt); this.discoverDepth = discoverDepth; this.status = status.toString(); this.parent = parent; this.score = score; }
public Resource(String url, Integer discoverDepth, JobContext sparklerJob, ResourceStatus status, Date fetchTimestamp, String parent) throws MalformedURLException { this(url, new URL(url).getHost(), sparklerJob); this.id = resourceId(url, sparklerJob, fetchTimestamp); this.discoverDepth = discoverDepth; this.status = status.toString(); this.parent = parent; }
@Override public FetchedData apply(Resource resource) { try { return this.fetch(resource); } catch (Exception e) { int statusCode = DEFAULT_ERROR_CODE; if (e instanceof FileNotFoundException){ statusCode = 404; } LOG.warn("FETCH-ERROR {}", resource.getUrl()); LOG.debug(e.getMessage(), e); FetchedData fetchedData = new FetchedData(new byte[0], "", statusCode); resource.setStatus(ResourceStatus.ERROR.toString()); fetchedData.setResource(resource); return fetchedData; } } }
IOUtils.closeQuietly(bufferOutStream); FetchedData fetchedData = new FetchedData(rawData, urlConn.getContentType(), responseCode); resource.setStatus(ResourceStatus.FETCHED.toString()); fetchedData.setResource(resource); fetchedData.setHeaders(urlConn.getHeaderFields());
resource.setStatus(ResourceStatus.FETCHED.toString()); fetchedData.setResource(resource); return fetchedData;