public void apply(String field, CaptureSearchResult result) throws CDXFormatException { result.setUrlKey(field); }
public void apply(String field, CaptureSearchResult result) throws CDXFormatException { result.setUrlKey(field); }
private boolean checkExclusionFilter( String uri ) { // Default to no exclusions: if( smef == null ) return true; // Otherwise: ExclusionFilter ef = smef.get(); CaptureSearchResult r = new CaptureSearchResult(); // r.setOriginalUrl(uri); r.setUrlKey( uri ); try { if( ef.filterObject( r ) == ExclusionFilter.FILTER_INCLUDE ) { return true; } } catch( Exception e ) { log.error( "Exclusion filtering failed with exception: " + e ); e.printStackTrace(); } log.debug( "EXCLUDING this URL due to filter: " + uri ); // Exclude: return false; }
private boolean checkExclusionFilter( String uri ) { // Default to no exclusions: if( smef == null ) return true; // Otherwise: ExclusionFilter ef = smef.get(); CaptureSearchResult r = new CaptureSearchResult(); // r.setOriginalUrl(uri); r.setUrlKey( uri ); try { if( ef.filterObject( r ) == ExclusionFilter.FILTER_INCLUDE ) { return true; } } catch( Exception e ) { log.error( "Exclusion filtering failed with exception: " + e ); e.printStackTrace(); } log.debug( "EXCLUDING this URL due to filter: " + uri ); // Exclude: return false; }
result.setUrlKey(urlKey); result.setCaptureTimestamp(captureTS); result.setOriginalUrl(originalUrl);
result.setUrlKey(urlKey); result.setCaptureTimestamp(captureTS); result.setOriginalUrl(originalUrl);
@Override public boolean includeUrl(String urlKey, String originalUrl) { // return include(urlKey, originalUrl, true); if (UrlOperations.urlToScheme(originalUrl) == null) { originalUrl = UrlOperations.HTTP_SCHEME + originalUrl; } CaptureSearchResult resultTester = new FastCaptureSearchResult(); resultTester.setUrlKey(urlKey); resultTester.setOriginalUrl(originalUrl); // null captureTimestamp signifies per-URL access-check. resultTester.setCaptureTimestamp(null); return include(resultTester, true); }
@Override public boolean includeUrl(String urlKey, String originalUrl) { // return include(urlKey, originalUrl, true); if (UrlOperations.urlToScheme(originalUrl) == null) { originalUrl = UrlOperations.HTTP_SCHEME + originalUrl; } CaptureSearchResult resultTester = new FastCaptureSearchResult(); resultTester.setUrlKey(urlKey); resultTester.setOriginalUrl(originalUrl); // null captureTimestamp signifies per-URL access-check. resultTester.setCaptureTimestamp(null); return include(resultTester, true); }
@Override public CaptureSearchResult next() { String line = it.next(); String[] splits = line.split("\\s+"); CaptureSearchResult r = new CaptureSearchResult(); try { r.setCaptureDate(ArchiveUtils.parse14DigitDate(splits[0])); } catch (ParseException e) { e.printStackTrace(); } r.setOriginalUrl(url); r.setUrlKey(urlKey); // doesn't matter, or we get NPE r.setMimeType(splits[1]); r.setFile("foo"); // needed, or otherwise we'll get a NPE in CalendarResults.jsp r.setRedirectUrl("-"); r.setHttpCode("200"); r.setOffset(0); return r; }
WARCConstants.HEADER_KEY_FILENAME).toString(); result.setOriginalUrl("filedesc:"+filename); result.setUrlKey("filedesc:"+filename); } else { result.setOriginalUrl(DEFAULT_VALUE); result.setUrlKey(DEFAULT_VALUE); try { String urlKey = canonicalizer.urlStringToKey(origUrl); result.setUrlKey(urlKey); } catch (URIException e) { String shortUrl = LOGGER.warning("FAILED canonicalize(" + shortUrl + "):" + file + " " + offset); result.setUrlKey(origUrl);
result.setRedirectUrl(redirectUrl); result.setOriginalUrl(originalUrl); result.setUrlKey(urlKey); results.add(result);
WARCConstants.HEADER_KEY_FILENAME).toString(); result.setOriginalUrl("filedesc:"+filename); result.setUrlKey("filedesc:"+filename); } else { result.setOriginalUrl(DEFAULT_VALUE); result.setUrlKey(DEFAULT_VALUE); try { String urlKey = canonicalizer.urlStringToKey(origUrl); result.setUrlKey(urlKey); } catch (URIException e) { String shortUrl = LOGGER.warning("FAILED canonicalize(" + shortUrl + "):" + file + " " + offset); result.setUrlKey(origUrl);
result.setRedirectUrl(redirectUrl); result.setOriginalUrl(originalUrl); result.setUrlKey(urlKey); results.add(result);
private boolean isBlocked(ObjectFilter<CaptureSearchResult> filter, String url) throws URIException { CaptureSearchResult result = new CaptureSearchResult(); result.setOriginalUrl(url); result.setUrlKey(canonicalizer.urlStringToKey(url)); int filterResult = filter.filterObject(result); if(filterResult == ObjectFilter.FILTER_EXCLUDE) { return true; } return false; }
result.setUrlKey(canonUrl);
result.setUrlKey(canonUrl);