protected void loadWaybackCdx(String urlkey, WaybackRequest wbRequest, CDXQuery query, AuthToken waybackAuthToken, CDXToSearchResultWriter resultWriter, boolean fuzzy) throws IOException, AccessControlException { if ((remoteCdxPath != null) && !wbRequest.isUrlQueryRequest()) { try { // Not supported for remote requests, caching the entire cdx wbRequest.setTimestampSearchKey(false); this.remoteCdxServerQuery(urlkey, resultWriter.getQuery(), waybackAuthToken, resultWriter); return; } catch (IOException io) { // Try again below } catch (RuntimeIOException rte) { Throwable cause = rte.getCause(); if (cause instanceof AccessControlException) { throw (AccessControlException)cause; } else { LOGGER.warning(rte.toString()); } } } cdxServer.getCdx(resultWriter.getQuery(), waybackAuthToken, resultWriter); }
protected void loadWaybackCdx(String urlkey, WaybackRequest wbRequest, CDXQuery query, AuthToken waybackAuthToken, CDXToSearchResultWriter resultWriter, boolean fuzzy) throws IOException, AccessControlException { if ((remoteCdxPath != null) && !wbRequest.isUrlQueryRequest()) { try { // Not supported for remote requests, caching the entire cdx wbRequest.setTimestampSearchKey(false); this.remoteCdxServerQuery(urlkey, resultWriter.getQuery(), waybackAuthToken, resultWriter); return; } catch (IOException io) { // Try again below } catch (RuntimeIOException rte) { Throwable cause = rte.getCause(); if (cause instanceof AccessControlException) { throw (AccessControlException)cause; } else { LOGGER.warning(rte.toString()); } } } cdxServer.getCdx(resultWriter.getQuery(), waybackAuthToken, resultWriter); }
public SearchResults query(WaybackRequest wbRequest) throws ResourceIndexNotAvailableException, ResourceNotInArchiveException, BadQueryException, AccessControlException { SearchResults results = null; // return value placeholder if (wbRequest.isReplayRequest()) { results = doCaptureQuery(wbRequest, TYPE_REPLAY); results.putFilter(WaybackRequest.REQUEST_TYPE, WaybackRequest.REQUEST_REPLAY_QUERY); } else if (wbRequest.isCaptureQueryRequest()) { results = doCaptureQuery(wbRequest, TYPE_CAPTURE); results.putFilter(WaybackRequest.REQUEST_TYPE, WaybackRequest.REQUEST_CAPTURE_QUERY); } else if (wbRequest.isUrlQueryRequest()) { results = doUrlQuery(wbRequest); results.putFilter(WaybackRequest.REQUEST_TYPE, WaybackRequest.REQUEST_URL_QUERY); } else { throw new BadQueryException("Unknown query type, must be " + WaybackRequest.REQUEST_REPLAY_QUERY + ", " + WaybackRequest.REQUEST_CAPTURE_QUERY + ", or " + WaybackRequest.REQUEST_URL_QUERY); } return results; }
public SearchResults query(WaybackRequest wbRequest) throws ResourceIndexNotAvailableException, ResourceNotInArchiveException, BadQueryException, AccessControlException { SearchResults results = null; // return value placeholder if (wbRequest.isReplayRequest()) { results = doCaptureQuery(wbRequest, TYPE_REPLAY); results.putFilter(WaybackRequest.REQUEST_TYPE, WaybackRequest.REQUEST_REPLAY_QUERY); } else if (wbRequest.isCaptureQueryRequest()) { results = doCaptureQuery(wbRequest, TYPE_CAPTURE); results.putFilter(WaybackRequest.REQUEST_TYPE, WaybackRequest.REQUEST_CAPTURE_QUERY); } else if (wbRequest.isUrlQueryRequest()) { results = doUrlQuery(wbRequest); results.putFilter(WaybackRequest.REQUEST_TYPE, WaybackRequest.REQUEST_URL_QUERY); } else { throw new BadQueryException("Unknown query type, must be " + WaybackRequest.REQUEST_REPLAY_QUERY + ", " + WaybackRequest.REQUEST_CAPTURE_QUERY + ", or " + WaybackRequest.REQUEST_URL_QUERY); } return results; }
ms.append('+'); if(wbRequest.isUrlQueryRequest()) { ms.append("url%3A"); } else {
ms.append('+'); if(wbRequest.isUrlQueryRequest()) { ms.append("url%3A"); } else {
resultWriter = this.getCaptureSearchWriter(wbRequest, waybackAuthToken, false); } else if (wbRequest.isUrlQueryRequest()) { resultWriter = this.getUrlSearchWriter(wbRequest); } else {
resultWriter = this.getCaptureSearchWriter(wbRequest, waybackAuthToken, false); } else if (wbRequest.isUrlQueryRequest()) { resultWriter = this.getUrlSearchWriter(wbRequest); } else {
/** * test of {@link PathPrefixDateRangeQueryRequestParser}. * <p>explicit timestamp range and trailing "*" in URL. * this is a URL-query request, timestamp range becomes * start and end timestamp.</p> * <p>timerange without "*" is not recognized. it could be.</p> */ public void testPathPrefixDateRange() throws Exception { { WaybackRequest wbr1 = parse("/web/20100101-20100531*/http://www.yahoo.com/*"); assertNotNull(wbr1); assertTrue(wbr1.isUrlQueryRequest()); assertEquals("20100101000000", wbr1.getStartTimestamp()); assertEquals("20100531235959", wbr1.getEndTimestamp()); assertEquals("http://www.yahoo.com/", wbr1.getRequestUrl()); } // TODO: date range without "*" }
/** * test of {@link PathPrefixDatePrefixQueryRequestParser}. * <p>this is a URL query with (optional) single timestamp date range. * timestamp, if non-empty, becomes start and end. * </p> */ public void testPathPrefixDatePrefix() throws Exception { WaybackRequest wbr1 = parse("/web/2010*/http://www.yahoo.com/*"); assertNotNull(wbr1); assertTrue(wbr1.isUrlQueryRequest()); assertEquals("20100101000000", wbr1.getStartTimestamp()); assertEquals("20101231235959", wbr1.getEndTimestamp()); // does not set replayTimestamp, but it is not a required behavior. assertEquals("http://www.yahoo.com/", wbr1.getRequestUrl()); WaybackRequest wbr2 = parse("/web/*/http://www.yahoo.com/*"); assertNotNull(wbr2); assertTrue(wbr2.isUrlQueryRequest()); assertEquals(EXPECTED_START_TIMESTAMP, wbr2.getStartTimestamp()); assertEquals(null, wbr2.getEndTimestamp()); // does not set replayTimestamp, but it is not a required behavior. assertEquals("http://www.yahoo.com/", wbr2.getRequestUrl()); // timestamp is up to 13 digits. 14-digit timestamp results in null (-> 404). // TODO: there'd be a nicer way. WaybackRequest wbr3 = parse("/web/20130101000000*/http://www.yahoo.com/*"); assertNull(wbr3); } /**
} else if(request.isUrlQueryRequest()) { chain.addFilter(new UrlPrefixMatchFilter(keyUrl)); chain.addFilter(new DateRangeFilter(startDate, endDate));
} else if(request.isUrlQueryRequest()) { chain.addFilter(new UrlPrefixMatchFilter(keyUrl)); chain.addFilter(new DateRangeFilter(startDate, endDate));
/** * test of {@link PathPrefixDatePrefixQueryRequestParser}. * <p>%-encoded timestamp.</p> * @throws Exception */ public void testPathPrefixDatePrefixEncoded() throws Exception { { WaybackRequest wbr = parse("/web/2010%2A/http://www.yahoo.com/*"); assertNotNull(wbr); assertTrue(wbr.isUrlQueryRequest()); assertEquals("20100101000000", wbr.getStartTimestamp()); assertEquals("20101231235959", wbr.getEndTimestamp()); assertEquals("http://www.yahoo.com/", wbr.getRequestUrl()); } // negative case - %2A doesn't make it path-prefix. { WaybackRequest wbr = parse("/web/2010%2A/http://www.yahoo.com/%2A"); assertNotNull(wbr); assertTrue(wbr.isCaptureQueryRequest()); assertEquals("20100101000000", wbr.getStartTimestamp()); assertEquals("20101231235959", wbr.getEndTimestamp()); assertEquals("http://www.yahoo.com/%2A", wbr.getRequestUrl()); } }
/** * test of {@link PathPrefixDateRangeQueryRequestParser}, * %-encoded version. * @throws Exception */ public void testPathPrefixdateRangeEncoded() throws Exception { { WaybackRequest wbr = parse("/web/20100101%2D20100531%2A/http://www.yahoo.com/*"); assertNotNull(wbr); assertTrue(wbr.isUrlQueryRequest()); assertEquals("20100101000000", wbr.getStartTimestamp()); assertEquals("20100531235959", wbr.getEndTimestamp()); assertEquals("http://www.yahoo.com/", wbr.getRequestUrl()); } { // URL part shall not be URL-decoded WaybackRequest wbr = parse("/web/20100101%2D20100531%2A/http%3a//www.yahoo.com/%2A"); assertNotNull(wbr); assertTrue(wbr.isCaptureQueryRequest()); assertEquals("20100101000000", wbr.getStartTimestamp()); assertEquals("20100531235959", wbr.getEndTimestamp()); // WaybackRequet#setRequestUrl() prepends "http://" if urlStr does // not begin with http:// assertEquals("http://http%3a//www.yahoo.com/%2A", wbr.getRequestUrl()); } }