@Override public String url() { return mRequest.url().toString(); }
public boolean matches(Request request, Response response) { return url.equals(request.url().toString()) && requestMethod.equals(request.method()) && HttpHeaders.varyMatches(response, varyHeaders, request); }
@ToJson String urlToJson(HttpUrl httpUrl) { return httpUrl.toString(); }
private void drainQueue() throws Exception { for (HttpUrl url; (url = queue.take()) != null; ) { if (!fetchedUrls.add(url)) { continue; } Thread currentThread = Thread.currentThread(); String originalName = currentThread.getName(); currentThread.setName("Crawler " + url.toString()); try { fetch(url); } catch (IOException e) { System.out.printf("XXX: %s %s%n", url, e); } finally { currentThread.setName(originalName); } } }
@Override public @Nullable Uri getUri() { return response == null ? null : Uri.parse(response.request().url().toString()); }
Entry(Response response) { this.url = response.request().url().toString(); this.varyHeaders = HttpHeaders.varyHeaders(response); this.requestMethod = response.request().method(); this.protocol = response.protocol(); this.code = response.code(); this.message = response.message(); this.responseHeaders = response.headers(); this.handshake = response.handshake(); this.sentRequestMillis = response.sentRequestAtMillis(); this.receivedResponseMillis = response.receivedResponseAtMillis(); }
/** * Returns a string with containing this URL with its username, password, query, and fragment * stripped, and its path replaced with {@code /...}. For example, redacting {@code * http://username:password@example.com/path} returns {@code http://example.com/...}. */ public String redact() { return newBuilder("/...") .username("") .password("") .build() .toString(); }
/** See https://api.slack.com/methods/oauth.access. */ public OAuthSession exchangeCode(String code, HttpUrl redirectUrl) throws IOException { HttpUrl url = baseUrl.newBuilder("oauth.access") .addQueryParameter("client_id", clientId) .addQueryParameter("client_secret", clientSecret) .addQueryParameter("code", code) .addQueryParameter("redirect_uri", redirectUrl.toString()) .build(); Request request = new Request.Builder() .url(url) .build(); Call call = httpClient.newCall(request); try (Response response = call.execute()) { JsonAdapter<OAuthSession> jsonAdapter = moshi.adapter(OAuthSession.class); return jsonAdapter.fromJson(response.body().source()); } }
public static String key(HttpUrl url) { return ByteString.encodeUtf8(url.toString()).md5().hex(); }
/** See https://api.slack.com/docs/oauth. */ public HttpUrl authorizeUrl(String scopes, HttpUrl redirectUrl, ByteString state, String team) { HttpUrl.Builder builder = baseUrl.newBuilder("/oauth/authorize") .addQueryParameter("client_id", clientId) .addQueryParameter("scope", scopes) .addQueryParameter("redirect_uri", redirectUrl.toString()) .addQueryParameter("state", state.base64()); if (team != null) { builder.addQueryParameter("team", team); } return builder.build(); }
@Test public void testHandlesRedirect301s() throws Exception { String expected = "fakedata"; mockWebServer.enqueue(new MockResponse().setResponseCode(301) .setHeader("Location", mockWebServer.url("/redirect").toString())); mockWebServer.enqueue(new MockResponse().setResponseCode(200).setBody(expected)); getFetcher().loadData(Priority.LOW, callback); waitForResponseLatch.await(); verify(callback).onDataReady(streamCaptor.capture()); assertStreamOf(expected, streamCaptor.getValue()); }
@Test public void testHandlesRedirect302s() throws Exception { String expected = "fakedata"; mockWebServer.enqueue(new MockResponse().setResponseCode(302) .setHeader("Location", mockWebServer.url("/redirect").toString())); mockWebServer.enqueue(new MockResponse().setResponseCode(200).setBody(expected)); getFetcher().loadData(Priority.LOW, callback); waitForResponseLatch.await(); verify(callback).onDataReady(streamCaptor.capture()); assertStreamOf(expected, streamCaptor.getValue()); }
@Test public void testCallsLoadFailedAfterTooManyRedirects() throws Exception { for (int i = 0; i < 20; i++) { mockWebServer.enqueue(new MockResponse().setResponseCode(301) .setHeader("Location", mockWebServer.url("/redirect" + i).toString())); } getFetcher().loadData(Priority.NORMAL, callback); waitForResponseLatch.await(); verify(callback).onLoadFailed(isA(VolleyError.class)); }
@Before public void setup() { this.server = new MockWebServer(); this.webClient = WebClient .builder() .clientConnector(this.connector) .baseUrl(this.server.url("/").toString()) .build(); }
@Test public void testFailsOnRedirectLoops() throws Exception { mockWebServer.enqueue(new MockResponse().setResponseCode(301) .setHeader("Location", mockWebServer.url("/redirect").toString())); mockWebServer.enqueue(new MockResponse().setResponseCode(301) .setHeader("Location", mockWebServer.url("/redirect").toString())); getFetcher().loadData(Priority.IMMEDIATE, callback); verify(callback).onLoadFailed(isA(IOException.class)); }
@Test public void testHandlesRedirect301s() throws Exception { String expected = "fakedata"; mockWebServer.enqueue(new MockResponse().setResponseCode(301) .setHeader("Location", mockWebServer.url("/redirect").toString())); mockWebServer.enqueue(new MockResponse().setResponseCode(200).setBody(expected)); getFetcher().loadData(Priority.LOW, callback); verify(callback).onDataReady(streamCaptor.capture()); TestUtil.assertStreamOf(expected, streamCaptor.getValue()); assertThat(mockWebServer.takeRequest().getMethod()).isEqualTo("GET"); assertThat(mockWebServer.takeRequest().getMethod()).isEqualTo("GET"); }
@Test public void testHandlesRedirect302s() throws Exception { String expected = "fakedata"; mockWebServer.enqueue(new MockResponse().setResponseCode(302) .setHeader("Location", mockWebServer.url("/redirect").toString())); mockWebServer.enqueue(new MockResponse().setResponseCode(200).setBody(expected)); getFetcher().loadData(Priority.LOW, callback); verify(callback).onDataReady(streamCaptor.capture()); TestUtil.assertStreamOf(expected, streamCaptor.getValue()); assertThat(mockWebServer.takeRequest().getMethod()).isEqualTo("GET"); assertThat(mockWebServer.takeRequest().getMethod()).isEqualTo("GET"); }
@Test public void testFailsAfterTooManyRedirects() throws Exception { for (int i = 0; i < 10; i++) { mockWebServer.enqueue(new MockResponse().setResponseCode(301) .setHeader("Location", mockWebServer.url("/redirect" + i).toString())); } getFetcher().loadData(Priority.NORMAL, callback); verify(callback).onLoadFailed(isA(IOException.class)); }
@Test public void testHandlesUpToFiveRedirects() throws Exception { int numRedirects = 4; String expected = "redirectedData"; String redirectBase = "/redirect"; for (int i = 0; i < numRedirects; i++) { mockWebServer.enqueue(new MockResponse().setResponseCode(301) .setHeader("Location", mockWebServer.url(redirectBase + i).toString())); } mockWebServer.enqueue(new MockResponse().setResponseCode(200).setBody(expected)); getFetcher().loadData(Priority.NORMAL, callback); waitForResponseLatch.await(); verify(callback).onDataReady(streamCaptor.capture()); assertStreamOf(expected, streamCaptor.getValue()); assertThat(mockWebServer.takeRequest().getPath()).contains(DEFAULT_PATH); for (int i = 0; i < numRedirects; i++) { assertThat(mockWebServer.takeRequest().getPath()).contains(redirectBase + i); } }
@Before public void setUp() { this.factory = new ReactorResourceFactory(); this.factory.afterPropertiesSet(); this.server = new MockWebServer(); this.webClient = WebClient .builder() .clientConnector(initConnector()) .baseUrl(this.server.url("/").toString()) .build(); }