/** * @param tweetQueue tweet Queue. * @param hosts Hostes. * @param endpoint Endpoint. * @return Client. */ protected Client buildClient(BlockingQueue<String> tweetQueue, HttpHosts hosts, StreamingEndpoint endpoint) { Authentication authentication = new OAuth1(oAuthSettings.getConsumerKey(), oAuthSettings.getConsumerSecret(), oAuthSettings.getAccessToken(), oAuthSettings.getAccessTokenSecret()); ClientBuilder builder = new ClientBuilder() .name("Ignite-Twitter-Client") .hosts(hosts) .authentication(authentication) .endpoint(endpoint) .processor(new StringDelimitedProcessor(tweetQueue)); return builder.build(); }
/** * @return the current rate if it is available, NaN if not. * The rate is unavailable if <code>granularityMillis</code> hasn't elapsed */ public double getCurrentRateSeconds() { return rateUpdater.getCurrentRateSeconds(); }
/** * Pauses the rate tracker: the rate will be frozen. */ @Override public void pause() { rateUpdater.pause(); }
@OnScheduled public void onScheduled(final ProcessContext context) { final String endpointName = context.getProperty(ENDPOINT).getValue(); final Authentication oauth = new OAuth1(context.getProperty(CONSUMER_KEY).getValue(), context.getProperty(CONSUMER_SECRET).getValue(), context.getProperty(ACCESS_TOKEN).getValue(), context.getProperty(ACCESS_TOKEN_SECRET).getValue()); final ClientBuilder clientBuilder = new ClientBuilder(); clientBuilder.name("GetTwitter[id=" + getIdentifier() + "]") .authentication(oauth) .eventMessageQueue(eventQueue) .processor(new StringDelimitedProcessor(messageQueue)); if (ENDPOINT_SAMPLE.getValue().equals(endpointName)) { host = Constants.STREAM_HOST; final StatusesSampleEndpoint sse = new StatusesSampleEndpoint(); streamingEndpoint = sse; if (languages != null) { sse.languages(languages); final StatusesFirehoseEndpoint firehoseEndpoint = new StatusesFirehoseEndpoint(); streamingEndpoint = firehoseEndpoint; if (languages != null) { firehoseEndpoint.languages(languages); final StatusesFilterEndpoint filterEndpoint = new StatusesFilterEndpoint(); filterEndpoint.trackTerms(terms);
/** * TODO: This must be limited to 25 adds per seconds */ public void addUser(String streamId, long userId) throws IOException, ControlStreamException { Endpoint endpoint = SitestreamEndpoint.addUserEndpoint(streamId); endpoint.addPostParameter(Constants.USER_ID_PARAM, Long.toString(userId)); HttpUriRequest request = HttpConstants.constructRequest(hosts.nextHost(), endpoint, auth); consumeHttpEntityContent(makeControlStreamRequest(request)); }
public String getInfo(String streamId) throws IOException, ControlStreamException { Endpoint endpoint = SitestreamEndpoint.streamInfoEndpoint(streamId); HttpUriRequest request = HttpConstants.constructRequest(hosts.nextHost(), endpoint, auth); HttpResponse response = makeControlStreamRequest(request); return consumeHttpEntityContent(response); }
@Override public void handleExponentialBackoff() { handleBackoff(incrAndGetExponentialBackoff()); }
@Override public void handleLinearBackoff() { handleBackoff(incrAndGetLinearBackoff()); }
public String getFriends(String streamId, long userId) throws IOException, ControlStreamException { return getFriends(streamId, userId, 0); }
public ClientBuilder() { enableGZip = true; name = "hosebird-client-" + clientNum.getAndIncrement(); ThreadFactory threadFactory = new ThreadFactoryBuilder() .setDaemon(true) .setNameFormat("hosebird-client-io-thread-%d") .build(); executorService = Executors.newSingleThreadExecutor(threadFactory); ThreadFactory rateTrackerThreadFactory = new ThreadFactoryBuilder() .setDaemon(true) .setNameFormat("hosebird-client-rateTracker-thread-%d") .build(); ScheduledExecutorService scheduledExecutor = Executors.newScheduledThreadPool(1, rateTrackerThreadFactory); rateTracker = new BasicRateTracker(30000, 100, true, scheduledExecutor); reconnectionManager = new BasicReconnectionManager(5); socketTimeoutMillis = 60000; connectionTimeoutMillis = 4000; schemeRegistry = SchemeRegistryFactory.createDefault(); }
/** * @param retries Number of retries to attempt when we experience retryable connection errors */ public ClientBuilder retries(int retries) { this.reconnectionManager = new BasicReconnectionManager(retries); return this; }
@Override public void eventObserved() { rateUpdater.eventObserved(); }
@Override public void resume() { rateUpdater.resume(); }
@VisibleForTesting int incrAndGetLinearBackoff() { exponentialBackoffCount = 0; linearBackoffCount += 1; return calculateLinearBackoffMillis(); }
/** * @deprecated info.json does not take a user_id param (@see https://dev.twitter.com/docs/streaming-apis/streams/site/control#info) */ @Deprecated public String getInfo(String streamId, long userId) throws IOException, ControlStreamException { return getInfo(streamId); }
/** * Only used for testing */ @VisibleForTesting void recalculate() { rateUpdater.run(); }
@VisibleForTesting int incrAndGetExponentialBackoff() { linearBackoffCount = 0; exponentialBackoffCount += 1; return calculateExponentialBackoffMillis(); }
/** * Stops and shuts down the underlying executor */ @Override public void shutdown() { stop(); executor.shutdown(); }
public BasicRateTracker(int granularityMillis, int numBuckets, boolean startPaused, ScheduledExecutorService executor) { Preconditions.checkArgument(numBuckets > 0); Preconditions.checkArgument(granularityMillis > 0); Preconditions.checkArgument(granularityMillis / numBuckets > 0); this.granularityMillis = granularityMillis; this.numBuckets = numBuckets; this.executor = Preconditions.checkNotNull(executor); this.rateUpdater = new RateUpdater(startPaused); }
Authentication auth = new OAuth1(properties.getProperty(CONSUMER_KEY), properties.getProperty(CONSUMER_SECRET), properties.getProperty(TOKEN), properties.getProperty(TOKEN_SECRET)); client = new ClientBuilder() .name(properties.getProperty(CLIENT_NAME, "flink-twitter-source")) .hosts(properties.getProperty(CLIENT_HOSTS, Constants.STREAM_HOST)) .endpoint(endpoint) .authentication(auth) .processor(new HosebirdMessageProcessor() { public DelimitedStreamReader reader; .build(); client.connect(); running = true;