@Override public synchronized void start() { LOGGER.info("Starting twitter source {} ...", this); docCount = 0; startTime = System.currentTimeMillis(); exceptionCount = 0; totalTextIndexed = 0; skippedDocs = 0; batchEndTime = System.currentTimeMillis() + maxBatchDurationMillis; twitterStream.sample(); LOGGER.info("Twitter source {} started.", getName()); // This should happen at the end of the start method, since this will // change the lifecycle status of the component to tell the Flume // framework that this component has started. Doing this any earlier // tells the framework that the component started successfully, even // if the method actually fails later. super.start(); }
twitterStream.sample(); // creates a generic StatusStream log.info("returned from sample()");
twitterStream.sample();
@Override public void activate(OperatorContext context) { ts.addListener(this); // we can only listen to tweets containing links by callng ts.links(). // it seems it requires prior signed agreement with twitter. ts.sample(); }
@Override public void activate(OperatorContext context) { ts.addListener(this); // we can only listen to tweets containing links by callng ts.links(). // it seems it requires prior signed agreement with twitter. ts.sample(); }
@Override public void start() { getTwitterStream().sample(); }
@Override public synchronized void start() { LOGGER.info("Starting twitter source {} ...", this); docCount = 0; startTime = System.currentTimeMillis(); exceptionCount = 0; totalTextIndexed = 0; skippedDocs = 0; batchEndTime = System.currentTimeMillis() + maxBatchDurationMillis; twitterStream.sample(); LOGGER.info("Twitter source {} started.", getName()); // This should happen at the end of the start method, since this will // change the lifecycle status of the component to tell the Flume // framework that this component has started. Doing this any earlier // tells the framework that the component started successfully, even // if the method actually fails later. super.start(); }
private synchronized void register(Subscriber<? super Status> subscriber) { if (subscribers.isEmpty()) { subscribers.add(subscriber); twitterStream.sample(); } else { subscribers.add(subscriber); } }
/** * Asynchronously retrieves a random sample of all public statuses. The sample * size and quality varies depending on the account permissions * <p/> * The default access level provides a small proportion of the Firehose. The "Gardenhose" * access level provides a proportion more suitable for data mining * and research applications that desire a larger proportion to be * statistically significant sample. * <p/> * Only one Twitter stream can be consumed using the same credentials. As a consequence, * only one twitter stream can be consumed per connector instance. * <p/> * {@sample.xml ../../../doc/twitter-connector.xml.sample twitter:sampleStream} * * @param callback the {@link SourceCallback} used to dispatch messages when a response is received */ @Source public void sampleStream(final SourceCallback callback) { listenToStatues(callback).sample(); }
public TwitterPushRecordReader(TwitterStream twitterStream, TwitterUtil.TweetListener tweetListener) { init(twitterStream); tweetListener.setInputQ(inputQ); this.twitterStream.addListener(tweetListener); twitterStream.sample(); }
public static void main(String[] args) throws TwitterException, IOException{ StatusListener listener = new StatusListener(){ public void onStatus(Status status) { System.out.println(status.getUser().getName() + " : " + status.getText()); } public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {} public void onTrackLimitationNotice(int numberOfLimitedStatuses) {} public void onException(Exception ex) { ex.printStackTrace(); } }; TwitterStream twitterStream = new TwitterStreamFactory().getInstance(); twitterStream.addListener(listener); // sample() method internally creates a thread which manipulates TwitterStream and calls these adequate listener methods continuously. twitterStream.sample(); }
@Override public void feedStatus(final PicSlurper slurper) { final StatusListener listener = new PriorityQueueStatusListener(slurper); final TwitterStream twitterStream = new TwitterStreamFactory().getInstance(accessToken); twitterStream.addListener(listener); twitterStream.sample(); } }
private void setUpTwitterConnection() { ConfigurationBuilder cb = setupConfigurationBuilder(); ts = new TwitterStreamFactory(cb.build()).getInstance(); ts.addListener(TwitterSampleInput.this); // we can only listen to tweets containing links by callng ts.links(). // it seems it requires prior signed agreement with twitter. ts.sample(); }
private void setUpTwitterConnection() { ConfigurationBuilder cb = setupConfigurationBuilder(); ts = new TwitterStreamFactory(cb.build()).getInstance(); ts.addListener(TwitterSampleInput.this); // we can only listen to tweets containing links by callng ts.links(). // it seems it requires prior signed agreement with twitter. ts.sample(); }
/** * Construct with the given Twitter API credentials and buffer * * @param token * the Twitter api authentication credentials * @param buffer * the backing buffer for storing data before consumption from * the stream */ public AbstractTwitterStreamDataset(TwitterAPIToken token, BlockingDroppingQueue<T> buffer) { super(buffer); final TwitterStream twitterStream = new TwitterStreamFactory(makeConfiguration(token)).getInstance(); twitterStream.addListener(new StatusAdapter() { @Override public void onStatus(Status status) { try { registerStatus(status, DataObjectFactory.getRawJSON(status)); } catch (final InterruptedException e) { // ignore } } }); twitterStream.sample(); }
private Observable<Status> status() { return Observable.create(subscriber -> { System.out.println("Establishing connection"); TwitterStream twitterStream = new TwitterStreamFactory().getInstance(); //... subscriber.add(Subscriptions.create(() -> { System.out.println("Disconnecting"); twitterStream.shutdown(); })); twitterStream.sample(); }); }
public void run(AdapterDescription adapterDescription) { StatusListener listener = getListener(); ConfigurationBuilder cb = new ConfigurationBuilder(); cb.setDebugEnabled(true); cb.setJSONStoreEnabled(true); twitterStream = new TwitterStreamFactory(cb.build()).getInstance(); // twitterStream.setOAuthAccessToken(new AccessToken("", "")); // twitterStream.addListener(listener); // twitterStream.sample(); // FilterQuery tweetFilterQuery = new FilterQuery(); // See // tweetFilterQuery.track(new String[]{"Bieber", "Teletubbies"}); // OR on keywords // tweetFilterQuery.locations(new double[][]{new double[]{-126.562500,30.448674}, // new double[]{-61.171875,44.087585 // }}); // See https://dev.twitter.com/docs/streaming-apis/parameters#locations for proper location doc. //Note that not all tweets have location metadata set. // tweetFilterQuery.language(new String[]{"en"}); // // // twitterStream.filter(tweetFilterQuery); }
public void run(AdapterDescription adapterDescription) { StatusListener listener = getListener(); ConfigurationBuilder cb = new ConfigurationBuilder(); cb.setDebugEnabled(true); cb.setJSONStoreEnabled(true); twitterStream = new TwitterStreamFactory(cb.build()).getInstance(); // twitterStream.setOAuthAccessToken(new AccessToken("", "")); // twitterStream.addListener(listener); // twitterStream.sample(); // FilterQuery tweetFilterQuery = new FilterQuery(); // See // tweetFilterQuery.track(new String[]{"Bieber", "Teletubbies"}); // OR on keywords // tweetFilterQuery.locations(new double[][]{new double[]{-126.562500,30.448674}, // new double[]{-61.171875,44.087585 // }}); // See https://dev.twitter.com/docs/streaming-apis/parameters#locations for proper location doc. //Note that not all tweets have location metadata set. // tweetFilterQuery.language(new String[]{"en"}); // // // twitterStream.filter(tweetFilterQuery); }
public static void main(String[] args) throws IOException { ConfigurationBuilder cb = new ConfigurationBuilder(); cb.setDebugEnabled(true) .setOAuthConsumerKey("*********") .setOAuthConsumerSecret("************") .setOAuthAccessToken("*******************") .setOAuthAccessTokenSecret("****************"); TwitterStream twitterStream = new TwitterStreamFactory(cb.build()).getInstance(); TwitterDownloader td = new TwitterDownloader(); StatusListener listener = td.makeListener(twitterStream); twitterStream.addListener(listener); twitterStream.sample(); }
@Override public void initialize() { this.queue = new LinkedBlockingQueue<>(1000); StatusListener statusListener = new TwitterStatusListener(queue); // Twitter stream authentication setup ConfigurationBuilder cfgBuilder = new ConfigurationBuilder(); cfgBuilder.setIncludeEntitiesEnabled(true); cfgBuilder.setJSONStoreEnabled(true); cfgBuilder.setOAuthAccessToken(config.getString(getConfigKey(BaseConf.TWITTER_ACCESS_TOKEN))); cfgBuilder.setOAuthAccessTokenSecret(config.getString(getConfigKey(BaseConf.TWITTER_ACCESS_TOKEN_SECRET))); cfgBuilder.setOAuthConsumerKey(config.getString(getConfigKey(BaseConf.TWITTER_CONSUMER_KEY))); cfgBuilder.setOAuthConsumerSecret(config.getString(getConfigKey(BaseConf.TWITTER_CONSUMER_SECRET))); twitterStream = new TwitterStreamFactory(cfgBuilder.build()).getInstance(); twitterStream.addListener(statusListener); if (filterQuery != null) { twitterStream.filter(filterQuery); } else { twitterStream.sample(); } }