/** * Tokenizes a Tweet based off of {@link Status#getURLEntities()}. * * @param status The status to tokenize * * @return A list of {@link Token}s. */ @Override public List<Token> extractTokens(final Status status) { return Arrays.stream(status.getURLEntities()).map(this::linkOfEntity).collect(Collectors.toList()); }
@Override public void onStatus(Status status) { // add the tweet into the queue buffer //queue.offer(status.getText()); //add URLs to the queue buffer String url; if(status.getURLEntities().length != 0){ url = status.getURLEntities()[0].getURL(); queue.offer(url); } }
@Override public void onStatus(Status status) { if (status.getURLEntities() != null && status.getURLEntities().length != 0) { synchronized (queue) { queue.add(status); } logger.debug("Adding status to queue, current queue size: " + queue.size()); // try { // Thread.sleep(100); // } catch (InterruptedException e) { // } } }
private String filterOutURLFromTweet(final Status status) { final String tweet = status.getText(); final URLEntity[] urlEntities = status.getURLEntities(); int startOfURL; int endOfURL; String truncatedTweet = ""; for(final URLEntity urlEntity: urlEntities){ startOfURL = urlEntity.getStart(); endOfURL = urlEntity.getEnd(); truncatedTweet += tweet.substring(0, startOfURL) + tweet.substring(endOfURL); } return truncatedTweet; }
private String processTweet(Status status) { String tweet = status.getText(); for (URLEntity entity : status.getURLEntities()) { String url = String.format("<a target=\"_blank\" href=\"%s\">%s</a>", entity.getURL(), entity.getURL()); tweet = tweet.replace(entity.getURL(), url); } return tweet; }
@Override public List<Object> filter(Status status) { URLEntity urls[] = status.getURLEntities();
/** * @return The list of {@link Token}s that cannot be built from {@link Status#getURLEntities()} alone. */ @Override public List<Token> extractTokens(final Status status) { final String statusText = status.getText(); final URLEntity[] managedUrlEntities = status.getURLEntities(); String unamanagedText = statusText; for (final URLEntity processedEntity : managedUrlEntities) { unamanagedText = unamanagedText.replaceAll(processedEntity.getURL(), ""); } return URLMatcher.findAllUrlsWithPosition(unamanagedText).stream().map( urlWithPos -> new Token( urlWithPos._1, urlWithPos._2, urlWithPos._3, Token.TokenType.CLICKABLE, () -> browserSupport.openUrl(urlWithPos._1) ) ).collect(Collectors.toList()); }
@Override public void process(Status status) { URLEntity[] entities = status.getURLEntities(); if (entities != null) { for (URLEntity ue: entities) { if (ue != null) { // see why we intermittently get NPEs url.emit((ue.getExpandedURL() == null ? ue.getURL() : ue.getExpandedURL()).toString()); } } } } };
if (status.getURLEntities() != null) { for (final URLEntity map : status.getURLEntities()) { String u = map.getExpandedURL();
if (status.getURLEntities() != null) { for (final URLEntity map : status.getURLEntities()) { String u = map.getExpandedURL(); if (u == null) {
private void extractUrls(Status tweet, Set<Content> contents) { for (URLEntity urlEntity : tweet.getURLEntities()) { String url = urlEntity.getExpandedURL(); url = url == null ? urlEntity.getURL() : url; Content shareUrl = newBase(tweet); shareUrl.setContentName(url); shareUrl.setContentType("url"); contents.add(shareUrl); } }
@Override public void execute(Tuple tuple) { Status status = (Status)tuple.getValue(0); URLEntity urls[] = status.getURLEntities(); if(urls != null && urls.length > 0) { collector.emit(tuple, new Values(status)); collector.ack(tuple); } }
@Override public List<Object> filter(Status status) { URLEntity urls[] = status.getURLEntities(); if(urls == null || urls.length == 0) { return null;
private void prepareEntities(Status status, Tweet.Builder tweetBuilder) { for (URLEntity urlEntity : status.getURLEntities()) { tweetBuilder.addUrlEntity(urlEntity.getURL(), urlEntity.getExpandedURL()); } for (MediaEntity mediaEntity : status.getMediaEntities()) { if (isImage(mediaEntity)) { tweetBuilder.addImageEntity(mediaEntity.getURL(), mediaEntity.getMediaURL()); } } }
@Override public void emitTuples() { for (int size = statuses.size(); size-- > 0;) { Status s = statuses.poll(); if (status.isConnected()) { status.emit(s); } if (text.isConnected()) { text.emit(s.getText()); } if (url.isConnected()) { URLEntity[] entities = s.getURLEntities(); if (entities != null) { for (URLEntity ue : entities) { url.emit((ue.getExpandedURL() == null ? ue.getURL() : ue.getExpandedURL()).toString()); } } } if (hashtag.isConnected()) { HashtagEntity[] hashtagEntities = s.getHashtagEntities(); if (hashtagEntities != null) { for (HashtagEntity he : hashtagEntities) { hashtag.emit(he.getText()); } } } } }
@Override public void emitTuples() { for (int size = statuses.size(); size-- > 0;) { Status s = statuses.poll(); if (status.isConnected()) { status.emit(s); } if (text.isConnected()) { text.emit(s.getText()); } if (url.isConnected()) { URLEntity[] entities = s.getURLEntities(); if (entities != null) { for (URLEntity ue : entities) { url.emit((ue.getExpandedURL() == null ? ue.getURL() : ue.getExpandedURL()).toString()); } } } if (hashtag.isConnected()) { HashtagEntity[] hashtagEntities = s.getHashtagEntities(); if (hashtagEntities != null) { for (HashtagEntity he : hashtagEntities) { hashtag.emit(he.getText()); } } } } }
for (final URLEntity link : this.status.getURLEntities()) { final USMFStatus.Link l = new USMFStatus.Link(); l.href = link.getExpandedURL();
for (final URLEntity link : this.status.getURLEntities()) { final USMFStatus.Link l = new USMFStatus.Link(); l.href = link.getExpandedURL();
struct.put("MediaEntities", convert(status.getMediaEntities())); struct.put("SymbolEntities", convert(status.getSymbolEntities())); struct.put("URLEntities", convert(status.getURLEntities()));