public static ScalarDay scan(Token token, Token postToken, Options options) { if (ScalarDay.DAY_PATTERN.matcher(token.getWord()).matches()) { int scalarValue = Integer.parseInt(token.getWord()); if (!(scalarValue > 31 || (postToken != null && Scalar.TIMES.contains(postToken.getWord())))) { return new ScalarDay(Integer.valueOf(scalarValue)); } } return null; } }
/** * Split the text on spaces and convert each word into * a Token */ protected static List<Token> baseTokenize(String text) { String[] words = text.split(" "); List<Token> tokens = new LinkedList<Token>(); for (String word : words) { tokens.add(new Token(word)); } return tokens; }
for( Token token: tokenList ){ System.out.print( token.getWord() + ":" ); for( RefCount refCount: token.getReferences() ){ System.out.print( " " + refCount.getFileId() + "*" + refCount.getCount() ); } System.out.println(); }
for (int i = 0; dayPortionIndex == -1 && i < tokenSize; i++) { Token t = tokens.get(i); if (t.getTag(RepeaterDayPortion.class) != null) { dayPortionIndex = i; for (int i = 0; timeIndex == -1 && i < tokenSize; i++) { Token t = tokens.get(i); if (t.getTag(RepeaterTime.class) != null) { timeIndex = i; Tag<RepeaterDayPortion<?>> t1Tag = t1.getTag(RepeaterDayPortion.class); System.out.println("Chronic.dealiasAndDisambiguateTimes: morning->am"); t1.untag(RepeaterDayPortion.class); t1.tag(new EnumRepeaterDayPortion(RepeaterDayPortion.DayPortion.AM)); System.out.println("Chronic.dealiasAndDisambiguateTimes: " + t1TagType + "->pm"); t1.untag(RepeaterDayPortion.class); t1.tag(new EnumRepeaterDayPortion(RepeaterDayPortion.DayPortion.PM)); t1 = tokens.get(i + 1); if (t0.getTag(RepeaterTime.class) != null && t0.getTag(RepeaterTime.class).getType().isAmbiguous() && (t1 == null || t1.getTag(RepeaterDayPortion.class) == null)) { Token distoken = new Token("disambiguator"); distoken.tag(new IntegerRepeaterDayPortion(Integer.valueOf(options.getAmbiguousTimeRange()))); ttokens.add(distoken);
public static List<Repeater<?>> getRepeaters(List<Token> tokens) { List<Repeater<?>> repeaters = new LinkedList<Repeater<?>>(); for (Token token : tokens) { Repeater<?> tag = token.getTag(Repeater.class); if (tag != null) { repeaters.add(tag); } } Collections.sort(repeaters); Collections.reverse(repeaters); return repeaters; }
public static List<Token> scan(List<Token> tokens, Options options) { for (Token token : tokens) { Pointer t = Pointer.scanForAll(token, options); if (t != null) { token.tag(t); } } return tokens; }
/** * Return the Tag that matches the given class */ @SuppressWarnings("unchecked") public <T extends Tag> T getTag(Class<T> tagClass) { List<T> matches = getTags(tagClass); T matchingTag = null; if (matches.size() > 0) { matchingTag = matches.get(0); } // if (matches.size() >= 2) { // throw new IllegalStateException("Multiple identical tags found (" + matches + ")"); // } // else if (matches.size() == 1) { // matchingTag = matches.get(0); // } return matchingTag; }
if (token.isTagged()) { taggedTokens.add(token);
public static List<Token> scan(List<Token> tokens, Options options) { for (Token token : tokens) { TimeZone t = TimeZone.scanForAll(token, options); if (t != null) { token.tag(t); } } return tokens; }
boolean optional = pattern.isOptional(); if (pattern instanceof TagPattern) { boolean match = (tokenIndex < tokens.size() && tokens.get(tokenIndex).getTags(((TagPattern) pattern).getTagClass()).size() > 0);
public static ScalarMonth scan(Token token, Token postToken, Options options) { if (ScalarMonth.MONTH_PATTERN.matcher(token.getWord()).matches()) { int scalarValue = Integer.parseInt(token.getWord()); if (!(scalarValue > 12 || (postToken != null && Scalar.TIMES.contains(postToken.getWord())))) { return new ScalarMonth(Integer.valueOf(scalarValue)); } } return null; } }
public static List<Token> scan(List<Token> tokens, Options options) { for (Token token : tokens) { Grabber t = Grabber.scanForAll(token, options); if (t != null) { token.tag(t); } } return tokens; }
public static Scalar scan(Token token, Token postToken, Options options) { if (Scalar.SCALAR_PATTERN.matcher(token.getWord()).matches()) { if (token.getWord() != null && token.getWord().length() > 0 && !(postToken != null && Scalar.TIMES.contains(postToken.getWord()))) { return new Scalar(Integer.valueOf(token.getWord())); } } else { Integer intStrValue = StringUtils.integerValue(token.getWord()); if (intStrValue != null) { return new Scalar(intStrValue); } } return null; }
public Span handle(List<Token> tokens, Span span, Options options) { int distance = tokens.get(0).getTag(Scalar.class).getType().intValue(); Repeater<?> repeater = tokens.get(1).getTag(Repeater.class); Pointer.PointerType pointer = tokens.get(2).getTag(Pointer.class).getType(); return repeater.getOffset(span, distance, pointer); }
public static Ordinal scan(Token token, Options options) { Matcher ordinalMatcher = ORDINAL_PATTERN.matcher(token.getWord()); if (ordinalMatcher.find()) { return new Ordinal(Integer.valueOf(ordinalMatcher.group(1))); } return null; }
public Span handle(List<Token> tokens, Options options) { int month = tokens.get(0).getTag(ScalarMonth.class).getType().intValue(); int day = tokens.get(1).getTag(ScalarDay.class).getType().intValue(); int year = tokens.get(2).getTag(ScalarYear.class).getType().intValue(); Span span; try { List<Token> timeTokens = tokens.subList(3, tokens.size()); Calendar dayStart = Time.construct(year, month, day); span = Handler.dayOrTime(dayStart, timeTokens, options); } catch (IllegalArgumentException e) { if (options.isDebug()) { e.printStackTrace(System.out); } span = null; } return span; }
public static List<Token> scan(List<Token> tokens, Options options) { for (Token token : tokens) { Separator t; t = SeparatorComma.scan(token, options); if (t != null) { token.tag(t); } t = SeparatorSlashOrDash.scan(token, options); if (t != null) { token.tag(t); } t = SeparatorAt.scan(token, options); if (t != null) { token.tag(t); } t = SeparatorIn.scan(token, options); if (t != null) { token.tag(t); } } return tokens; }