@Test public void testIndexWithNewline() throws IOException { String[] sentence = "He belongs to Apache \n Software Foundation .".split(" "); NameContextGenerator CG = new DefaultNameContextGenerator( (AdaptiveFeatureGenerator[]) null); NameSample nameSample = new NameSample(sentence, new Span[] { new Span(3, 7) }, false); ObjectStream<Event> eventStream = new NameFinderEventStream( ObjectStreamUtils.createObjectStream(nameSample), "org", CG, null); DataIndexer indexer = new TwoPassDataIndexer(); indexer.init(new TrainingParameters(Collections.emptyMap()), null); indexer.index(eventStream); Assert.assertEquals(5, indexer.getContexts().length); } }
@Override protected Iterator<Event> createEvents(NameSample sample) { if (sample.isClearAdaptiveDataSet()) { contextGenerator.clearAdaptiveData(); } Span[] names = sample.getNames(); if (!Objects.isNull(this.defaultType)) { overrideType(names); } String[] outcomes = codec.encode(names, sample.getSentence().length); // String outcomes[] = generateOutcomes(sample.getNames(), type, sample.getSentence().length); additionalContextFeatureGenerator.setCurrentContext(sample.getAdditionalContext()); String[] tokens = new String[sample.getSentence().length]; for (int i = 0; i < sample.getSentence().length; i++) { tokens[i] = sample.getSentence()[i]; } return generateEvents(tokens, outcomes, contextGenerator).iterator(); }
@SuppressWarnings("unchecked") public Event[] updateContext(Sequence sequence, AbstractModel model) { TokenNameFinder tagger = new NameFinderME(new TokenNameFinderModel( "x-unspecified", model, Collections.emptyMap(), null)); String[] sentence = ((Sequence<NameSample>) sequence).getSource().getSentence(); String[] tags = seqCodec.encode(tagger.find(sentence), sentence.length); Event[] events = new Event[sentence.length]; NameFinderEventStream.generateEvents(sentence,tags,pcg).toArray(events); return events; }
String[] tokens = line.split(" "); Span[] spans = chunker.nameFinder.find(tokens); String[] outcomes = NameFinderEventStream.generateOutcomes(spans, null, tokens.length);
@SuppressWarnings("unchecked") public Event[] updateContext(Sequence sequence, AbstractModel model) { TokenNameFinder tagger = new NameFinderME(new TokenNameFinderModel( "x-unspecified", model, Collections.emptyMap(), null)); String[] sentence = ((Sequence<NameSample>) sequence).getSource().getSentence(); String[] tags = seqCodec.encode(tagger.find(sentence), sentence.length); Event[] events = new Event[sentence.length]; NameFinderEventStream.generateEvents(sentence,tags,pcg).toArray(events); return events; }
nameSpans[fi] = finders[fi].nameFinder.find(tokens); nameOutcomes[fi] = NameFinderEventStream.generateOutcomes(nameSpans[fi], null, tokens.length);
ObjectStream<Event> eventStream = new NameFinderEventStream(samples, type, factory.createContextGenerator(), factory.createSequenceCodec());
@Override protected Iterator<Event> createEvents(NameSample sample) { if (sample.isClearAdaptiveDataSet()) { contextGenerator.clearAdaptiveData(); } Span[] names = sample.getNames(); if (!Objects.isNull(this.defaultType)) { overrideType(names); } String[] outcomes = codec.encode(names, sample.getSentence().length); // String outcomes[] = generateOutcomes(sample.getNames(), type, sample.getSentence().length); additionalContextFeatureGenerator.setCurrentContext(sample.getAdditionalContext()); String[] tokens = new String[sample.getSentence().length]; for (int i = 0; i < sample.getSentence().length; i++) { tokens[i] = sample.getSentence()[i]; } return generateEvents(tokens, outcomes, contextGenerator).iterator(); }
@SuppressWarnings("unchecked") public Event[] updateContext(Sequence sequence, AbstractModel model) { TokenNameFinder tagger = new NameFinderME(new TokenNameFinderModel( "x-unspecified", model, Collections.emptyMap(), null)); String[] sentence = ((Sequence<NameSample>) sequence).getSource().getSentence(); String[] tags = seqCodec.encode(tagger.find(sentence), sentence.length); Event[] events = new Event[sentence.length]; NameFinderEventStream.generateEvents(sentence,tags,pcg).toArray(events); return events; }
nameOutcomes[fi] = NameFinderEventStream.generateOutcomes(nameSpans[fi], null, tokens.length);
/** * Tests the correctly generated outcomes for a test sentence. */ @Test public void testOutcomesForSingleTypeSentence() throws IOException { NameSample nameSample = new NameSample(SENTENCE, new Span[]{new Span(0, 2, "person")}, false); try (ObjectStream<Event> eventStream = new NameFinderEventStream( ObjectStreamUtils.createObjectStream(nameSample))) { Assert.assertEquals("person-" + NameFinderME.START, eventStream.read().getOutcome()); Assert.assertEquals("person-" + NameFinderME.CONTINUE, eventStream.read().getOutcome()); for (int i = 0; i < 10; i++) { Assert.assertEquals(NameFinderME.OTHER, eventStream.read().getOutcome()); } Assert.assertNull(eventStream.read()); } }
@Override protected Iterator<Event> createEvents(NameSample sample) { if (sample.isClearAdaptiveDataSet()) { contextGenerator.clearAdaptiveData(); } Span[] names = sample.getNames(); if (!Objects.isNull(this.defaultType)) { overrideType(names); } String[] outcomes = codec.encode(names, sample.getSentence().length); // String outcomes[] = generateOutcomes(sample.getNames(), type, sample.getSentence().length); additionalContextFeatureGenerator.setCurrentContext(sample.getAdditionalContext()); String[] tokens = new String[sample.getSentence().length]; for (int i = 0; i < sample.getSentence().length; i++) { tokens[i] = sample.getSentence()[i]; } return generateEvents(tokens, outcomes, contextGenerator).iterator(); }
/** * Tests the correctly generated outcomes for a test sentence. If the Span * does not declare its type and the user passed a type, use the type from * user */ @Test public void testOutcomesWithType() throws IOException { String type = "XYZ"; NameSample nameSample = new NameSample(SENTENCE, new Span[] { new Span(0, 2) }, false); ObjectStream<Event> eventStream = new NameFinderEventStream( ObjectStreamUtils.createObjectStream(nameSample), type, CG, null); String prefix = type + "-"; Assert.assertEquals(prefix + NameFinderME.START, eventStream.read().getOutcome()); Assert.assertEquals(prefix + NameFinderME.CONTINUE, eventStream.read().getOutcome()); for (int i = 0; i < 10; i++) { Assert.assertEquals(NameFinderME.OTHER, eventStream.read().getOutcome()); } Assert.assertNull(eventStream.read()); eventStream.close(); }
/** * Tests the correctly generated outcomes for a test sentence. If the Span * declares its type, passing the type to event stream has no effect */ @Test public void testOutcomesTypeCantOverride() throws IOException { String type = "XYZ"; NameSample nameSample = new NameSample(SENTENCE, new Span[] { new Span(0, 2, "person") }, false); ObjectStream<Event> eventStream = new NameFinderEventStream( ObjectStreamUtils.createObjectStream(nameSample), type, CG, null); String prefix = type + "-"; Assert.assertEquals(prefix + NameFinderME.START, eventStream.read().getOutcome()); Assert.assertEquals(prefix + NameFinderME.CONTINUE, eventStream.read().getOutcome()); for (int i = 0; i < 10; i++) { Assert.assertEquals(NameFinderME.OTHER, eventStream.read().getOutcome()); } Assert.assertNull(eventStream.read()); eventStream.close(); }
/** * Tests the correctly generated outcomes for a test sentence. If the Span * does not declare its type and the user did not set a type, it will use * "default". */ @Test public void testOutcomesTypeEmpty() throws IOException { NameSample nameSample = new NameSample(SENTENCE, new Span[] { new Span(0, 2) }, false); ObjectStream<Event> eventStream = new NameFinderEventStream( ObjectStreamUtils.createObjectStream(nameSample), null, CG, null); String prefix = "default-"; Assert.assertEquals(prefix + NameFinderME.START, eventStream.read().getOutcome()); Assert.assertEquals(prefix + NameFinderME.CONTINUE, eventStream.read().getOutcome()); for (int i = 0; i < 10; i++) { Assert.assertEquals(NameFinderME.OTHER, eventStream.read().getOutcome()); } Assert.assertNull(eventStream.read()); eventStream.close(); } }
ObjectStream<Event> eventStream = new NameFinderEventStream(samples, type, factory.createContextGenerator(), factory.createSequenceCodec());
ObjectStream<Event> eventStream = new NameFinderEventStream(samples, type, factory.createContextGenerator(), factory.createSequenceCodec());