@Override public Collection<L> labels() { return labelIndex.objectsList(); }
@Override public Collection<L> labels() { return labelIndex.objectsList(); }
@Override public Collection<L> labels() { return labelIndex.objectsList(); }
public Collection<F> features() { return featureIndex.objectsList(); }
public List<L> getLabels() { return labelIndex.objectsList(); }
/** Return the Set of tags used by this tagger (available after training the tagger). * * @return The Set of tags used by this tagger */ public Set<String> tagSet() { return new HashSet<>(index.objectsList()); }
@Override public List<E> objectsList() { List<E> result = new ArrayList<>(); if (result.size() > backingIndexSize) { // we told you not to do this result.addAll(backingIndex.objectsList().subList(0, backingIndexSize)); } else { result.addAll(backingIndex.objectsList()); } result.addAll(spilloverIndex.objectsList()); return Collections.unmodifiableList(result); }
/** {@inheritDoc} */ @Override public Set<String> tagSet(Function<String,String> basicCategoryFunction) { Set<String> tagSet = new HashSet<>(); for (String tag : tagIndex.objectsList()) { tagSet.add(basicCategoryFunction.apply(tag)); } return tagSet; }
/** {@inheritDoc} */ @Override public Set<String> tagSet(Function<String,String> basicCategoryFunction) { Set<String> tagSet = new HashSet<>(); for (String tag : tagIndex.objectsList()) { tagSet.add(basicCategoryFunction.apply(tag)); } return tagSet; }
/** {@inheritDoc} */ @Override public Set<String> tagSet(Function<String,String> basicCategoryFunction) { Set<String> tagSet = new HashSet<>(); for (String tag : tagIndex.objectsList()) { tagSet.add(basicCategoryFunction.apply(tag)); } return tagSet; }
/** * Returns the Set of entities recognized by this Classifier. * * @return The Set of entities recognized by this Classifier. */ public Set<String> getTags() { Set<String> tags = Generics.newHashSet(classIndex.objectsList()); tags.remove(flags.backgroundSymbol); return tags; }
public HashIndex(Index<? extends E> index) { this(); // TODO: this assumes that no index supports deletion addAll(index.objectsList()); }
public Set<String> labels() { return Generics.newHashSet(classIndex.objectsList()); }
public static <L,F> OneVsAllClassifier<L,F> train(ClassifierFactory<String,F, Classifier<String,F>> classifierFactory, GeneralDataset<L, F> dataset) { Index<L> labelIndex = dataset.labelIndex(); return train(classifierFactory, dataset, labelIndex.objectsList()); }
/** * Makes a copy of the crf classifier */ public CRFClassifier(CRFClassifier<IN> crf) { super(crf.flags); this.windowSize = crf.windowSize; this.featureFactories = crf.featureFactories; this.pad = crf.pad; if (crf.knownLCWords == null) { this.knownLCWords = new MaxSizeConcurrentHashSet<>(crf.flags.maxAdditionalKnownLCWords); } else { this.knownLCWords = new MaxSizeConcurrentHashSet<>(crf.knownLCWords); this.knownLCWords.setMaxSize(this.knownLCWords.size() + crf.flags.maxAdditionalKnownLCWords); } this.featureIndex = (crf.featureIndex != null) ? new HashIndex<>(crf.featureIndex.objectsList()) : null; this.classIndex = (crf.classIndex != null) ? new HashIndex<>(crf.classIndex.objectsList()) : null; if (crf.labelIndices != null) { this.labelIndices = new ArrayList<>(crf.labelIndices.size()); for (int i = 0; i < crf.labelIndices.size(); i++) { this.labelIndices.add((crf.labelIndices.get(i) != null) ? new HashIndex<>(crf.labelIndices.get(i).objectsList()) : null); } } else { this.labelIndices = null; } this.cliquePotentialFunction = crf.cliquePotentialFunction; }
@Override public void finishTraining() { lex.finishTraining(); int numTags = tagIndex.size(); POSes = Generics.newHashSet(tagIndex.objectsList()); initialPOSDist = Distribution.laplaceSmoothedDistribution(initial, numTags, 0.5); markovPOSDists = Generics.newHashMap(); Set entries = ruleCounter.lowestLevelCounterEntrySet(); for (Object entry1 : entries) { Map.Entry entry = (Map.Entry) entry1; // Map.Entry<List<String>, Counter> entry = (Map.Entry<List<String>, Counter>) iter.next(); Distribution d = Distribution.laplaceSmoothedDistribution((ClassicCounter) entry.getValue(), numTags, 0.5); markovPOSDists.put(((List<String>) entry.getKey()).get(0), d); } }
public void loadTagIndex() { if (tagIndex == null) { tagIndex = new HashIndex<>(); for (String tag: classIndex.objectsList()) { String[] parts = tag.split("-"); // if (parts.length > 1) tagIndex.add(parts[parts.length-1]); } tagIndex.add(flags.backgroundSymbol); } if (flags.useNERPriorBIO) { if (entityMatrices == null) entityMatrices = readEntityMatrices(flags.entityMatrix, tagIndex); } }
public ExhaustivePCFGParser(BinaryGrammar bg, UnaryGrammar ug, Lexicon lex, Options op, Index<String> stateIndex, Index<String> wordIndex, Index<String> tagIndex) { // System.out.println("ExhaustivePCFGParser constructor called."); this.bg = bg; this.ug = ug; this.lex = lex; this.op = op; this.tlp = op.langpack(); goalStr = tlp.startSymbol(); this.stateIndex = stateIndex; this.wordIndex = wordIndex; this.tagIndex = tagIndex; tf = new LabeledScoredTreeFactory(); numStates = stateIndex.size(); isTag = new boolean[numStates]; // tag index is smaller, so we fill by iterating over the tag index // rather than over the state index for (String tag : tagIndex.objectsList()) { int state = stateIndex.indexOf(tag); if (state < 0) { continue; } isTag[state] = true; } }
private void ensureProbs(int word, boolean subtractTagScore) { if (word == lastWord) { return; } lastWord = word; if (functionWordTags.containsKey(wordIndex.get(word))) { logProbs = new ClassicCounter<>(); String trueTag = functionWordTags.get(wordIndex.get(word)); for (String tag : tagIndex.objectsList()) { if (ctlp.basicCategory(tag).equals(trueTag)) { logProbs.setCount(tag, 0); } else { logProbs.setCount(tag, Double.NEGATIVE_INFINITY); } } return; } Datum datum = new BasicDatum(featExtractor.makeFeatures(wordIndex.get(word))); logProbs = scorer.logProbabilityOf(datum); if (subtractTagScore) { Set<String> tagSet = logProbs.keySet(); for (String tag : tagSet) { logProbs.incrementCount(tag, -Math.log(tagDist.probabilityOf(tag))); } } }
public <F> double score(Classifier<L,F> classifier, GeneralDataset<L,F> data) { labelIndex = new HashIndex<>(); labelIndex.addAll(classifier.labels()); labelIndex.addAll(data.labelIndex.objectsList()); clearCounts(); int[] labelsArr = data.getLabelsArray(); for (int i = 0; i < data.size(); i++) { Datum<L, F> d = data.getRVFDatum(i); L guess = classifier.classOf(d); addGuess(guess, labelIndex.get(labelsArr[i])); } finalizeCounts(); return getFMeasure(); }