public LanguageSample read() throws IOException { String sampleString; while ((sampleString = samples.read()) != null) { int tabIndex = sampleString.indexOf("\t"); if (tabIndex > 0) { String lang = sampleString.substring(0, tabIndex); String context = sampleString.substring(tabIndex + 1); return new LanguageSample(new Language(lang), context); } } return null; } }
@Override public LanguageSample read() throws IOException { StringBuilder sampleString = new StringBuilder(); int count = 0; while (count < sentencesPerSample && lineIterator.hasNext()) { String line = lineIterator.next(); int textStart = line.indexOf('\t') + 1; sampleString.append(line.substring(textStart) + " "); count++; } if (sampleString.length() > 0) { return new LanguageSample(new Language(lang), sampleString); } return null; } }
@Test(expected = NullPointerException.class) public void emptyLanguage() throws Exception { new Language(null); }
@Test(expected = NullPointerException.class) public void emptyLanguageConfidence() throws Exception { new Language(null, 0.05); }
@Override public Language[] predictLanguages(CharSequence content) { double[] eval = model.getMaxentModel().eval(mContextGenerator.getContext(content.toString())); Language[] arr = new Language[eval.length]; for (int i = 0; i < eval.length; i++) { arr[i] = new Language(model.getMaxentModel().getOutcome(i), eval[i]); } Arrays.sort(arr, (o1, o2) -> Double.compare(o2.getConfidence(), o1.getConfidence())); return arr; }
@Test public void testEquals() throws Exception { LanguageSample sampleA = new LanguageSample(new Language("aLang"), "aContext"); LanguageSample sampleA1 = new LanguageSample(new Language("aLang"), "aContext"); LanguageSample sampleB = new LanguageSample(new Language("bLang"), "aContext"); LanguageSample sampleC = new LanguageSample(new Language("aLang"), "bContext"); Assert.assertEquals(sampleA, sampleA); Assert.assertEquals(sampleA, sampleA1); Assert.assertNotEquals(sampleA, sampleB); Assert.assertNotEquals(sampleA, sampleC); Assert.assertNotEquals(sampleB, sampleC); Assert.assertNotEquals(sampleA, "something else"); } }
@Test public void testHash() { int hashA = new LanguageSample(new Language("aLang"), "aContext").hashCode(); int hashB = new LanguageSample(new Language("bLang"), "aContext").hashCode(); int hashC = new LanguageSample(new Language("aLang"), "bContext").hashCode(); Assert.assertNotEquals(hashA, hashB); Assert.assertNotEquals(hashA, hashC); Assert.assertNotEquals(hashB, hashC); }
@Test(expected = NullPointerException.class) public void testNullContext() { Language lang = new Language("aLang"); new LanguageSample(lang, null); }
@Test public void testEquals() { Language langA = new Language("langA"); Language langB = new Language("langB"); Language langA5 = new Language("langA5", 5.0); Language langA6 = new Language("langA5", 6.0); Assert.assertEquals(langA, langA); Assert.assertEquals(langA5, langA5); Assert.assertNotEquals(langA, langA5); Assert.assertNotEquals(langA, langB); Assert.assertEquals(langA6, langA5); Assert.assertNotEquals(langA, "something else"); } }
@Test public void testHash() { int hashA = new Language("aLang").hashCode(); int hashAA = new Language("aLang").hashCode(); int hashB = new Language("BLang").hashCode(); int hashA5 = new Language("aLang", 5.0).hashCode(); int hashA6 = new Language("BLang", 6.0).hashCode(); Assert.assertEquals(hashA, hashAA); Assert.assertNotEquals(hashA, hashB); Assert.assertNotEquals(hashA, hashA5); Assert.assertNotEquals(hashB, hashA5); Assert.assertNotEquals(hashA5, hashA6); }
@Test public void testToString() { Language lang = new Language("aLang"); CharSequence context = "aContext"; LanguageSample sample = new LanguageSample(lang, context); Assert.assertEquals(lang.getLang() + "\t" + context, sample.toString()); }
evaluator.evaluateSample(new LanguageSample(new Language("pob"), "escreve e faz palestras pelo mundo inteiro sobre anjos")); evaluator.evaluateSample(new LanguageSample(new Language("fra"), "escreve e faz palestras pelo mundo inteiro sobre anjos")); evaluator.evaluateSample(new LanguageSample(new Language("fra"), "escreve e faz palestras pelo mundo inteiro sobre anjos"));
@Test public void testLanguageSampleSerDe() throws IOException { Language lang = new Language("aLang"); CharSequence context = "aContext"; LanguageSample languageSample = new LanguageSample(lang, context); ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); ObjectOutput out = new ObjectOutputStream(byteArrayOutputStream); out.writeObject(languageSample); out.flush(); byte[] bytes = byteArrayOutputStream.toByteArray(); ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); ObjectInput objectInput = new ObjectInputStream(byteArrayInputStream); LanguageSample deSerializedLanguageSample = null; try { deSerializedLanguageSample = (LanguageSample) objectInput.readObject(); } catch (ClassNotFoundException e) { // do nothing } Assert.assertNotNull(deSerializedLanguageSample); Assert.assertEquals(languageSample.getContext(), deSerializedLanguageSample.getContext()); Assert.assertEquals(languageSample.getLanguage(), deSerializedLanguageSample.getLanguage()); Assert.assertEquals(languageSample, deSerializedLanguageSample); }
@Test public void emptyConfidence() throws Exception { String languageCode = "aLanguage"; Language lang = new Language(languageCode); Assert.assertEquals(languageCode, lang.getLang()); Assert.assertEquals(0, lang.getConfidence(), 0); }
@Test public void nonEmptyConfidence() throws Exception { String languageCode = "aLanguage"; double confidence = 0.05; Language lang = new Language(languageCode, confidence); Assert.assertEquals(languageCode, lang.getLang()); Assert.assertEquals(confidence, lang.getConfidence(), 0); }
@Test public void testConstructor() { Language lang = new Language("aLang"); CharSequence context = "aContext"; LanguageSample sample = new LanguageSample(lang, context); Assert.assertEquals(lang, sample.getLanguage()); Assert.assertEquals(context, sample.getContext()); }
public LanguageSample read() throws IOException { String sampleString; while ((sampleString = samples.read()) != null) { int tabIndex = sampleString.indexOf("\t"); if (tabIndex > 0) { String lang = sampleString.substring(0, tabIndex); String context = sampleString.substring(tabIndex + 1); return new LanguageSample(new Language(lang), context); } } return null; } }
@Override public Language[] predictLanguages(CharSequence content) { double[] eval = model.getMaxentModel().eval(mContextGenerator.getContext(content.toString())); Language[] arr = new Language[eval.length]; for (int i = 0; i < eval.length; i++) { arr[i] = new Language(model.getMaxentModel().getOutcome(i), eval[i]); } Arrays.sort(arr, (o1, o2) -> Double.compare(o2.getConfidence(), o1.getConfidence())); return arr; }
@Override public Language[] predictLanguages(CharSequence content) { double[] eval = model.getMaxentModel().eval(mContextGenerator.getContext(content.toString())); Language[] arr = new Language[eval.length]; for (int i = 0; i < eval.length; i++) { arr[i] = new Language(model.getMaxentModel().getOutcome(i), eval[i]); } Arrays.sort(arr, (o1, o2) -> Double.compare(o2.getConfidence(), o1.getConfidence())); return arr; }