@Test public void testNullValue() { //encounter null value when building dictionary ArrayList<String> strs = new ArrayList<String>(); strs.add(null); strs.add("abc"); System.out.println(strs); int maxTreeSize = 0; TrieDictionaryForestBuilder<String> builder = newDictBuilder(strs, 0, maxTreeSize); TrieDictionaryForest<String> dict = builder.build(); dict.dump(System.out); //null value query int id = dict.getIdFromValue(null, 0); assertEquals(255, id); id = dict.getIdFromValue(null, 1); assertEquals(255, id); id = dict.getIdFromValue(null, -1); assertEquals(255, id); assertSameBehaviorAsTrie(dict, strs, 0); }
@Test public void testBasicFound() { ArrayList<String> strs = new ArrayList<String>(); strs.add("part"); strs.add("par"); strs.add("partition"); strs.add("party"); strs.add("parties"); strs.add("paint"); Collections.sort(strs); int baseId = 0; TrieDictionaryForestBuilder<String> builder = newDictBuilder(strs, baseId); TrieDictionaryForest<String> dict = builder.build(); dict.dump(System.out); int expectId = baseId; for (String s : strs) { assertEquals(expectId, dict.getIdFromValue(s)); expectId++; } assertSameBehaviorAsTrie(dict, strs, baseId); }
int id; try { id = dict.getIdFromValue(smallerStr, 0); fail("should throw IllegalArgumentException,but id is:" + id); } catch (IllegalArgumentException e) { id = dict.getIdFromValue(smallerStr, -1); fail("should throw IllegalArgumentException,but id is:" + id); } catch (IllegalArgumentException e) { id = dict.getIdFromValue(smallerStr, 1); assertEquals(baseId, id); id = dict.getIdFromValue(middleStr, 0); fail("should throw IllegalArgumentException,but id is:" + id); } catch (IllegalArgumentException e) { id = dict.getIdFromValue(middleStr, -1); assertEquals(baseId, id); id = dict.getIdFromValue(middleStr, 1); assertEquals(baseId + 1, id); id = dict.getIdFromValue(rightStr, 0); fail("should throw IllegalArgumentException,but id is:" + id); } catch (IllegalArgumentException e) { id = dict.getIdFromValue(rightStr, -1); assertEquals(baseId + 2, id); try {
int actualId = dict.getIdFromValue(value); assertTrue(actualId >= id); assertTrue(actualId > previousId);
newTrieBuilder.addValue(str); TrieDictionaryForest<String> newDict = newTrieBuilder.build(); keep |= newDict.getIdFromValue(testData.get(0)); newDictTotalBuildTime += (System.currentTimeMillis() - startTime); System.out.println("times:" + i);
@Test public void testAllNullValue() { ArrayList<String> strs = new ArrayList<String>(); strs.add(""); int maxTreeSize = 10; TrieDictionaryForestBuilder<String> builder = newDictBuilder(strs, 0, maxTreeSize); TrieDictionaryForest<String> dict = builder.build(); assertEquals(1, dict.getSize()); assertEquals(0, dict.getIdFromValue("")); }
@Test //one string one tree public void testMultiTree() { ArrayList<String> strs = new ArrayList<String>(); strs.add("part"); strs.add("par"); strs.add("partition"); strs.add("party"); strs.add("parties"); strs.add("paint"); strs.add("一二三"); //Chinese test strs.add("四五六"); strs.add(""); Collections.sort(strs, new ByteComparator<String>(new StringBytesConverter())); int baseId = 5; int maxTreeSize = 0; TrieDictionaryForestBuilder<String> builder = newDictBuilder(strs, baseId, maxTreeSize); TrieDictionaryForest<String> dict = builder.build(); dict.dump(System.out); assertEquals(strs.size(), dict.getTrees().size()); int expectId = baseId; for (String s : strs) { assertEquals(expectId, dict.getIdFromValue(s)); expectId++; } assertSameBehaviorAsTrie(dict, strs, baseId); }
@Test public void serializeTest() { List<String> testData = new ArrayList<>(); testData.add("1"); testData.add("2"); testData.add("100"); //TrieDictionaryForestBuilder.MaxTrieTreeSize = 0; NumberDictionaryForestBuilder b = new NumberDictionaryForestBuilder(); for (String str : testData) b.addValue(str); TrieDictionaryForest<String> dict = b.build(); dict = testSerialize(dict); //dict.dump(System.out); for (String str : testData) { assertEquals(str, dict.getValueFromId(dict.getIdFromValue(str))); } }
assertEquals(0, dict.getIdFromValue(entry.getKey()) - entry.getValue()); assertEquals(entry.getKey(), dict.getValueFromId(entry.getValue()));
private void testData(List<String> humanList, List<String> expectedList, SelfDefineSortableKey.TypeFlag flag) { //stimulate map-reduce job ArrayList<SelfDefineSortableKey> keyList = createKeyList(humanList, (byte) flag.ordinal()); Collections.sort(keyList); //build tree NumberDictionaryForestBuilder b = new NumberDictionaryForestBuilder(0, 0); expectedList = numberSort(expectedList); for (String value : expectedList) { b.addValue(value); } TrieDictionaryForest<String> dict = b.build(); //dict.dump(System.out); ArrayList<Integer> resultIds = new ArrayList<>(); for (int i = 0; i < keyList.size(); i++) { SelfDefineSortableKey key = keyList.get(i); String fieldValue = getFieldValue(key); resultIds.add(dict.getIdFromValue(fieldValue)); assertEquals(expectedList.get(i), dict.getValueFromId(dict.getIdFromValue(fieldValue))); } assertTrue(isIncreasedOrder(resultIds, new Comparator<Integer>() { @Override public int compare(Integer o1, Integer o2) { return o1.compareTo(o2); } })); }
assertEquals(id, dict.getIdFromValue(value)); assertEquals(value, dict.getValueFromId(id)); assertArrayEquals(Bytes.toBytes(value), dict.getValueByteFromId(id)); for (String s : notFound) { try { int nullId = dict.getIdFromValue(s); System.out.println("null value id:" + nullId); fail("For not found value '" + s + "', IllegalArgumentException is expected"); int nullId = dict.getIdFromValue(null); assertNull(dict.getValueFromId(nullId)); assertNull(dict.getValueByteFromId(nullId));
String query = rand.nextInt(2 * num) + ""; try { int id = dict.getIdFromValue(query, 1); assertEquals(set.ceiling(query), dict.getValueFromId(id)); } catch (IllegalArgumentException e) { String query = rand.nextInt(2 * num) + ""; try { int id = dict.getIdFromValue(query, -1); assertEquals(set.floor(query), dict.getValueFromId(id)); } catch (IllegalArgumentException e) {
int id1 = dict.getIdFromValue(query1, 1); String actualValue = dict.getValueFromId(id1); continue; try { int id = dict.getIdFromValue(query, 1); assertEquals(set.ceiling(query), dict.getValueFromId(id)); } catch (IllegalArgumentException e) { continue; try { int id = dict.getIdFromValue(query, -1); assertEquals(set.floor(query), dict.getValueFromId(id)); } catch (IllegalArgumentException e) {
@Test public void serializeTest() { ArrayList<String> testData = getTestData(10); TrieDictionaryForestBuilder<String> b = newDictBuilder(testData, 10, 0); TrieDictionaryForest<String> dict = b.build(); dict = testSerialize(dict); dict.dump(System.out); for (String str : testData) { assertEquals(str, dict.getValueFromId(dict.getIdFromValue(str))); } }
int id = dict.getIdFromValue(query, 1); assertEquals(set.ceiling(query), dict.getValueFromId(id)); } catch (IllegalArgumentException e) { String query = it3.next(); try { int id = dict.getIdFromValue(query, -1); assertEquals(set.floor(query), dict.getValueFromId(id)); } catch (IllegalArgumentException e) {
@Test public void emptyDictTest() throws Exception { TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter()); TrieDictionaryForest<String> dict = b.build(); try { int id = dict.getIdFromValue("123", 0); fail("id should not exist"); } catch (IllegalArgumentException e) { //right } try { String value = dict.getValueFromIdImpl(123); fail("value should not exist"); } catch (IllegalArgumentException e) { //right } }
@Test public void testMerge() { // mimic the logic as in MergeCuboidMapper NumberDictionaryForestBuilder b1 = new NumberDictionaryForestBuilder(); b1.addValue("0"); b1.addValue("3"); b1.addValue("23"); TrieDictionaryForest<String> dict1 = b1.build(); NumberDictionaryForestBuilder b2 = new NumberDictionaryForestBuilder(); b2.addValue("0"); b2.addValue("2"); b2.addValue("3"); b2.addValue("15"); b2.addValue("23"); TrieDictionaryForest<String> dict2 = b2.build(); assertTrue(dict1.getSizeOfId() == dict2.getSizeOfId()); assertTrue(dict1.getSizeOfValue() == dict2.getSizeOfValue()); { int newId = dict2.getIdFromValue(dict1.getValueFromId(0)); assertTrue(newId == 0); } { int newId = dict2.getIdFromValue(dict1.getValueFromId(1)); assertTrue(newId == 2); } { int newId = dict2.getIdFromValue(dict1.getValueFromId(2)); assertTrue(newId == 4); } }