@Override public boolean addValue(String value) { if (StringUtils.isBlank(value)) // empty string is treated as null return false; builder.addValue(value); return true; }
@Ignore @Test public void memoryUsageBenchmarkOldDictTest() throws Exception { System.out.println("max memory:" + Runtime.getRuntime().maxMemory()); System.gc(); Thread.currentThread().sleep(1000); NumberDictionaryBuilder b = new NumberDictionaryBuilder(); int k = 0; while (true) { b.addValue(k + ""); if (k % 100000 == 0) System.out.println(k); k++; } //memroy:1908932608 entry:17500000 }
@Test public void testVerySmallDouble() { List<String> testData = new ArrayList<>(); testData.add(-1.0 + ""); testData.add(Double.MIN_VALUE + ""); testData.add("1.01"); testData.add("2.0"); NumberDictionaryForestBuilder b = new NumberDictionaryForestBuilder(); for (String str : testData) b.addValue(str); TrieDictionaryForest<String> dict = b.build(); //dict.dump(System.out); NumberDictionaryBuilder b2 = new NumberDictionaryBuilder(); for (String str : testData) b2.addValue(str); NumberDictionary<String> dict2 = b2.build(0); //dict2.dump(System.out); }
@Test public void testMinMax() { NumberDictionaryBuilder builder = new NumberDictionaryBuilder(); builder.addValue("" + Long.MAX_VALUE); builder.addValue("" + Long.MIN_VALUE); NumberDictionary<String> dict = builder.build(0); int minId = dict.getIdFromValue("" + Long.MIN_VALUE); int maxId = dict.getIdFromValue("" + Long.MAX_VALUE); assertEquals(0, minId); assertEquals(1, maxId); }
String num = randNumber(); if (set.add(new BigDecimal(num))) { builder.addValue(num);
private static Dictionary buildNumberDict(List<byte[]> values, int baseId, int nSamples, ArrayList samples) { NumberDictionaryBuilder builder = new NumberDictionaryBuilder(new StringBytesConverter()); for (byte[] value : values) { String v = Bytes.toString(value); builder.addValue(v); if (samples.size() < nSamples && samples.contains(v) == false) samples.add(v); } return builder.build(baseId); }
String num = randNumber(); if (set.add(new BigDecimal(num))) { builder.addValue(num);
@Override public boolean addValue(String value) { if (StringUtils.isBlank(value)) // empty string is treated as null return false; builder.addValue(value); return true; }
private static org.apache.kylin.common.util.Dictionary buildNumberDict(IDictionaryValueEnumerator valueEnumerator, int baseId, int nSamples, ArrayList samples) throws IOException { NumberDictionaryBuilder builder = new NumberDictionaryBuilder(new StringBytesConverter()); byte[] value; while (valueEnumerator.moveNext()) { value = valueEnumerator.current(); if (value == null) continue; String v = Bytes.toString(value); if (StringUtils.isBlank(v)) // empty string is null for numbers continue; builder.addValue(v); if (samples.size() < nSamples && !samples.contains(v)) samples.add(v); } return builder.build(baseId); }