/** * Tests whether v remains the same when encoded and decoded with the given encoder. * * @throws IllegalArgumentException * if the test fails. */ public static <V> void testEncoder(Encoder<V> encoder, V v) { try { if (!v.equals(encoder.decode(encoder.encode(v)))) throw new IllegalArgumentException("something wrong with " + encoder.getClass().getName() + " -- doesn't encode and decode " + v + " properly"); } catch (ClassCastException e) { throw new IllegalArgumentException( encoder.getClass().getName() + " doesn't encode " + v.getClass().getName()); } }
mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(10L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_RI, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(19L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_RI, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(11L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_RI, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(21L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_RI, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(12L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_RI, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(22L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_RI, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(12L))); mutation.put(ColumnFamilyConstants.COLF_T, new Text(datatype + "\u0000" + normalizerForColumn("AGE")), emptyValue); bw.addMutation(mutation); mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(3L)));
mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(3L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_T, new Text(datatype + "\u0000" + lcNoDiacriticsType.getClass().getName()), emptyValue); mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(3L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_T, new Text(datatype + "\u0000" + lcNoDiacriticsType.getClass().getName()), emptyValue); mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(3L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_T, new Text(datatype + "\u0000" + lcNoDiacriticsType.getClass().getName()), emptyValue); mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(3L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_T, new Text(datatype + "\u0000" + lcNoDiacriticsType.getClass().getName()), emptyValue);
mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(3L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_RI, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(10L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_RI, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(19L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_RI, new Text(datatype), emptyValue);
mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(3L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_T, new Text(datatype + "\u0000" + lcNoDiacriticsType.getClass().getName()), emptyValue); mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(3L))); bw.addMutation(mutation); mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(3L))); bw.addMutation(mutation); mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(3L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_T, new Text(datatype + "\u0000" + lcNoDiacriticsType.getClass().getName()), emptyValue); mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(3L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_T, new Text(datatype + "\u0000" + lcNoDiacriticsType.getClass().getName()), emptyValue);
SummingCombiner.VAR_LEN_ENCODER.encode(-1L)));
m.put(ColumnFamilyConstants.COLF_E, new Text(datatype), NULL_VALUE); m.put(ColumnFamilyConstants.COLF_F, new Text(datatype + NULL_BYTE + DateHelper.format(timestamp)), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(1L)));
Mutation mut = new Mutation(column); mut.put(ColumnFamilyConstants.COLF_E, dtText, EMPTY_VALUE); Value colVal = new Value(SummingCombiner.VAR_LEN_ENCODER.encode((long) rawData.get(column).size())); mut.put(ColumnFamilyConstants.COLF_F, new Text(this.dataType + NULL_SEP + shardDate), colVal); if (this.fieldConfig.getIndexFields().contains(column)) {
private long getRowCountFromTable(String tableName, Text signalColumn, Authorizations authorizations) { try { LOGGER.debug("BEGIN getRowCountFromTable(%s)", tableName); Scanner scanner = createScanner(tableName, null, authorizations); try { scanner.fetchColumnFamily(signalColumn); IteratorSetting countingIterator = new IteratorSetting( 100, CountingIterator.class.getSimpleName(), CountingIterator.class ); scanner.addScanIterator(countingIterator); GRAPH_LOGGER.logStartIterator(tableName, scanner); long count = 0; for (Map.Entry<Key, Value> entry : scanner) { Long countForKey = LongCombiner.FIXED_LEN_ENCODER.decode(entry.getValue().get()); LOGGER.debug("getRowCountFromTable(%s): %s: %d", tableName, entry.getKey().getRow(), countForKey); count += countForKey; } LOGGER.debug("getRowCountFromTable(%s): TOTAL: %d", tableName, count); return count; } finally { scanner.close(); } } catch (TableNotFoundException ex) { throw new VertexiumException("Could not get count from table: " + tableName, ex); } }
private long getRowCountFromTable(String tableName, Text signalColumn, Authorizations authorizations) { try { LOGGER.debug("BEGIN getRowCountFromTable(%s)", tableName); Scanner scanner = createScanner(tableName, null, authorizations); try { scanner.fetchColumnFamily(signalColumn); IteratorSetting countingIterator = new IteratorSetting( 100, CountingIterator.class.getSimpleName(), CountingIterator.class ); scanner.addScanIterator(countingIterator); GRAPH_LOGGER.logStartIterator(tableName, scanner); long count = 0; for (Map.Entry<Key, Value> entry : scanner) { Long countForKey = LongCombiner.FIXED_LEN_ENCODER.decode(entry.getValue().get()); LOGGER.debug("getRowCountFromTable(%s): %s: %d", tableName, entry.getKey().getRow(), countForKey); count += countForKey; } LOGGER.debug("getRowCountFromTable(%s): TOTAL: %d", tableName, count); return count; } finally { scanner.close(); } } catch (TableNotFoundException ex) { throw new VertexiumException("Could not get count from table: " + tableName, ex); } }
public static AtomKeyValueParser parse(Key key, Value value) throws IOException { AtomKeyValueParser atom = new AtomKeyValueParser(); String row = key.getRow().toString(); int splitPoint = row.indexOf(DELIMITER); if (splitPoint != -1) { atom.setCollectionName(row.substring(0, splitPoint)); int delim = key.getRow().find(DELIMITER); byte[] b = key.getRow().getBytes(); long diff = LongCombiner.FIXED_LEN_ENCODER.decode(Arrays.copyOfRange(b, delim + 1, key.getRow().getLength())); long time = (Long.MAX_VALUE - diff); atom.setUpdated(new Date(time)); } else { throw new IllegalArgumentException("Atom entry is missing row parts: " + key); } atom.setId(AtomKeyValueParser.encodeId(key.getColumnFamily().toString())); String colf = key.getColumnFamily().toString(); splitPoint = colf.indexOf(DELIMITER); if (splitPoint != -1 && splitPoint + 1 != colf.length()) { atom.setValue(colf.substring(0, splitPoint)); atom.setUuid(colf.substring(splitPoint + 1)); } else { throw new IllegalArgumentException("Atom entry is missing column qualifier parts: " + key); } atom.setColumnVisibility(key.getColumnQualifier().toString()); return atom; }
private void prepNext() { Map.Entry<Key, Value> eA; Map.Entry<Text, Value> eB = itBreset.next(); if (!itBreset.hasNext()) { eA = itAonce.next(); // advance itA if (itAonce.hasNext()) // STOP if no more itA itBreset = BrowMap.tailMap(getTextAfterColumn()).entrySet().iterator(); else return; } else eA = itAonce.peek(); Text cola = eA.getKey().getColumnQualifier(); Key nk = new Key(cola, EMPTY_TEXT, eB.getKey()); long a = LEX.decode(eA.getValue().get()); long b = LEX.decode(eB.getValue().get()); long da = degMap.get(cola); long db = degMap.get(eB.getKey()); double nd = Math.min(((double)a)/da, ((double)b)/db); /// (da * db); // full calc is 1 - 2* Value nv = new Value(LEXDOUBLE.encode(nd)); //new Value(Double.toString(nd).getBytes(UTF_8)); nextEntry = new AbstractMap.SimpleImmutableEntry<>(nk, nv); }
public static Mutation buildMutation(ColumnVisibility cv, String path, boolean isDir, boolean isHidden, boolean canExec, long length, long lastmod, String hash) { if (path.equals("/")) path = ""; Mutation m = new Mutation(QueryUtil.getRow(path)); Text colf = null; if (isDir) colf = QueryUtil.DIR_COLF; else colf = new Text(encoder.encode(Long.MAX_VALUE - lastmod)); m.put(colf, new Text(LENGTH_CQ), cv, new Value(Long.toString(length).getBytes())); m.put(colf, new Text(HIDDEN_CQ), cv, new Value(Boolean.toString(isHidden).getBytes())); m.put(colf, new Text(EXEC_CQ), cv, new Value(Boolean.toString(canExec).getBytes())); m.put(colf, new Text(LASTMOD_CQ), cv, new Value(Long.toString(lastmod).getBytes())); if (hash != null && hash.length() > 0) m.put(colf, new Text(HASH_CQ), cv, new Value(hash.getBytes())); return m; }
/** * Tests whether v remains the same when encoded and decoded with the given encoder. * * @throws IllegalArgumentException * if the test fails. */ public static <V> void testEncoder(Encoder<V> encoder, V v) { try { if (!v.equals(encoder.decode(encoder.encode(v)))) throw new IllegalArgumentException("something wrong with " + encoder.getClass().getName() + " -- doesn't encode and decode " + v + " properly"); } catch (ClassCastException e) { throw new IllegalArgumentException( encoder.getClass().getName() + " doesn't encode " + v.getClass().getName()); } }
@Override public boolean hasNext() { if (hasNext) return true; while (true) { if (!source.hasNext()) return false; try { next = encoder.decode(source.next().get()); return hasNext = true; } catch (ValueFormatException vfe) { if (!lossy) throw vfe; } } }
public static void store(BigInteger machineId, long lastUsedTid) throws Exception { if (ZkSnowflakeCache.isInitialized()) { String timestampPath = String.format("/snowflake/hosts/%s/timestamp", machineId); Stat stat = curator.checkExists().forPath(timestampPath); byte[] newTid = LongCombiner.FIXED_LEN_ENCODER.encode(lastUsedTid); if (stat == null) { curator.create().creatingParentContainersIfNeeded().forPath(timestampPath, newTid); } else { curator.setData().forPath(timestampPath, newTid); } } else { LOGGER.error("ZkSnowflakeCache was not initialized"); throw new RuntimeException("ZkSnowflakeCache was not initialized"); } }
public void map(List<Entry<Key,Value>> k, InputStream v, Context context) throws IOException, InterruptedException { Long[] hist = new Long[256]; for (int i = 0; i < hist.length; i++) hist[i] = 0l; int b = v.read(); while (b >= 0) { hist[b] += 1l; b = v.read(); } v.close(); Mutation m = new Mutation(k.get(0).getKey().getRow()); m.put("info", "hist", cv, new Value(SummingArrayCombiner.STRING_ARRAY_ENCODER.encode(Arrays.asList(hist)))); context.write(new Text(), m); }
@Override public Value getTopValue() { try { long count = 0; while (source.hasTop()) { count++; source.next(); } return new Value(LongCombiner.FIXED_LEN_ENCODER.encode(count)); } catch (IOException e) { throw new VertexiumAccumuloIteratorException("could not iterate", e); } }
public static void textAppend(Text t, long s) { t.append(nullByte, 0, 1); t.append(LongCombiner.FIXED_LEN_ENCODER.encode(s), 0, 8); }
protected void addToResults(Multimap<BulkIngestKey,Value> results, Long value, Key key, Text tableName) { BulkIngestKey bk = new BulkIngestKey(tableName, key); results.put(bk, new Value(SummingCombiner.VAR_LEN_ENCODER.encode(value))); }