private Collection<Mutation> getMetricsMutations() { ImmutableList.Builder<Mutation> mutationBuilder = ImmutableList.builder(); // Mapping of column value to column to number of row IDs that contain that value for (Entry<MetricsKey, AtomicLong> entry : metrics.entrySet()) { // Row ID: Column value // Family: columnfamily_columnqualifier // Qualifier: CARDINALITY_CQ // Visibility: Inherited from indexed Mutation // Value: Cardinality Mutation mut = new Mutation(entry.getKey().row.array()); mut.put(entry.getKey().family.array(), CARDINALITY_CQ, entry.getKey().visibility, ENCODER.encode(entry.getValue().get())); // Add to our list of mutations mutationBuilder.add(mut); } // If the first row and last row are both not null, // which would really be for a brand new table that has zero rows and no indexed elements... // Talk about your edge cases! if (firstRow != null && lastRow != null) { // Add a some columns to the special metrics table row ID for the first/last row. // Note that if the values on the server side are greater/lesser, // the configured iterator will take care of this at scan/compaction time Mutation firstLastMutation = new Mutation(METRICS_TABLE_ROW_ID.array()); firstLastMutation.put(METRICS_TABLE_ROWS_CF.array(), METRICS_TABLE_FIRST_ROW_CQ.array(), firstRow); firstLastMutation.put(METRICS_TABLE_ROWS_CF.array(), METRICS_TABLE_LAST_ROW_CQ.array(), lastRow); mutationBuilder.add(firstLastMutation); } return mutationBuilder.build(); }
/** * Tests whether v remains the same when encoded and decoded with the given encoder. * * @throws IllegalArgumentException * if the test fails. */ public static <V> void testEncoder(Encoder<V> encoder, V v) { try { if (!v.equals(encoder.decode(encoder.encode(v)))) throw new IllegalArgumentException("something wrong with " + encoder.getClass().getName() + " -- doesn't encode and decode " + v + " properly"); } catch (ClassCastException e) { throw new IllegalArgumentException( encoder.getClass().getName() + " doesn't encode " + v.getClass().getName()); } }
/** * Tests whether v remains the same when encoded and decoded with the given encoder. * * @throws IllegalArgumentException * if the test fails. */ public static <V> void testEncoder(Encoder<V> encoder, V v) { try { if (!v.equals(encoder.decode(encoder.encode(v)))) throw new IllegalArgumentException("something wrong with " + encoder.getClass().getName() + " -- doesn't encode and decode " + v + " properly"); } catch (ClassCastException e) { throw new IllegalArgumentException( encoder.getClass().getName() + " doesn't encode " + v.getClass().getName()); } }
mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(3L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_T, new Text(datatype + "\u0000" + lcNoDiacriticsType.getClass().getName()), emptyValue); mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(3L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_T, new Text(datatype + "\u0000" + lcNoDiacriticsType.getClass().getName()), emptyValue); mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(3L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_T, new Text(datatype + "\u0000" + lcNoDiacriticsType.getClass().getName()), emptyValue); mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(3L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_T, new Text(datatype + "\u0000" + lcNoDiacriticsType.getClass().getName()), emptyValue);
mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(3L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_RI, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(10L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_RI, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(19L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_RI, new Text(datatype), emptyValue);
mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(3L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_T, new Text(datatype + "\u0000" + lcNoDiacriticsType.getClass().getName()), emptyValue); mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(3L))); bw.addMutation(mutation); mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(3L))); bw.addMutation(mutation); mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(3L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_T, new Text(datatype + "\u0000" + lcNoDiacriticsType.getClass().getName()), emptyValue); mutation.put(ColumnFamilyConstants.COLF_F, new Text(datatype + "\u0000" + date), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(3L))); mutation.put(ColumnFamilyConstants.COLF_I, new Text(datatype), emptyValue); mutation.put(ColumnFamilyConstants.COLF_T, new Text(datatype + "\u0000" + lcNoDiacriticsType.getClass().getName()), emptyValue);
SummingCombiner.VAR_LEN_ENCODER.encode(-1L)));
m.put(ColumnFamilyConstants.COLF_E, new Text(datatype), NULL_VALUE); m.put(ColumnFamilyConstants.COLF_F, new Text(datatype + NULL_BYTE + DateHelper.format(timestamp)), new Value(SummingCombiner.VAR_LEN_ENCODER.encode(1L)));
Mutation mut = new Mutation(column); mut.put(ColumnFamilyConstants.COLF_E, dtText, EMPTY_VALUE); Value colVal = new Value(SummingCombiner.VAR_LEN_ENCODER.encode((long) rawData.get(column).size())); mut.put(ColumnFamilyConstants.COLF_F, new Text(this.dataType + NULL_SEP + shardDate), colVal); if (this.fieldConfig.getIndexFields().contains(column)) {
private void prepNext() { Map.Entry<Key, Value> eA; Map.Entry<Text, Value> eB = itBreset.next(); if (!itBreset.hasNext()) { eA = itAonce.next(); // advance itA if (itAonce.hasNext()) // STOP if no more itA itBreset = BrowMap.tailMap(getTextAfterColumn()).entrySet().iterator(); else return; } else eA = itAonce.peek(); Text cola = eA.getKey().getColumnQualifier(); Key nk = new Key(cola, EMPTY_TEXT, eB.getKey()); long a = LEX.decode(eA.getValue().get()); long b = LEX.decode(eB.getValue().get()); long da = degMap.get(cola); long db = degMap.get(eB.getKey()); double nd = Math.min(((double)a)/da, ((double)b)/db); /// (da * db); // full calc is 1 - 2* Value nv = new Value(LEXDOUBLE.encode(nd)); //new Value(Double.toString(nd).getBytes(UTF_8)); nextEntry = new AbstractMap.SimpleImmutableEntry<>(nk, nv); }
public static Mutation buildMutation(ColumnVisibility cv, String path, boolean isDir, boolean isHidden, boolean canExec, long length, long lastmod, String hash) { if (path.equals("/")) path = ""; Mutation m = new Mutation(QueryUtil.getRow(path)); Text colf = null; if (isDir) colf = QueryUtil.DIR_COLF; else colf = new Text(encoder.encode(Long.MAX_VALUE - lastmod)); m.put(colf, new Text(LENGTH_CQ), cv, new Value(Long.toString(length).getBytes())); m.put(colf, new Text(HIDDEN_CQ), cv, new Value(Boolean.toString(isHidden).getBytes())); m.put(colf, new Text(EXEC_CQ), cv, new Value(Boolean.toString(canExec).getBytes())); m.put(colf, new Text(LASTMOD_CQ), cv, new Value(Long.toString(lastmod).getBytes())); if (hash != null && hash.length() > 0) m.put(colf, new Text(HASH_CQ), cv, new Value(hash.getBytes())); return m; }
public static void textAppend(Text t, long s) { t.append(nullByte, 0, 1); t.append(LongCombiner.FIXED_LEN_ENCODER.encode(s), 0, 8); }
@Override public Value getTopValue() { try { long count = 0; while (source.hasTop()) { count++; source.next(); } return new Value(LongCombiner.FIXED_LEN_ENCODER.encode(count)); } catch (IOException e) { throw new VertexiumAccumuloIteratorException("could not iterate", e); } }
public void map(List<Entry<Key,Value>> k, InputStream v, Context context) throws IOException, InterruptedException { Long[] hist = new Long[256]; for (int i = 0; i < hist.length; i++) hist[i] = 0l; int b = v.read(); while (b >= 0) { hist[b] += 1l; b = v.read(); } v.close(); Mutation m = new Mutation(k.get(0).getKey().getRow()); m.put("info", "hist", cv, new Value(SummingArrayCombiner.STRING_ARRAY_ENCODER.encode(Arrays.asList(hist)))); context.write(new Text(), m); }
public static void store(BigInteger machineId, long lastUsedTid) throws Exception { if (ZkSnowflakeCache.isInitialized()) { String timestampPath = String.format("/snowflake/hosts/%s/timestamp", machineId); Stat stat = curator.checkExists().forPath(timestampPath); byte[] newTid = LongCombiner.FIXED_LEN_ENCODER.encode(lastUsedTid); if (stat == null) { curator.create().creatingParentContainersIfNeeded().forPath(timestampPath, newTid); } else { curator.setData().forPath(timestampPath, newTid); } } else { LOGGER.error("ZkSnowflakeCache was not initialized"); throw new RuntimeException("ZkSnowflakeCache was not initialized"); } }
public static void textAppend(Text t, long s) { t.append(nullByte, 0, 1); t.append(SummingCombiner.FIXED_LEN_ENCODER.encode(s), 0, 8); }
@Override public Value getTopValue() { try { long count = 0; while (source.hasTop()) { count++; source.next(); } return new Value(LongCombiner.FIXED_LEN_ENCODER.encode(count)); } catch (IOException e) { throw new VertexiumAccumuloIteratorException("could not iterate", e); } }
protected void addToResults(Multimap<BulkIngestKey,Value> results, Long value, Key key, Text tableName) { BulkIngestKey bk = new BulkIngestKey(tableName, key); results.put(bk, new Value(SummingCombiner.VAR_LEN_ENCODER.encode(value))); }
@Override public Value reduce(Key key, Iterator<Value> iter) { return new Value(encoder.encode(typedReduce(key, new VIterator<>(iter, encoder, lossy)))); }
@Override public Value reduce(Key key, Iterator<Value> iter) { return new Value(encoder.encode(typedReduce(key, new VIterator<>(iter, encoder, lossy)))); }