/** * {@inheritDoc} */ @Override public void put(Put put) throws IOException { byte[] row = put.getRow(); NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> rowData = forceFind(data, row, new TreeMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>>(Bytes.BYTES_COMPARATOR)); for (byte[] family : put.getFamilyMap().keySet()) { if (columnFamilies.contains(new String(family, StandardCharsets.UTF_8)) == false) { throw new RuntimeException("Not Exists columnFamily : " + new String(family, StandardCharsets.UTF_8)); } NavigableMap<byte[], NavigableMap<Long, byte[]>> familyData = forceFind(rowData, family, new TreeMap<byte[], NavigableMap<Long, byte[]>>(Bytes.BYTES_COMPARATOR)); for (KeyValue kv : put.getFamilyMap().get(family)) { kv.updateLatestStamp(Bytes.toBytes(System.currentTimeMillis())); byte[] qualifier = kv.getQualifier(); NavigableMap<Long, byte[]> qualifierData = forceFind(familyData, qualifier, new TreeMap<Long, byte[]>()); qualifierData.put(kv.getTimestamp(), kv.getValue()); } } }
rowResult.put( Bytes.toString(kv.getQualifier()), new ByteArrayByteIterator(kv.getValue()));
@SuppressWarnings("deprecation") @Test @Ignore("not maintaining") public void testMapper2() throws IOException { mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cube_name); mapDriver.addInput(new Text("52010tech"), new Text("35.432")); List<Pair<RowKeyWritable, KeyValue>> result = mapDriver.run(); assertEquals(2, result.size()); byte[] bytes = { 0, 0, 0, 0, 0, 0, 0, 119, 33, 0, 22, 1, 0, 121, 7 }; ImmutableBytesWritable key = new ImmutableBytesWritable(bytes); Pair<RowKeyWritable, KeyValue> p1 = result.get(0); Pair<RowKeyWritable, KeyValue> p2 = result.get(1); assertEquals(key, p1.getFirst()); assertEquals("cf1", new String(p1.getSecond().getFamily(), StandardCharsets.UTF_8)); assertEquals("usd_amt", new String(p1.getSecond().getQualifier(), StandardCharsets.UTF_8)); assertEquals("35.43", new String(p1.getSecond().getValue(), StandardCharsets.UTF_8)); assertEquals(key, p2.getFirst()); assertEquals("cf1", new String(p2.getSecond().getFamily(), StandardCharsets.UTF_8)); assertEquals("item_count", new String(p2.getSecond().getQualifier(), StandardCharsets.UTF_8)); assertEquals("2", new String(p2.getSecond().getValue(), StandardCharsets.UTF_8)); } }
for (Result result = scanner.next(); (result != null); result = scanner.next()) { for(KeyValue keyValue : result.list()) { System.out.println("Qualifier : " + keyValue.getKeyString() + " : Value : " + Bytes.toString(keyValue.getValue())); } }
private static byte[] getMutationKVByteValue(Mutation headerRow, byte[] key) { KeyValue kv = getMutationKeyValue(headerRow, key); // FIXME: byte copy return kv == null ? ByteUtil.EMPTY_BYTE_ARRAY : kv.getValue(); }
private String toString(KeyValue kv) { return kv.toString() + "/value=" + Bytes.toString(kv.getValue()); }
public String loadString(String key) throws Exception { Get keyGet = new Get(key.getBytes()); HTable htable = HTableFactory.getHTable(configMap); Result res = htable.get(keyGet); KeyValue kv = res.getColumnLatest(configMap.get(HTableFactory.FAMILY_TAG).getBytes(), configMap.get(HTableFactory.QUALIFIER_TAG).getBytes()); if (kv == null) { throw new NoSuchElementException("No value found for " + key); } byte[] bytes = kv.getValue(); String jsonStr = new String(bytes); return jsonStr; }
@Override public String toString() { if(keyValue != null) { return keyValue.toString() + " value = " + Bytes.toStringBinary(keyValue.getValue()); } else { return super.toString(); } } }
public void setColumns(List<KeyValue> columns) { for(KeyValue column : columns) { putColumn(column.getQualifier(), column.getValue()); } }
@Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("keyvalues="); if(this.result.isEmpty()) { sb.append("NONE"); return sb.toString(); } sb.append("{"); boolean moreThanOne = false; for(KeyValue kv : this.result.list()) { if(moreThanOne) { sb.append(", \n"); } else { moreThanOne = true; } sb.append(kv.toString()+"/value="+Bytes.toString(kv.getValue())); } sb.append("}\n"); return sb.toString(); }
@Override public String toString() { StringBuffer sb = new StringBuffer("Pending Index Updates:\n"); for (Entry<ImmutableBytesPtr, Collection<Mutation>> entry : map.entrySet()) { String tableName = Bytes.toString(entry.getKey().get()); sb.append(" Table: '" + tableName + "'\n"); for (Mutation m : entry.getValue()) { sb.append("\t"); if (shouldBeRemoved(m)) { sb.append("[REMOVED]"); } sb.append(m.getClass().getSimpleName() + ":" + ((m instanceof Put) ? m.getTimeStamp() + " " : "")); sb.append(" row=" + Bytes.toString(m.getRow())); sb.append("\n"); if (m.getFamilyMap().isEmpty()) { sb.append("\t\t=== EMPTY ===\n"); } for (List<KeyValue> kvs : m.getFamilyMap().values()) { for (KeyValue kv : kvs) { sb.append("\t\t" + kv.toString() + "/value=" + Bytes.toStringBinary(kv.getValue())); sb.append("\n"); } } } } return sb.toString(); } }
@Override public void verify(TableState state) { try { Scanner kvs = ((LocalTableState) state).getIndexedColumnsTableState(Arrays.asList(columns)).getFirst(); int count = 0; KeyValue kv; while ((kv = kvs.next()) != null) { KeyValue next = expectedKvs.get(count++); assertEquals( msg + ": Unexpected kv in table state!\nexpected v1: " + Bytes.toString(next.getValue()) + "\nactual v1:" + Bytes.toString(kv.getValue()), next, kv); } assertEquals(msg + ": Didn't find enough kvs in table state!", expectedKvs.size(), count); } catch (IOException e) { fail(msg + ": Got an exception while reading local table state! " + e.getMessage()); } } }
/** * Does a deep comparison of two Results, down to the byte arrays. * @param res1 first result to compare * @param res2 second result to compare * @throws Exception Every difference is throwing an exception */ public static void compareTuples(Tuple res1, Tuple res2) throws Exception { if (res2 == null) { throw new Exception("There wasn't enough rows, we stopped at " + res1); } if (res1.size() != res2.size()) { throw new Exception("This row doesn't have the same number of KVs: " + res1.toString() + " compared to " + res2.toString()); } for (int i = 0; i < res1.size(); i++) { KeyValue ourKV = res1.getValue(i); KeyValue replicatedKV = res2.getValue(i); if (!ourKV.equals(replicatedKV) || !Bytes.equals(ourKV.getValue(), replicatedKV.getValue())) { throw new Exception("This result was different: " + res1.toString() + " compared to " + res2.toString()); } } }
private static void assertRow2(Iterator<KeyValue> iterator, byte[] expectedRowKey2) { KeyValue kv; assertTrue(iterator.hasNext()); kv = iterator.next(); assertArrayEquals(expectedRowKey2, kv.getRow()); assertEquals(name2, PDataType.VARCHAR.toObject(kv.getValue())); assertTrue(iterator.hasNext()); kv = iterator.next(); assertArrayEquals(expectedRowKey2, kv.getRow()); assertEquals(now, PDataType.DATE.toObject(kv.getValue())); assertTrue(iterator.hasNext()); kv = iterator.next(); assertArrayEquals(expectedRowKey2, kv.getRow()); assertNull(PDataType.VARCHAR.toObject(kv.getValue())); }
private static void assertRow1(Iterator<KeyValue> iterator, byte[] expectedRowKey1) { KeyValue kv; assertTrue(iterator.hasNext()); kv = iterator.next(); assertArrayEquals(expectedRowKey1, kv.getRow()); assertEquals(name1, PDataType.VARCHAR.toObject(kv.getValue())); assertTrue(iterator.hasNext()); kv = iterator.next(); assertArrayEquals(expectedRowKey1, kv.getRow()); assertEquals(now, PDataType.DATE.toObject(kv.getValue())); assertTrue(iterator.hasNext()); kv = iterator.next(); assertArrayEquals(expectedRowKey1, kv.getRow()); assertNull(PDataType.VARCHAR.toObject(kv.getValue())); }
@Override public void put(Put put) throws IOException { addToPutLog(put); byte[] row = put.getRow(); NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> rowData = forceFind(data, row, new TreeMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>>(Bytes.BYTES_COMPARATOR)); for (byte[] family : put.getFamilyMap().keySet()){ NavigableMap<byte[], NavigableMap<Long, byte[]>> familyData = forceFind(rowData, family, new TreeMap<byte[], NavigableMap<Long, byte[]>>(Bytes.BYTES_COMPARATOR)); for (KeyValue kv : put.getFamilyMap().get(family)){ kv.updateLatestStamp(Bytes.toBytes(System.currentTimeMillis())); byte[] qualifier = kv.getQualifier(); NavigableMap<Long, byte[]> qualifierData = forceFind(familyData, qualifier, new TreeMap<Long, byte[]>()); qualifierData.put(kv.getTimestamp(), kv.getValue()); } } }
private void assertNResult(Result result, byte[] row, byte[] family, byte[] qualifier, long[] stamps, byte[][] values, int start, int end) throws IOException { assertTrue( "Expected row [" + Bytes.toString(row) + "] " + "Got row [" + Bytes.toString(result.getRow()) + "]", equals(row, result.getRow())); int expectedResults = end - start + 1; assertEquals(expectedResults, result.size()); KeyValue[] keys = result.raw(); for (int i = 0; i < keys.length; i++) { byte[] value = values[end - i]; long ts = stamps[end - i]; KeyValue key = keys[i]; assertTrue("(" + i + ") Expected family [" + Bytes.toString(family) + "] " + "Got family [" + Bytes.toString(key.getFamily()) + "]", equals(family, key.getFamily())); assertTrue("(" + i + ") Expected qualifier [" + Bytes.toString(qualifier) + "] " + "Got qualifier [" + Bytes.toString(key.getQualifier()) + "]", equals(qualifier, key.getQualifier())); assertTrue("Expected ts [" + ts + "] " + "Got ts [" + key.getTimestamp() + "]", ts == key.getTimestamp()); assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] " + "Got value [" + Bytes.toString(key.getValue()) + "]", equals(value, key.getValue())); } }
@SuppressWarnings("deprecation") @Test @Ignore("not maintaining") public void testMapper2() throws IOException { mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cube_name); mapDriver.addInput(new Text("52010tech"), new Text("35.432")); List<Pair<ImmutableBytesWritable, KeyValue>> result = mapDriver.run(); assertEquals(2, result.size()); byte[] bytes = { 0, 0, 0, 0, 0, 0, 0, 119, 33, 0, 22, 1, 0, 121, 7 }; ImmutableBytesWritable key = new ImmutableBytesWritable(bytes); Pair<ImmutableBytesWritable, KeyValue> p1 = result.get(0); Pair<ImmutableBytesWritable, KeyValue> p2 = result.get(1); assertEquals(key, p1.getFirst()); assertEquals("cf1", new String(p1.getSecond().getFamily())); assertEquals("usd_amt", new String(p1.getSecond().getQualifier())); assertEquals("35.43", new String(p1.getSecond().getValue())); assertEquals(key, p2.getFirst()); assertEquals("cf1", new String(p2.getSecond().getFamily())); assertEquals("item_count", new String(p2.getSecond().getQualifier())); assertEquals("2", new String(p2.getSecond().getValue())); } }
private void validate(KeyValue kv, byte[] row, byte[] family, byte[] qualifier, long ts, Type type, byte[] value) throws IOException { DataOutputBuffer out = new DataOutputBuffer(); kv.write(out); out.close(); byte[] data = out.getData(); // read it back in KeyValue read = new KeyValue(); DataInputBuffer in = new DataInputBuffer(); in.reset(data, data.length); read.readFields(in); in.close(); // validate that its the same assertTrue("Row didn't match!", Bytes.equals(row, read.getRow())); assertTrue("Family didn't match!", Bytes.equals(family, read.getFamily())); assertTrue("Qualifier didn't match!", Bytes.equals(qualifier, read.getQualifier())); assertTrue("Value didn't match!", Bytes.equals(value, read.getValue())); assertEquals("Timestamp didn't match", ts, read.getTimestamp()); assertEquals("Type didn't match", type.getCode(), read.getType()); }