/** * Convert byte array encoded with the {@link #toMetadataRowKey(TopicId)} method back to the {@link TopicId}. * Same as calling {@link #toTopicId(byte[], int, int)} with {@code offset = 0} * and {@code length = topicBytes.length}. * * @param topicBytes byte array which contains the representation of the topic id * @return {@link TopicId} */ public static TopicId toTopicId(byte[] topicBytes) { return toTopicId(topicBytes, 0, topicBytes.length); }
/** * Convert byte array encoded with the {@link #toMetadataRowKey(TopicId)} method back to the {@link TopicId}. * Same as calling {@link #toTopicId(byte[], int, int)} with {@code offset = 0} * and {@code length = topicBytes.length}. * * @param topicBytes byte array which contains the representation of the topic id * @return {@link TopicId} */ public static TopicId toTopicId(byte[] topicBytes) { return toTopicId(topicBytes, 0, topicBytes.length); }
public ImmutableMessageTableEntry(byte[] row, @Nullable byte[] payload, @Nullable byte[] txPtr) { this.topicId = MessagingUtils.toTopicId(row, 0, row.length - Bytes.SIZEOF_SHORT - Bytes.SIZEOF_LONG - Bytes.SIZEOF_INT); this.generation = Bytes.toInt(row, row.length - Bytes.SIZEOF_SHORT - Bytes.SIZEOF_LONG - Bytes.SIZEOF_INT); int topicLength = MessagingUtils.getTopicLengthMessageEntry(row.length); this.publishTimestamp = Bytes.toLong(row, topicLength); this.sequenceId = Bytes.toShort(row, topicLength + Bytes.SIZEOF_LONG); this.transactional = (txPtr != null); // since we mark tx as negative when tx is rolled back, we return the absolute value of tx this.transactionWritePointer = txPtr == null ? -1 : Math.abs(Bytes.toLong(txPtr)); this.payload = payload; }
@Override protected TopicMetadata computeNext() { if (closed || (!iterator.hasNext())) { return endOfData(); } Map.Entry<byte[], byte[]> entry = iterator.next(); TopicId topicId = MessagingUtils.toTopicId(entry.getKey()); Map<String, String> properties = GSON.fromJson(Bytes.toString(entry.getValue()), MAP_TYPE); return new TopicMetadata(topicId, properties); }
public ImmutableMessageTableEntry(byte[] row, @Nullable byte[] payload, @Nullable byte[] txPtr) { this.topicId = MessagingUtils.toTopicId(row, 0, row.length - Bytes.SIZEOF_SHORT - Bytes.SIZEOF_LONG - Bytes.SIZEOF_INT); this.generation = Bytes.toInt(row, row.length - Bytes.SIZEOF_SHORT - Bytes.SIZEOF_LONG - Bytes.SIZEOF_INT); int topicLength = MessagingUtils.getTopicLengthMessageEntry(row.length); this.publishTimestamp = Bytes.toLong(row, topicLength); this.sequenceId = Bytes.toShort(row, topicLength + Bytes.SIZEOF_LONG); this.transactional = (txPtr != null); // since we mark tx as negative when tx is rolled back, we return the absolute value of tx this.transactionWritePointer = txPtr == null ? -1 : Math.abs(Bytes.toLong(txPtr)); this.payload = payload; }
@Override protected TopicMetadata computeNext() { if (closed || (!iterator.hasNext())) { return endOfData(); } Map.Entry<byte[], byte[]> entry = iterator.next(); TopicId topicId = MessagingUtils.toTopicId(entry.getKey()); Map<String, String> properties = GSON.fromJson(Bytes.toString(entry.getValue()), MAP_TYPE); return new TopicMetadata(topicId, properties); }
public ImmutablePayloadTableEntry(byte[] row, byte[] payload) { this.topicId = MessagingUtils.toTopicId(row, 0, row.length - Bytes.SIZEOF_SHORT - (2 * Bytes.SIZEOF_LONG) - Bytes.SIZEOF_INT); this.generation = Bytes.toInt(row, row.length - Bytes.SIZEOF_SHORT - (2 * Bytes.SIZEOF_LONG) - Bytes.SIZEOF_INT); this.transactionWriterPointer = Bytes.toLong(row, row.length - Bytes.SIZEOF_SHORT - (2 * Bytes.SIZEOF_LONG)); this.writeTimestamp = Bytes.toLong(row, row.length - Bytes.SIZEOF_SHORT - Bytes.SIZEOF_LONG); this.sequenceId = Bytes.toShort(row, row.length - Bytes.SIZEOF_SHORT); this.payload = payload; }
public ImmutablePayloadTableEntry(byte[] row, byte[] payload) { this.topicId = MessagingUtils.toTopicId(row, 0, row.length - Bytes.SIZEOF_SHORT - (2 * Bytes.SIZEOF_LONG) - Bytes.SIZEOF_INT); this.generation = Bytes.toInt(row, row.length - Bytes.SIZEOF_SHORT - (2 * Bytes.SIZEOF_LONG) - Bytes.SIZEOF_INT); this.transactionWriterPointer = Bytes.toLong(row, row.length - Bytes.SIZEOF_SHORT - (2 * Bytes.SIZEOF_LONG)); this.writeTimestamp = Bytes.toLong(row, row.length - Bytes.SIZEOF_SHORT - Bytes.SIZEOF_LONG); this.sequenceId = Bytes.toShort(row, row.length - Bytes.SIZEOF_SHORT); this.payload = payload; }
/** * Scans the HBase table to get a list of {@link TopicId}. */ private List<TopicId> scanTopics(ScanBuilder scanBuilder) throws IOException { Scan scan = scanBuilder.setFilter(new FirstKeyOnlyFilter()).setCaching(scanCacheRows).build(); try { List<TopicId> topicIds = new ArrayList<>(); try (ResultScanner resultScanner = hTable.getScanner(scan)) { for (Result result : resultScanner) { TopicId topicId = MessagingUtils.toTopicId(result.getRow()); byte[] value = result.getValue(columnFamily, COL); Map<String, String> properties = GSON.fromJson(Bytes.toString(value), MAP_TYPE); TopicMetadata metadata = new TopicMetadata(topicId, properties); if (metadata.exists()) { topicIds.add(topicId); } } } return topicIds; } catch (IOException e) { throw exceptionHandler.handle(e); } }
/** * Scans the HBase table to get a list of {@link TopicId}. */ private List<TopicId> scanTopics(ScanBuilder scanBuilder) throws IOException { Scan scan = scanBuilder.setFilter(new FirstKeyOnlyFilter()).setCaching(scanCacheRows).build(); try { List<TopicId> topicIds = new ArrayList<>(); try (ResultScanner resultScanner = hTable.getScanner(scan)) { for (Result result : resultScanner) { TopicId topicId = MessagingUtils.toTopicId(result.getRow()); byte[] value = result.getValue(columnFamily, COL); Map<String, String> properties = GSON.fromJson(Bytes.toString(value), MAP_TYPE); TopicMetadata metadata = new TopicMetadata(topicId, properties); if (metadata.exists()) { topicIds.add(topicId); } } } return topicIds; } catch (IOException e) { throw exceptionHandler.handle(e); } }
@Override public ReturnCode filterKeyValue(Cell cell) throws IOException { int rowKeyOffset = cell.getRowOffset() + prefixLength; int sizeOfRowKey = cell.getRowLength() - prefixLength; long publishTimestamp = MessagingUtils.getPublishTimestamp(cell.getRowArray(), rowKeyOffset, sizeOfRowKey); int topicIdLength = MessagingUtils.getTopicLengthMessageEntry(sizeOfRowKey) - Bytes.SIZEOF_INT; int generationId = Bytes.toInt(cell.getRowArray(), rowKeyOffset + topicIdLength); if (prevTopicIdBytes == null || currentTTL == null || currentGen == null || (!Bytes.equals(prevTopicIdBytes, 0, prevTopicIdBytes.length, cell.getRowArray(), rowKeyOffset, topicIdLength))) { prevTopicIdBytes = Arrays.copyOfRange(cell.getRowArray(), rowKeyOffset, rowKeyOffset + topicIdLength); Map<String, String> properties = metadataCache.getTopicMetadata(ByteBuffer.wrap(prevTopicIdBytes)); if (properties == null) { LOG.debug("Region " + env.getRegionInfo().getRegionNameAsString() + ", could not get properties of topicId " + MessagingUtils.toTopicId(prevTopicIdBytes)); return ReturnCode.INCLUDE; } currentTTL = Long.parseLong(properties.get(MessagingUtils.Constants.TTL_KEY)); currentGen = Integer.parseInt(properties.get(MessagingUtils.Constants.GENERATION_KEY)); } // Old Generation (or deleted current generation) cleanup if (MessagingUtils.isOlderGeneration(generationId, currentGen)) { return ReturnCode.SKIP; } // TTL expiration cleanup only if the generation of metadata and row key are the same if ((generationId == currentGen) && ((timestamp - publishTimestamp) > currentTTL)) { return ReturnCode.SKIP; } return ReturnCode.INCLUDE; }
@Override public ReturnCode filterKeyValue(Cell cell) throws IOException { int rowKeyOffset = cell.getRowOffset() + prefixLength; int sizeOfRowKey = cell.getRowLength() - prefixLength; long publishTimestamp = MessagingUtils.getPublishTimestamp(cell.getRowArray(), rowKeyOffset, sizeOfRowKey); int topicIdLength = MessagingUtils.getTopicLengthMessageEntry(sizeOfRowKey) - Bytes.SIZEOF_INT; int generationId = Bytes.toInt(cell.getRowArray(), rowKeyOffset + topicIdLength); if (prevTopicIdBytes == null || currentTTL == null || currentGen == null || (!Bytes.equals(prevTopicIdBytes, 0, prevTopicIdBytes.length, cell.getRowArray(), rowKeyOffset, topicIdLength))) { prevTopicIdBytes = Arrays.copyOfRange(cell.getRowArray(), rowKeyOffset, rowKeyOffset + topicIdLength); Map<String, String> properties = metadataCache.getTopicMetadata(ByteBuffer.wrap(prevTopicIdBytes)); if (properties == null) { LOG.debug("Region " + env.getRegion().getRegionNameAsString() + ", could not get properties of topicId " + MessagingUtils.toTopicId(prevTopicIdBytes)); return ReturnCode.INCLUDE; } currentTTL = Long.parseLong(properties.get(MessagingUtils.Constants.TTL_KEY)); currentGen = Integer.parseInt(properties.get(MessagingUtils.Constants.GENERATION_KEY)); } // Old Generation (or deleted current generation) cleanup if (MessagingUtils.isOlderGeneration(generationId, currentGen)) { return ReturnCode.SKIP; } // TTL expiration cleanup only if the generation of metadata and row key are the same if ((generationId == currentGen) && ((timestamp - publishTimestamp) > currentTTL)) { return ReturnCode.SKIP; } return ReturnCode.INCLUDE; }
@Override public ReturnCode filterKeyValue(Cell cell) throws IOException { int rowKeyOffset = cell.getRowOffset() + prefixLength; int sizeOfRowKey = cell.getRowLength() - prefixLength; long publishTimestamp = MessagingUtils.getPublishTimestamp(cell.getRowArray(), rowKeyOffset, sizeOfRowKey); int topicIdLength = MessagingUtils.getTopicLengthMessageEntry(sizeOfRowKey) - Bytes.SIZEOF_INT; int generationId = Bytes.toInt(cell.getRowArray(), rowKeyOffset + topicIdLength); if (prevTopicIdBytes == null || currentTTL == null || currentGen == null || (!Bytes.equals(prevTopicIdBytes, 0, prevTopicIdBytes.length, cell.getRowArray(), rowKeyOffset, topicIdLength))) { prevTopicIdBytes = Arrays.copyOfRange(cell.getRowArray(), rowKeyOffset, rowKeyOffset + topicIdLength); Map<String, String> properties = metadataCache.getTopicMetadata(ByteBuffer.wrap(prevTopicIdBytes)); if (properties == null) { LOG.debug("Region " + env.getRegionInfo().getRegionNameAsString() + ", could not get properties of topicId " + MessagingUtils.toTopicId(prevTopicIdBytes)); return ReturnCode.INCLUDE; } currentTTL = Long.parseLong(properties.get(MessagingUtils.Constants.TTL_KEY)); currentGen = Integer.parseInt(properties.get(MessagingUtils.Constants.GENERATION_KEY)); } // Old Generation (or deleted current generation) cleanup if (MessagingUtils.isOlderGeneration(generationId, currentGen)) { return ReturnCode.SKIP; } // TTL expiration cleanup only if the generation of metadata and row key are the same if ((generationId == currentGen) && ((timestamp - publishTimestamp) > currentTTL)) { return ReturnCode.SKIP; } return ReturnCode.INCLUDE; }
if (properties == null) { LOG.debug("Region " + env.getRegionInfo().getRegionNameAsString() + ", could not get properties of topicId " + MessagingUtils.toTopicId(prevTopicIdBytes)); return ReturnCode.INCLUDE;
if (properties == null) { LOG.debug("Region " + env.getRegion().getRegionNameAsString() + ", could not get properties of topicId " + MessagingUtils.toTopicId(prevTopicIdBytes)); return ReturnCode.INCLUDE;
@Override public ReturnCode filterKeyValue(Cell cell) throws IOException { int rowKeyOffset = cell.getRowOffset() + prefixLength; int sizeOfRowKey = cell.getRowLength() - prefixLength; long publishTimestamp = MessagingUtils.getPublishTimestamp(cell.getRowArray(), rowKeyOffset, sizeOfRowKey); int topicIdLength = MessagingUtils.getTopicLengthMessageEntry(sizeOfRowKey) - Bytes.SIZEOF_INT; int generationId = Bytes.toInt(cell.getRowArray(), rowKeyOffset + topicIdLength); if (prevTopicIdBytes == null || currentTTL == null || currentGen == null || (!Bytes.equals(prevTopicIdBytes, 0, prevTopicIdBytes.length, cell.getRowArray(), rowKeyOffset, topicIdLength))) { prevTopicIdBytes = Arrays.copyOfRange(cell.getRowArray(), rowKeyOffset, rowKeyOffset + topicIdLength); Map<String, String> properties = metadataCache.getTopicMetadata(ByteBuffer.wrap(prevTopicIdBytes)); if (properties == null) { LOG.debug("Region " + env.getRegion().getRegionNameAsString() + ", could not get properties of topicId " + MessagingUtils.toTopicId(prevTopicIdBytes)); return ReturnCode.INCLUDE; } currentTTL = Long.parseLong(properties.get(MessagingUtils.Constants.TTL_KEY)); currentGen = Integer.parseInt(properties.get(MessagingUtils.Constants.GENERATION_KEY)); } // Old Generation (or deleted current generation) cleanup if (MessagingUtils.isOlderGeneration(generationId, currentGen)) { return ReturnCode.SKIP; } // TTL expiration cleanup only if the generation of metadata and row key are the same if ((generationId == currentGen) && ((timestamp - publishTimestamp) > currentTTL)) { return ReturnCode.SKIP; } return ReturnCode.INCLUDE; }
@Override public ReturnCode filterKeyValue(Cell cell) throws IOException { int rowKeyOffset = cell.getRowOffset() + prefixLength; int sizeOfRowKey = cell.getRowLength() - prefixLength; long publishTimestamp = MessagingUtils.getPublishTimestamp(cell.getRowArray(), rowKeyOffset, sizeOfRowKey); int topicIdLength = MessagingUtils.getTopicLengthMessageEntry(sizeOfRowKey) - Bytes.SIZEOF_INT; int generationId = Bytes.toInt(cell.getRowArray(), rowKeyOffset + topicIdLength); if (prevTopicIdBytes == null || currentTTL == null || currentGen == null || (!Bytes.equals(prevTopicIdBytes, 0, prevTopicIdBytes.length, cell.getRowArray(), rowKeyOffset, topicIdLength))) { prevTopicIdBytes = Arrays.copyOfRange(cell.getRowArray(), rowKeyOffset, rowKeyOffset + topicIdLength); Map<String, String> properties = metadataCache.getTopicMetadata(ByteBuffer.wrap(prevTopicIdBytes)); if (properties == null) { LOG.debug("Region " + env.getRegion().getRegionNameAsString() + ", could not get properties of topicId " + MessagingUtils.toTopicId(prevTopicIdBytes)); return ReturnCode.INCLUDE; } currentTTL = Long.parseLong(properties.get(MessagingUtils.Constants.TTL_KEY)); currentGen = Integer.parseInt(properties.get(MessagingUtils.Constants.GENERATION_KEY)); } // Old Generation (or deleted current generation) cleanup if (MessagingUtils.isOlderGeneration(generationId, currentGen)) { return ReturnCode.SKIP; } // TTL expiration cleanup only if the generation of metadata and row key are the same if ((generationId == currentGen) && ((timestamp - publishTimestamp) > currentTTL)) { return ReturnCode.SKIP; } return ReturnCode.INCLUDE; }
if (properties == null) { LOG.debug("Region " + env.getRegion().getRegionNameAsString() + ", could not get properties of topicId " + MessagingUtils.toTopicId(prevTopicIdBytes)); return ReturnCode.INCLUDE;
if (properties == null) { LOG.debug("Region " + env.getRegionInfo().getRegionNameAsString() + ", could not get properties of topicId " + MessagingUtils.toTopicId(prevTopicIdBytes)); return ReturnCode.INCLUDE;
@Test public void testTopicConversion() throws Exception { TopicId id = new TopicId("n1", "t1"); byte[] topicBytes = MessagingUtils.toMetadataRowKey(id); TopicId topicId = MessagingUtils.toTopicId(topicBytes); Assert.assertEquals(id, topicId); }