/** * Convert byte array encoded with the {@link #toMetadataRowKey(TopicId)} method back to the {@link TopicId}. * Same as calling {@link #toTopicId(byte[], int, int)} with {@code offset = 0} * and {@code length = topicBytes.length}. * * @param topicBytes byte array which contains the representation of the topic id * @return {@link TopicId} */ public static TopicId toTopicId(byte[] topicBytes) { return toTopicId(topicBytes, 0, topicBytes.length); }
public ImmutableMessageTableEntry(byte[] row, @Nullable byte[] payload, @Nullable byte[] txPtr) { this.topicId = MessagingUtils.toTopicId(row, 0, row.length - Bytes.SIZEOF_SHORT - Bytes.SIZEOF_LONG - Bytes.SIZEOF_INT); this.generation = Bytes.toInt(row, row.length - Bytes.SIZEOF_SHORT - Bytes.SIZEOF_LONG - Bytes.SIZEOF_INT); int topicLength = MessagingUtils.getTopicLengthMessageEntry(row.length); this.publishTimestamp = Bytes.toLong(row, topicLength); this.sequenceId = Bytes.toShort(row, topicLength + Bytes.SIZEOF_LONG); this.transactional = (txPtr != null); // since we mark tx as negative when tx is rolled back, we return the absolute value of tx this.transactionWritePointer = txPtr == null ? -1 : Math.abs(Bytes.toLong(txPtr)); this.payload = payload; }
/** * Fetch the write timestamp from the payload table row key. * * @param payloadTableRowKey byte array containing payload table row key * @param offset start of the row key in the byte array * @param rowKeyLength length of the row key byte array * @return write timestamp */ public static long getWriteTimestamp(byte[] payloadTableRowKey, int offset, int rowKeyLength) { return Bytes.toLong(payloadTableRowKey, offset + getTopicLengthPayloadEntry(rowKeyLength) + Bytes.SIZEOF_LONG); }
@Override public ReturnCode filterKeyValue(Cell cell) throws IOException { int rowKeyOffset = cell.getRowOffset() + prefixLength; int sizeOfRowKey = cell.getRowLength() - prefixLength; long publishTimestamp = MessagingUtils.getPublishTimestamp(cell.getRowArray(), rowKeyOffset, sizeOfRowKey); int topicIdLength = MessagingUtils.getTopicLengthMessageEntry(sizeOfRowKey) - Bytes.SIZEOF_INT; int generationId = Bytes.toInt(cell.getRowArray(), rowKeyOffset + topicIdLength); if (prevTopicIdBytes == null || currentTTL == null || currentGen == null || (!Bytes.equals(prevTopicIdBytes, 0, prevTopicIdBytes.length, cell.getRowArray(), rowKeyOffset, topicIdLength))) { prevTopicIdBytes = Arrays.copyOfRange(cell.getRowArray(), rowKeyOffset, rowKeyOffset + topicIdLength); Map<String, String> properties = metadataCache.getTopicMetadata(ByteBuffer.wrap(prevTopicIdBytes)); if (properties == null) { LOG.debug("Region " + env.getRegionInfo().getRegionNameAsString() + ", could not get properties of topicId " + MessagingUtils.toTopicId(prevTopicIdBytes)); return ReturnCode.INCLUDE; } currentTTL = Long.parseLong(properties.get(MessagingUtils.Constants.TTL_KEY)); currentGen = Integer.parseInt(properties.get(MessagingUtils.Constants.GENERATION_KEY)); } // Old Generation (or deleted current generation) cleanup if (MessagingUtils.isOlderGeneration(generationId, currentGen)) { return ReturnCode.SKIP; } // TTL expiration cleanup only if the generation of metadata and row key are the same if ((generationId == currentGen) && ((timestamp - publishTimestamp) > currentTTL)) { return ReturnCode.SKIP; } return ReturnCode.INCLUDE; }
int rowKeyOffset = cell.getRowOffset() + prefixLength; int sizeOfRowKey = cell.getRowLength() - prefixLength; long writeTimestamp = MessagingUtils.getWriteTimestamp(cell.getRowArray(), rowKeyOffset, sizeOfRowKey); int topicIdLength = MessagingUtils.getTopicLengthPayloadEntry(sizeOfRowKey) - Bytes.SIZEOF_INT; int generationId = Bytes.toInt(cell.getRowArray(), rowKeyOffset + topicIdLength); if (properties == null) { LOG.debug("Region " + env.getRegion().getRegionNameAsString() + ", could not get properties of topicId " + MessagingUtils.toTopicId(prevTopicIdBytes)); return ReturnCode.INCLUDE; if (MessagingUtils.isOlderGeneration(generationId, currentGen)) { return ReturnCode.SKIP;
WriteBatch writeBatch = levelDB.createWriteBatch(); long ttlInMs = TimeUnit.SECONDS.toMillis(topicMetadata.getTTL()); byte[] startRow = MessagingUtils.toDataKeyPrefix(topicMetadata.getTopicId(), Integer.parseInt(MessagingUtils.Constants.DEFAULT_GENERATION)); byte[] stopRow = Bytes.stopKeyForPrefix(startRow); if (MessagingUtils.isOlderGeneration(dataGeneration, currGeneration)) { writeBatch.delete(entry.getKey()); continue;
@Test public void testTopicConversion() throws Exception { TopicId id = new TopicId("n1", "t1"); byte[] topicBytes = MessagingUtils.toMetadataRowKey(id); TopicId topicId = MessagingUtils.toTopicId(topicBytes); Assert.assertEquals(id, topicId); }
public TopicIdMetaKey(TopicId topicId) { this.topicId = topicId; this.key = MessagingUtils.toMetadataRowKey(topicId); }
/** * Fetch the publish timestamp from the message table row key. * * @param messageTableRowKey byte array containing message table row key * @param offset start of the row key in the byte array * @param rowKeyLength length of the row key byte array * @return publish timestamp */ public static long getPublishTimestamp(byte[] messageTableRowKey, int offset, int rowKeyLength) { return Bytes.toLong(messageTableRowKey, offset + getTopicLengthMessageEntry(rowKeyLength)); }
@Test public void testGenerations() throws Exception { Assert.assertTrue(MessagingUtils.isOlderGeneration(3, 5)); Assert.assertTrue(MessagingUtils.isOlderGeneration(3, -3)); Assert.assertFalse(MessagingUtils.isOlderGeneration(6, 5)); Assert.assertFalse(MessagingUtils.isOlderGeneration(6, 6)); Assert.assertFalse(MessagingUtils.isOlderGeneration(6, -5)); }
@Override public ReturnCode filterKeyValue(Cell cell) throws IOException { int rowKeyOffset = cell.getRowOffset() + prefixLength; int sizeOfRowKey = cell.getRowLength() - prefixLength; long publishTimestamp = MessagingUtils.getPublishTimestamp(cell.getRowArray(), rowKeyOffset, sizeOfRowKey); int topicIdLength = MessagingUtils.getTopicLengthMessageEntry(sizeOfRowKey) - Bytes.SIZEOF_INT; int generationId = Bytes.toInt(cell.getRowArray(), rowKeyOffset + topicIdLength); if (prevTopicIdBytes == null || currentTTL == null || currentGen == null || (!Bytes.equals(prevTopicIdBytes, 0, prevTopicIdBytes.length, cell.getRowArray(), rowKeyOffset, topicIdLength))) { prevTopicIdBytes = Arrays.copyOfRange(cell.getRowArray(), rowKeyOffset, rowKeyOffset + topicIdLength); Map<String, String> properties = metadataCache.getTopicMetadata(ByteBuffer.wrap(prevTopicIdBytes)); if (properties == null) { LOG.debug("Region " + env.getRegion().getRegionNameAsString() + ", could not get properties of topicId " + MessagingUtils.toTopicId(prevTopicIdBytes)); return ReturnCode.INCLUDE; } currentTTL = Long.parseLong(properties.get(MessagingUtils.Constants.TTL_KEY)); currentGen = Integer.parseInt(properties.get(MessagingUtils.Constants.GENERATION_KEY)); } // Old Generation (or deleted current generation) cleanup if (MessagingUtils.isOlderGeneration(generationId, currentGen)) { return ReturnCode.SKIP; } // TTL expiration cleanup only if the generation of metadata and row key are the same if ((generationId == currentGen) && ((timestamp - publishTimestamp) > currentTTL)) { return ReturnCode.SKIP; } return ReturnCode.INCLUDE; }
int rowKeyOffset = cell.getRowOffset() + prefixLength; int sizeOfRowKey = cell.getRowLength() - prefixLength; long writeTimestamp = MessagingUtils.getWriteTimestamp(cell.getRowArray(), rowKeyOffset, sizeOfRowKey); int topicIdLength = MessagingUtils.getTopicLengthPayloadEntry(sizeOfRowKey) - Bytes.SIZEOF_INT; int generationId = Bytes.toInt(cell.getRowArray(), rowKeyOffset + topicIdLength); if (properties == null) { LOG.debug("Region " + env.getRegionInfo().getRegionNameAsString() + ", could not get properties of topicId " + MessagingUtils.toTopicId(prevTopicIdBytes)); return ReturnCode.INCLUDE; if (MessagingUtils.isOlderGeneration(generationId, currentGen)) { return ReturnCode.SKIP;
WriteBatch writeBatch = levelDB.createWriteBatch(); long ttlInMs = TimeUnit.SECONDS.toMillis(topicMetadata.getTTL()); byte[] startRow = MessagingUtils.toDataKeyPrefix(topicMetadata.getTopicId(), Integer.parseInt(MessagingUtils.Constants.DEFAULT_GENERATION)); byte[] stopRow = Bytes.stopKeyForPrefix(startRow); if (MessagingUtils.isOlderGeneration(dataGeneration, currGeneration)) { writeBatch.delete(entry.getKey()); continue;
public TopicIdMetaKey(TopicId topicId) { this.topicId = topicId; this.key = MessagingUtils.toMetadataRowKey(topicId); }
/** * Fetch the publish timestamp from the message table row key. * * @param messageTableRowKey byte array containing message table row key * @param offset start of the row key in the byte array * @param rowKeyLength length of the row key byte array * @return publish timestamp */ public static long getPublishTimestamp(byte[] messageTableRowKey, int offset, int rowKeyLength) { return Bytes.toLong(messageTableRowKey, offset + getTopicLengthMessageEntry(rowKeyLength)); }
@Override public ReturnCode filterKeyValue(Cell cell) throws IOException { int rowKeyOffset = cell.getRowOffset() + prefixLength; int sizeOfRowKey = cell.getRowLength() - prefixLength; long publishTimestamp = MessagingUtils.getPublishTimestamp(cell.getRowArray(), rowKeyOffset, sizeOfRowKey); int topicIdLength = MessagingUtils.getTopicLengthMessageEntry(sizeOfRowKey) - Bytes.SIZEOF_INT; int generationId = Bytes.toInt(cell.getRowArray(), rowKeyOffset + topicIdLength); if (prevTopicIdBytes == null || currentTTL == null || currentGen == null || (!Bytes.equals(prevTopicIdBytes, 0, prevTopicIdBytes.length, cell.getRowArray(), rowKeyOffset, topicIdLength))) { prevTopicIdBytes = Arrays.copyOfRange(cell.getRowArray(), rowKeyOffset, rowKeyOffset + topicIdLength); Map<String, String> properties = metadataCache.getTopicMetadata(ByteBuffer.wrap(prevTopicIdBytes)); if (properties == null) { LOG.debug("Region " + env.getRegionInfo().getRegionNameAsString() + ", could not get properties of topicId " + MessagingUtils.toTopicId(prevTopicIdBytes)); return ReturnCode.INCLUDE; } currentTTL = Long.parseLong(properties.get(MessagingUtils.Constants.TTL_KEY)); currentGen = Integer.parseInt(properties.get(MessagingUtils.Constants.GENERATION_KEY)); } // Old Generation (or deleted current generation) cleanup if (MessagingUtils.isOlderGeneration(generationId, currentGen)) { return ReturnCode.SKIP; } // TTL expiration cleanup only if the generation of metadata and row key are the same if ((generationId == currentGen) && ((timestamp - publishTimestamp) > currentTTL)) { return ReturnCode.SKIP; } return ReturnCode.INCLUDE; }
int rowKeyOffset = cell.getRowOffset() + prefixLength; int sizeOfRowKey = cell.getRowLength() - prefixLength; long writeTimestamp = MessagingUtils.getWriteTimestamp(cell.getRowArray(), rowKeyOffset, sizeOfRowKey); int topicIdLength = MessagingUtils.getTopicLengthPayloadEntry(sizeOfRowKey) - Bytes.SIZEOF_INT; int generationId = Bytes.toInt(cell.getRowArray(), rowKeyOffset + topicIdLength); if (properties == null) { LOG.debug("Region " + env.getRegionInfo().getRegionNameAsString() + ", could not get properties of topicId " + MessagingUtils.toTopicId(prevTopicIdBytes)); return ReturnCode.INCLUDE; if (MessagingUtils.isOlderGeneration(generationId, currentGen)) { return ReturnCode.SKIP;
/** * Convert byte array encoded with the {@link #toMetadataRowKey(TopicId)} method back to the {@link TopicId}. * Same as calling {@link #toTopicId(byte[], int, int)} with {@code offset = 0} * and {@code length = topicBytes.length}. * * @param topicBytes byte array which contains the representation of the topic id * @return {@link TopicId} */ public static TopicId toTopicId(byte[] topicBytes) { return toTopicId(topicBytes, 0, topicBytes.length); }
public ImmutableMessageTableEntry(byte[] row, @Nullable byte[] payload, @Nullable byte[] txPtr) { this.topicId = MessagingUtils.toTopicId(row, 0, row.length - Bytes.SIZEOF_SHORT - Bytes.SIZEOF_LONG - Bytes.SIZEOF_INT); this.generation = Bytes.toInt(row, row.length - Bytes.SIZEOF_SHORT - Bytes.SIZEOF_LONG - Bytes.SIZEOF_INT); int topicLength = MessagingUtils.getTopicLengthMessageEntry(row.length); this.publishTimestamp = Bytes.toLong(row, topicLength); this.sequenceId = Bytes.toShort(row, topicLength + Bytes.SIZEOF_LONG); this.transactional = (txPtr != null); // since we mark tx as negative when tx is rolled back, we return the absolute value of tx this.transactionWritePointer = txPtr == null ? -1 : Math.abs(Bytes.toLong(txPtr)); this.payload = payload; }
WriteBatch writeBatch = levelDB.createWriteBatch(); long ttlInMs = TimeUnit.SECONDS.toMillis(topicMetadata.getTTL()); byte[] startRow = MessagingUtils.toDataKeyPrefix(topicMetadata.getTopicId(), Integer.parseInt(MessagingUtils.Constants.DEFAULT_GENERATION)); byte[] stopRow = Bytes.stopKeyForPrefix(startRow); int currGeneration = topicMetadata.getGeneration(); checkTopic(topicMetadata.getTopicId(), topicMetadata.getGeneration()); if (MessagingUtils.isOlderGeneration(dataGeneration, currGeneration)) { writeBatch.delete(entry.getKey()); continue;