@Override public void register(SystemStreamPartition ssp, String offset) { LOG.info("Register called with ssp {} and offset {}. Offset will be ignored.", ssp, offset); String stream = ssp.getStream(); streams.add(stream); sspAllocator.free(ssp); super.register(ssp, offset); }
@Override public void register(SystemStreamPartition systemStreamPartition, String offset) { super.register(systemStreamPartition, offset); LOG.info(String.format("Eventhub consumer trying to register ssp %s, offset %s", systemStreamPartition, offset)); if (isStarted) { throw new SamzaException("Trying to add partition when the connection has already started."); } if (streamPartitionOffsets.containsKey(systemStreamPartition)) { // Only update if new offset is lower than previous offset if (END_OF_STREAM.equals(offset)) { return; } String prevOffset = streamPartitionOffsets.get(systemStreamPartition); if (!END_OF_STREAM.equals(prevOffset) && EventHubSystemAdmin.compareOffsets(offset, prevOffset) > -1) { return; } } streamPartitionOffsets.put(systemStreamPartition, offset); }
/** * {@inheritDoc} */ @Override public void register(SystemStreamPartition systemStreamPartition, String offset) { LOG.info("HdfsSystemConsumer register with partition: " + systemStreamPartition + " and offset " + offset); super.register(systemStreamPartition, offset); MultiFileHdfsReader reader = new MultiFileHdfsReader(readerType, systemStreamPartition, getPartitionDescriptor(systemStreamPartition), offset, numMaxRetires); readers.put(systemStreamPartition, reader); consumerMetrics.registerSystemStreamPartition(systemStreamPartition); }
@Override public void register(SystemStreamPartition systemStreamPartition, String offset) { super.register(systemStreamPartition, offset); setIsAtHead(systemStreamPartition, true); }
/** * record the ssp and the offset. Do not submit it to the consumer yet. * @param systemStreamPartition ssp to register * @param offset offset to register with */ @Override public void register(SystemStreamPartition systemStreamPartition, String offset) { if (started.get()) { String msg = String.format("%s: Trying to register partition after consumer has been started. ssp=%s", this, systemStreamPartition); throw new SamzaException(msg); } if (!systemStreamPartition.getSystem().equals(systemName)) { LOG.warn("{}: ignoring SSP {}, because this consumer's system doesn't match.", this, systemStreamPartition); return; } LOG.info("{}: Registering ssp = {} with offset {}", this, systemStreamPartition, offset); super.register(systemStreamPartition, offset); TopicPartition tp = toTopicPartition(systemStreamPartition); topicPartitionsToSSP.put(tp, systemStreamPartition); String existingOffset = topicPartitionsToOffset.get(tp); // register the older (of the two) offset in the consumer, to guarantee we do not miss any messages. if (existingOffset == null || compareOffsets(existingOffset, offset) > 0) { topicPartitionsToOffset.put(tp, offset); } metrics.registerTopicAndPartition(toTopicAndPartition(tp)); }
/** * record the ssp and the offset. Do not submit it to the consumer yet. * @param systemStreamPartition ssp to register * @param offset offset to register with */ @Override public void register(SystemStreamPartition systemStreamPartition, String offset) { if (started.get()) { String msg = String.format("%s: Trying to register partition after consumer has been started. ssp=%s", this, systemStreamPartition); throw new SamzaException(msg); } if (!systemStreamPartition.getSystem().equals(systemName)) { LOG.warn("{}: ignoring SSP {}, because this consumer's system doesn't match.", this, systemStreamPartition); return; } LOG.info("{}: Registering ssp = {} with offset {}", this, systemStreamPartition, offset); super.register(systemStreamPartition, offset); TopicPartition tp = toTopicPartition(systemStreamPartition); topicPartitionsToSSP.put(tp, systemStreamPartition); String existingOffset = topicPartitionsToOffset.get(tp); // register the older (of the two) offset in the consumer, to guarantee we do not miss any messages. if (existingOffset == null || compareOffsets(existingOffset, offset) > 0) { topicPartitionsToOffset.put(tp, offset); } metrics.registerTopicAndPartition(toTopicAndPartition(tp)); }
/** * record the ssp and the offset. Do not submit it to the consumer yet. * @param systemStreamPartition ssp to register * @param offset offset to register with */ @Override public void register(SystemStreamPartition systemStreamPartition, String offset) { if (started.get()) { String msg = String.format("%s: Trying to register partition after consumer has been started. ssp=%s", this, systemStreamPartition); throw new SamzaException(msg); } if (!systemStreamPartition.getSystem().equals(systemName)) { LOG.warn("{}: ignoring SSP {}, because this consumer's system doesn't match.", this, systemStreamPartition); return; } LOG.info("{}: Registering ssp = {} with offset {}", this, systemStreamPartition, offset); super.register(systemStreamPartition, offset); TopicPartition tp = toTopicPartition(systemStreamPartition); topicPartitionsToSSP.put(tp, systemStreamPartition); String existingOffset = topicPartitionsToOffset.get(tp); // register the older (of the two) offset in the consumer, to guarantee we do not miss any messages. if (existingOffset == null || compareOffsets(existingOffset, offset) > 0) { topicPartitionsToOffset.put(tp, offset); } metrics.registerTopicAndPartition(toTopicAndPartition(tp)); }
@Test public void testShouldBlockAtLeast100Ms() throws InterruptedException { BlockingEnvelopeMap map = new MockBlockingEnvelopeMap(); map.register(SSP, "0"); long now = System.currentTimeMillis(); map.poll(FETCH, 100); assertTrue(System.currentTimeMillis() - now >= 100); }
@Test public void testShouldBlockWhenNotAtHead() throws InterruptedException { MockQueue q = new MockQueue(); final BlockingEnvelopeMap map = new MockBlockingEnvelopeMap(q); map.register(SSP, "0"); Thread t = new Thread(new Runnable() { @Override public void run() { try { // Should trigger a take() call. map.poll(FETCH, -1); } catch (InterruptedException e) { throw new RuntimeException(e); } } }); t.setDaemon(true); t.start(); q.awaitPollTimeout(); t.join(60000); // 1000 = blocking timeout constant assertEquals(1000, q.timeout); assertFalse(t.isAlive()); }
map.register(SSP, "0");
@Test public void testShouldGetSomeMessages() throws InterruptedException { BlockingEnvelopeMap map = new MockBlockingEnvelopeMap(); map.register(SSP, "0"); map.put(SSP, ENVELOPE); Map<SystemStreamPartition, List<IncomingMessageEnvelope>> envelopes = map.poll(FETCH, 0); assertEquals(1, envelopes.size()); assertEquals(1, envelopes.get(SSP).size()); map.put(SSP, ENVELOPE); map.put(SSP, ENVELOPE); envelopes = map.poll(FETCH, 0); assertEquals(1, envelopes.size()); assertEquals(2, envelopes.get(SSP).size()); // Size info. assertEquals(0, map.getMessagesSizeInQueue(SSP)); }
@Test public void testEmptyMapReturnsEmptyList() throws InterruptedException { BlockingEnvelopeMap map = new MockBlockingEnvelopeMap(); map.register(SSP, "0"); map.poll(FETCH, 0); map.poll(FETCH, 30); map.setIsAtHead(SSP, true); map.poll(FETCH, -1); }
@Test public void testNoSizeComputation() throws InterruptedException { BlockingEnvelopeMap map = new MockBlockingEnvelopeMap(); map.register(SSP, "0"); map.put(SSP, ENVELOPE); map.put(SSP, ENVELOPE); Map<SystemStreamPartition, List<IncomingMessageEnvelope>> envelopes = map.poll(FETCH, 0); // Size info. assertEquals(0, map.getMessagesSizeInQueue(SSP)); }
@Test public void testSizeComputation() throws InterruptedException { BlockingEnvelopeMap map = new MockBlockingEnvelopeMap(true); map.register(SSP, "0"); map.put(SSP, ENVELOPE_WITH_SIZE); map.put(SSP, ENVELOPE_WITH_SIZE); // Size info. assertEquals(200, map.getMessagesSizeInQueue(SSP)); Map<SystemStreamPartition, List<IncomingMessageEnvelope>> envelopes = map.poll(FETCH, 0); assertEquals(0, map.getMessagesSizeInQueue(SSP)); }