private void initializeWatermarks() { initializeLowWatermarks(); initializeHighWatermarks(); }
@Test (dependsOnMethods = "testDeleteSpec") public void testResetConsumption() throws Exception { SimpleKafkaSpecConsumer seic = _closer .register(new SimpleKafkaSpecConsumer(ConfigUtils.propertiesToConfig(_properties))); List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = seic.changedSpecs().get(); Assert.assertTrue(consumedEvent.size() == 3, "Consumption was reset, we should see all events"); }
@Override public Future<? extends List<Pair<SpecExecutor.Verb, Spec>>> changedSpecs() { List<Pair<SpecExecutor.Verb, Spec>> changesSpecs = new ArrayList<>(); initializeWatermarks(); this.currentPartitionIdx = -1; while (!allPartitionsFinished()) { if (currentPartitionFinished()) { moveToNextPartition(); continue; this.messageIterator = fetchNextMessageBuffer(); } catch (Exception e) { log.error(String.format("Failed to fetch next message buffer for partition %s. Will skip this partition.", getCurrentPartition()), e); moveToNextPartition(); continue; moveToNextPartition(); continue; while (!currentPartitionFinished()) { if (!this.messageIterator.hasNext()) { break; record = decodeRecord((ByteArrayBasedKafkaRecord)nextValidMessage); } else if (nextValidMessage instanceof DecodeableKafkaRecord){ record = ((DecodeableKafkaRecord<?, AvroJobSpec>) nextValidMessage).getValue();
@Override public Future<? extends List<Pair<SpecExecutor.Verb, Spec>>> changedSpecs() { List<Pair<SpecExecutor.Verb, Spec>> changesSpecs = new ArrayList<>(); initializeWatermarks(); this.currentPartitionIdx = -1; while (!allPartitionsFinished()) { if (currentPartitionFinished()) { moveToNextPartition(); continue; this.messageIterator = fetchNextMessageBuffer(); } catch (Exception e) { log.error(String.format("Failed to fetch next message buffer for partition %s. Will skip this partition.", getCurrentPartition()), e); moveToNextPartition(); continue; moveToNextPartition(); continue; while (!currentPartitionFinished()) { if (!this.messageIterator.hasNext()) { break; record = decodeRecord((ByteArrayBasedKafkaRecord)nextValidMessage); } else if (nextValidMessage instanceof DecodeableKafkaRecord){ record = ((DecodeableKafkaRecord<?, AvroJobSpec>) nextValidMessage).getValue();
private void initializeWatermarks() { initializeLowWatermarks(); initializeHighWatermarks(); }
@Test public void testAddSpec() throws Exception { _closer = Closer.create(); _properties = new Properties(); // Properties for Producer _properties.setProperty(KafkaWriterConfigurationKeys.KAFKA_TOPIC, TOPIC); _properties.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "bootstrap.servers", _kafkaBrokers); _properties.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX+"value.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer"); // Properties for Consumer _properties.setProperty(ConfigurationKeys.KAFKA_BROKERS, _kafkaBrokers); _properties.setProperty(SimpleKafkaSpecExecutor.SPEC_KAFKA_TOPICS_KEY, TOPIC); // SEI Producer _seip = _closer.register(new SimpleKafkaSpecProducer(ConfigUtils.propertiesToConfig(_properties))); String addedSpecUriString = "/foo/bar/addedSpec"; Spec spec = initJobSpec(addedSpecUriString); WriteResponse writeResponse = (WriteResponse) _seip.addSpec(spec).get(); log.info("WriteResponse: " + writeResponse); try { Thread.sleep(1000); } catch(InterruptedException ex) { Thread.currentThread().interrupt(); } _seic = _closer.register(new SimpleKafkaSpecConsumer(ConfigUtils.propertiesToConfig(_properties))); List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = _seic.changedSpecs().get(); Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production"); Map.Entry<SpecExecutor.Verb, Spec> consumedSpecAction = consumedEvent.get(0); Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.ADD), "Verb did not match"); Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(addedSpecUriString), "Expected URI did not match"); Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec"); }