/** {@inheritDoc} */ @Override public void remove(URI uri) { ((MutableJobCatalog)_fallback).remove(uri); }
/** {@inheritDoc} */ @Override public void put(JobSpec jobSpec) { ((MutableJobCatalog)_fallback).put(jobSpec); }
@Override protected void processMessage(MessageAndMetadata<byte[], byte[]> message) { try { Collection<Either<JobSpec, URI>> parsedCollection = parseJobSpec(message.message()); for (Either<JobSpec, URI> parsedMessage : parsedCollection) { if (parsedMessage instanceof Either.Left) { this.newSpecs.inc(); this.jobCatalog.put(((Either.Left<JobSpec, URI>) parsedMessage).getLeft()); } else if (parsedMessage instanceof Either.Right) { this.remmovedSpecs.inc(); this.jobCatalog.remove(((Either.Right<JobSpec, URI>) parsedMessage).getRight()); } } } catch (IOException ioe) { String messageStr = new String(message.message(), Charsets.UTF_8); log.error(String.format("Failed to parse kafka message with offset %d: %s.", message.offset(), messageStr), ioe); } }
public StreamingKafkaSpecExecutorInstanceConsumer(Config config, MutableJobCatalog jobCatalog, Optional<Logger> log) { super(config, log); String topic = config.getString(SPEC_KAFKA_TOPICS_KEY); Config defaults = ConfigFactory.parseMap(ImmutableMap.of(AvroJobSpecKafkaJobMonitor.TOPIC_KEY, topic, KafkaJobMonitor.KAFKA_AUTO_OFFSET_RESET_KEY, KafkaJobMonitor.KAFKA_AUTO_OFFSET_RESET_SMALLEST)); try { _jobMonitor = (AvroJobSpecKafkaJobMonitor)(new AvroJobSpecKafkaJobMonitor.Factory()) .forConfig(config.withFallback(defaults), jobCatalog); } catch (IOException e) { throw new RuntimeException("Could not create job monitor", e); } _jobSpecQueue = new LinkedBlockingQueue<>(ConfigUtils.getInt(config, "SPEC_STREAMING_BLOCKING_QUEUE_SIZE", DEFAULT_SPEC_STREAMING_BLOCKING_QUEUE_SIZE)); // listener will add job specs to a blocking queue to send to callers of changedSpecs() jobCatalog.addListener(new JobSpecListener()); }
@Override public void run() { try { GobblinHelixJobScheduler.this.runJob(this.jobConfig, this.jobListener); // remove non-scheduled job catalog once done so it won't be re-executed if (GobblinHelixJobScheduler.this.jobCatalog != null) { try { GobblinHelixJobScheduler.this.jobCatalog.remove(new URI(jobUri)); } catch (URISyntaxException e) { LOGGER.error("Failed to remove job with bad uri " + jobUri, e); } } } catch (JobException je) { LOGGER.error("Failed to run job " + this.jobConfig.getProperty(ConfigurationKeys.JOB_NAME_KEY), je); } } }