@Override public boolean apply(@Nonnull GobblinTrackingEvent input) { return input.getMetadata().containsKey(metadataKey); } };
/** * Check if the given {@link GobblinTrackingEvent} is a lineage event */ public static boolean isLineageEvent(GobblinTrackingEvent event) { String eventType = event.getMetadata().get(EVENT_TYPE); return StringUtils.isNotEmpty(eventType) && eventType.equals(LINEAGE_EVENT_TYPE); }
/** * Check if the given {@link GobblinTrackingEvent} is a failure event */ public static boolean isFailureEvent(GobblinTrackingEvent event) { String eventType = event.getMetadata().get(EVENT_TYPE); return StringUtils.isNotEmpty(eventType) && eventType.equals(FAILURE_EVENT_TYPE); }
@Override public boolean apply(@Nonnull GobblinTrackingEvent input) { return input.getMetadata().get(metadataKey).equals(metadataValue); } };
private static GobblinTrackingEvent sanitizeEvent(GobblinTrackingEvent event) { Map<String, String> newMetadata = Maps.newHashMap(); for (Map.Entry<String, String> metadata : event.getMetadata().entrySet()) { newMetadata.put(metadata.getKey() == null ? NULL_STRING : metadata.getKey(), metadata.getValue() == null ? NULL_STRING : metadata.getValue()); } event.setMetadata(newMetadata); return event; } }
/** * Filter for {@link GobblinTrackingEvent}. Used to quickly determine whether an event should be used to produce * a {@link JobSpec}. */ protected boolean acceptEvent(GobblinTrackingEvent event) { if (!event.getMetadata().containsKey(SlaEventKeys.DATASET_URN_KEY)) { return false; } String datasetURN = event.getMetadata().get(SlaEventKeys.DATASET_URN_KEY); if (this.urnFilter.isPresent() && !this.urnFilter.get().matcher(datasetURN).find()) { return false; } if (this.nameFilter.isPresent() && !this.nameFilter.get().matcher(event.getName()).find()) { return false; } return true; } }
/** * Inject the tags of this {@link MetricContext} to the given {@link GobblinTrackingEvent} */ private void injectTagsToEvent(GobblinTrackingEvent event) { Map<String, String> originalMetadata = event.getMetadata(); Map<String, Object> tags = getTagMap(); Map<String, String> newMetadata = Maps.newHashMap(); for(Map.Entry<String, Object> entry : tags.entrySet()) { newMetadata.put(entry.getKey(), entry.getValue().toString()); } newMetadata.putAll(originalMetadata); event.setMetadata(newMetadata); }
@Override public void reportEventQueue(Queue<GobblinTrackingEvent> queue) { GobblinTrackingEvent nextEvent; List<Pair<String, byte[]>> events = Lists.newArrayList(); while(null != (nextEvent = queue.poll())) { StringBuilder sb = new StringBuilder(); String key = null; if (keys.isPresent()) { for (String keyPart : keys.get()) { if (nextEvent.getMetadata().containsKey(keyPart)) { sb.append(nextEvent.getMetadata().get(keyPart)); } else { log.error("{} not found in the GobblinTrackingEvent. Setting key to null.", keyPart); sb = null; break; } } key = (sb == null) ? null : sb.toString(); } events.add(Pair.of(key, this.serializer.serializeRecord(nextEvent))); } if (!events.isEmpty()) { this.kafkaPusher.pushMessages(events); } }
@Override public Collection<Either<JobSpec, URI>> parseJobSpec(GobblinTrackingEvent event) { if (!acceptEvent(event)) { this.rejectedEvents.inc(); return Lists.newArrayList(); } String datasetURN = event.getMetadata().get(SlaEventKeys.DATASET_URN_KEY); URI jobSpecURI = PathUtils.mergePaths(new Path(this.baseURI), new Path(datasetURN)).toUri(); Map<String, String> jobConfigMap = Maps.newHashMap(); for (Map.Entry<String, String> entry : this.extractKeys.entrySet()) { if (event.getMetadata().containsKey(entry.getKey())) { jobConfigMap.put(entry.getValue(), event.getMetadata().get(entry.getKey())); } } Config jobConfig = ConfigFactory.parseMap(jobConfigMap); JobSpec jobSpec = JobSpec.builder(jobSpecURI).withTemplate(this.template).withConfig(jobConfig).build(); return Lists.newArrayList(Either.<JobSpec, URI>left(jobSpec)); }
/** * Extracts the event and its metadata from {@link GobblinTrackingEvent} and creates * timestamped name value pairs * * @param event {@link GobblinTrackingEvent} to be reported * @throws IOException */ private void pushEvent(GobblinTrackingEvent event) throws IOException { Map<String, String> metadata = event.getMetadata(); String name = getMetricName(metadata, event.getName()); long timestamp = event.getTimestamp(); MultiPartEvent multiPartEvent = MultiPartEvent.getEvent(metadata.get(EventSubmitter.EVENT_TYPE)); if (multiPartEvent == null) { influxDBPusher.push(buildEventAsPoint(name, EMTPY_VALUE, timestamp)); } else { List<Point> points = Lists.newArrayList(); for (String field : multiPartEvent.getMetadataFields()) { Point point = buildEventAsPoint(JOINER.join(name, field), convertValue(field, metadata.get(field)), timestamp); points.add(point); } influxDBPusher.push(points); } }
Map<String, String> metadata = event.getMetadata(); String name = getMetricName(metadata, event.getName()); long timestamp = event.getTimestamp() / 1000l;
/** * Create a {@link LineageEventBuilder} from a {@link GobblinEventBuilder}. An inverse function * to {@link LineageEventBuilder#build()} */ public static LineageEventBuilder fromEvent(GobblinTrackingEvent event) { Map<String, String> metadata = event.getMetadata(); LineageEventBuilder lineageEvent = new LineageEventBuilder(event.getName()); metadata.forEach((key, value) -> { switch (key) { case SOURCE: lineageEvent.setSource(Descriptor.fromJson(value)); break; case DESTINATION: lineageEvent.setDestination(Descriptor.fromJson(value)); break; default: lineageEvent.addMetadata(key, value); break; } }); return lineageEvent; }
Assert.assertEquals(retrievedEvent.getNamespace(), namespace); Assert.assertEquals(retrievedEvent.getName(), eventName); Assert.assertEquals(retrievedEvent.getMetadata().size(), 4);
Assert.assertEquals(retrievedEvent.getNamespace(), namespace); Assert.assertEquals(retrievedEvent.getName(), eventName); Assert.assertEquals(retrievedEvent.getMetadata().size(), 4); Assert.assertEquals(retrievedEvent.getMetadata().get(tag1), metadataValue1); Assert.assertEquals(retrievedEvent.getMetadata().get(tag2), value2);
@Override public boolean apply(@Nonnull GobblinTrackingEvent input) { return input.getMetadata().containsKey(metadataKey); } };
/** * Check if the given {@link GobblinTrackingEvent} is a failure event */ public static boolean isFailureEvent(GobblinTrackingEvent event) { String eventType = event.getMetadata().get(EVENT_TYPE); return StringUtils.isNotEmpty(eventType) && eventType.equals(FAILURE_EVENT_TYPE); }
/** * Check if the given {@link GobblinTrackingEvent} is a lineage event */ public static boolean isLineageEvent(GobblinTrackingEvent event) { String eventType = event.getMetadata().get(EVENT_TYPE); return StringUtils.isNotEmpty(eventType) && eventType.equals(LINEAGE_EVENT_TYPE); }
@Override public boolean apply(@Nonnull GobblinTrackingEvent input) { return input.getMetadata().get(metadataKey).equals(metadataValue); } };
private static GobblinTrackingEvent sanitizeEvent(GobblinTrackingEvent event) { Map<String, String> newMetadata = Maps.newHashMap(); for (Map.Entry<String, String> metadata : event.getMetadata().entrySet()) { newMetadata.put(metadata.getKey() == null ? NULL_STRING : metadata.getKey(), metadata.getValue() == null ? NULL_STRING : metadata.getValue()); } event.setMetadata(newMetadata); return event; } }
/** * Inject the tags of this {@link MetricContext} to the given {@link GobblinTrackingEvent} */ private void injectTagsToEvent(GobblinTrackingEvent event) { Map<String, String> originalMetadata = event.getMetadata(); Map<String, Object> tags = getTagMap(); Map<String, String> newMetadata = Maps.newHashMap(); for(Map.Entry<String, Object> entry : tags.entrySet()) { newMetadata.put(entry.getKey(), entry.getValue().toString()); } newMetadata.putAll(originalMetadata); event.setMetadata(newMetadata); }