/** * Extracts the event and its metadata from {@link GobblinTrackingEvent} and creates * timestamped name value pairs * * @param event {@link GobblinTrackingEvent} to be reported * @throws IOException */ private void pushEvent(GobblinTrackingEvent event) throws IOException { Map<String, String> metadata = event.getMetadata(); String name = getMetricName(metadata, event.getName()); long timestamp = event.getTimestamp(); MultiPartEvent multiPartEvent = MultiPartEvent.getEvent(metadata.get(EventSubmitter.EVENT_TYPE)); if (multiPartEvent == null) { influxDBPusher.push(buildEventAsPoint(name, EMTPY_VALUE, timestamp)); } else { List<Point> points = Lists.newArrayList(); for (String field : multiPartEvent.getMetadataFields()) { Point point = buildEventAsPoint(JOINER.join(name, field), convertValue(field, metadata.get(field)), timestamp); points.add(point); } influxDBPusher.push(points); } }
/** * Build as {@link GobblinTrackingEvent} */ public GobblinTrackingEvent build() { return new GobblinTrackingEvent(0L, namespace, name, metadata); } /**
private void submitTrackingEvent(URI jobSpecURI, String jobSpecVersion, String operType) { GobblinTrackingEvent e = GobblinTrackingEvent.newBuilder() .setName(TRACKING_EVENT_NAME) .setNamespace(JobCatalog.class.getName()) .setMetadata(ImmutableMap.<String, String>builder() .put(GobblinMetricsKeys.OPERATION_TYPE_META, operType) .put(GobblinMetricsKeys.JOB_SPEC_URI_META, jobSpecURI.toString()) .put(GobblinMetricsKeys.JOB_SPEC_VERSION_META, jobSpecVersion) .build()) .build(); this.metricsContext.submitEvent(e); }
private static GobblinTrackingEvent sanitizeEvent(GobblinTrackingEvent event) { Map<String, String> newMetadata = Maps.newHashMap(); for (Map.Entry<String, String> metadata : event.getMetadata().entrySet()) { newMetadata.put(metadata.getKey() == null ? NULL_STRING : metadata.getKey(), metadata.getValue() == null ? NULL_STRING : metadata.getValue()); } event.setMetadata(newMetadata); return event; } }
/** * Filter for {@link GobblinTrackingEvent}. Used to quickly determine whether an event should be used to produce * a {@link JobSpec}. */ protected boolean acceptEvent(GobblinTrackingEvent event) { if (!event.getMetadata().containsKey(SlaEventKeys.DATASET_URN_KEY)) { return false; } String datasetURN = event.getMetadata().get(SlaEventKeys.DATASET_URN_KEY); if (this.urnFilter.isPresent() && !this.urnFilter.get().matcher(datasetURN).find()) { return false; } if (this.nameFilter.isPresent() && !this.nameFilter.get().matcher(event.getName()).find()) { return false; } return true; } }
String eventName = "testEvent"; GobblinTrackingEvent event = new GobblinTrackingEvent(); event.setName(eventName); event.setNamespace(namespace); Map<String, String> metadata = Maps.newHashMap(); metadata.put("m1", "v1"); metadata.put("m2", null); event.setMetadata(metadata); context.submitEvent(event); Assert.assertEquals(retrievedEvent.getNamespace(), namespace); Assert.assertEquals(retrievedEvent.getName(), eventName); Assert.assertEquals(retrievedEvent.getMetadata().size(), 4);
@Override public boolean apply(@Nonnull GobblinTrackingEvent input) { return input.getMetadata().containsKey(metadataKey); } };
String eventName = "testEvent"; GobblinTrackingEvent event = new GobblinTrackingEvent(); event.setName(eventName); event.setNamespace(namespace); Map<String, String> metadata = Maps.newHashMap(); metadata.put("m1", "v1"); metadata.put("m2", null); event.setMetadata(metadata); context.submitEvent(event); Assert.assertNull(retrievedEvent.getKey()); event = new GobblinTrackingEvent(); event.setName(eventName); event.setNamespace(namespace); metadata = Maps.newHashMap(); metadata.put("k1", "v1"); metadata.put("k2", "v2"); metadata.put("k3", "v3"); event.setMetadata(metadata); context.submitEvent(event);
@Override public boolean apply(@Nonnull GobblinTrackingEvent input) { return input.getName().equals(expectedName); } };
@Override public boolean apply(@Nonnull GobblinTrackingEvent input) { return input.getNamespace().equals(expectedNamespace); } };
String eventName = "testEvent"; GobblinTrackingEvent event = new GobblinTrackingEvent(); event.setName(eventName); event.setNamespace(namespace); Map<String, String> metadata = Maps.newHashMap(); metadata.put(tag1, metadataValue1); event.setMetadata(metadata); context.submitEvent(event); Assert.assertEquals(retrievedEvent.getNamespace(), namespace); Assert.assertEquals(retrievedEvent.getName(), eventName); Assert.assertEquals(retrievedEvent.getMetadata().size(), 4); Assert.assertEquals(retrievedEvent.getMetadata().get(tag1), metadataValue1); Assert.assertEquals(retrievedEvent.getMetadata().get(tag2), value2);
/** * Create a {@link LineageEventBuilder} from a {@link GobblinEventBuilder}. An inverse function * to {@link LineageEventBuilder#build()} */ public static LineageEventBuilder fromEvent(GobblinTrackingEvent event) { Map<String, String> metadata = event.getMetadata(); LineageEventBuilder lineageEvent = new LineageEventBuilder(event.getName()); metadata.forEach((key, value) -> { switch (key) { case SOURCE: lineageEvent.setSource(Descriptor.fromJson(value)); break; case DESTINATION: lineageEvent.setDestination(Descriptor.fromJson(value)); break; default: lineageEvent.addMetadata(key, value); break; } }); return lineageEvent; }
/** * Check if the given {@link GobblinTrackingEvent} is a lineage event */ public static boolean isLineageEvent(GobblinTrackingEvent event) { String eventType = event.getMetadata().get(EVENT_TYPE); return StringUtils.isNotEmpty(eventType) && eventType.equals(LINEAGE_EVENT_TYPE); }
/** * Inject the tags of this {@link MetricContext} to the given {@link GobblinTrackingEvent} */ private void injectTagsToEvent(GobblinTrackingEvent event) { Map<String, String> originalMetadata = event.getMetadata(); Map<String, Object> tags = getTagMap(); Map<String, String> newMetadata = Maps.newHashMap(); for(Map.Entry<String, Object> entry : tags.entrySet()) { newMetadata.put(entry.getKey(), entry.getValue().toString()); } newMetadata.putAll(originalMetadata); event.setMetadata(newMetadata); }
@Override public Collection<Either<JobSpec, URI>> parseJobSpec(GobblinTrackingEvent message) { this.events.add(message); return Lists.newArrayList(Either.<JobSpec, URI>left(JobSpec.builder(message.getName()).build())); } }
@Override public boolean apply(@Nonnull GobblinTrackingEvent input) { return input.getNamespace().equals(expectedNamespace); } };
Map<String, String> metadata = event.getMetadata(); String name = getMetricName(metadata, event.getName()); long timestamp = event.getTimestamp() / 1000l; MultiPartEvent multipartEvent = MultiPartEvent.getEvent(metadata.get(EventSubmitter.EVENT_TYPE)); if (multipartEvent == null) {
/** * Filter for {@link GobblinTrackingEvent}. Used to quickly determine whether an event should be used to produce * a {@link JobSpec}. */ protected boolean acceptEvent(GobblinTrackingEvent event) { if (!event.getMetadata().containsKey(SlaEventKeys.DATASET_URN_KEY)) { return false; } String datasetURN = event.getMetadata().get(SlaEventKeys.DATASET_URN_KEY); if (this.urnFilter.isPresent() && !this.urnFilter.get().matcher(datasetURN).find()) { return false; } if (this.nameFilter.isPresent() && !this.nameFilter.get().matcher(event.getName()).find()) { return false; } return true; } }
/** * Build as {@link GobblinTrackingEvent} */ public GobblinTrackingEvent build() { if (rootCause != null) { metadata.put(ROOT_CAUSE, ExceptionUtils.getStackTrace(rootCause)); } return new GobblinTrackingEvent(0L, namespace, name, metadata); }
/** * Check if the given {@link GobblinTrackingEvent} is a failure event */ public static boolean isFailureEvent(GobblinTrackingEvent event) { String eventType = event.getMetadata().get(EVENT_TYPE); return StringUtils.isNotEmpty(eventType) && eventType.equals(FAILURE_EVENT_TYPE); }