@Override public String getTransitUri() { return event.getTransitUri(); }
@Override public String getTransitUri() { return record.getTransitUri(); }
private static String stringify(final ProvenanceEventRecord event, final int index, final long byteOffset) { final StringBuilder sb = new StringBuilder(); sb.append("Event Index in File = ").append(index).append(", Byte Offset = ").append(byteOffset); sb.append("\n\t").append("Event ID = ").append(event.getEventId()); sb.append("\n\t").append("Event Type = ").append(event.getEventType()); sb.append("\n\t").append("Event Time = ").append(new Date(event.getEventTime())); sb.append("\n\t").append("Event UUID = ").append(event.getFlowFileUuid()); sb.append("\n\t").append("Component ID = ").append(event.getComponentId()); sb.append("\n\t").append("Event ID = ").append(event.getComponentType()); sb.append("\n\t").append("Transit URI = ").append(event.getTransitUri()); sb.append("\n\t").append("Parent IDs = ").append(event.getParentUuids()); sb.append("\n\t").append("Child IDs = ").append(event.getChildUuids()); sb.append("\n\t").append("Previous Attributes = ").append(event.getPreviousAttributes()); sb.append("\n\t").append("Updated Attributes = ").append(event.getUpdatedAttributes()); return sb.toString(); } }
protected DataSetRefs executeAnalyzer(AnalysisContext analysisContext, ProvenanceEventRecord event) { final NiFiProvenanceEventAnalyzer analyzer = NiFiProvenanceEventAnalyzerFactory.getAnalyzer(event.getComponentType(), event.getTransitUri(), event.getEventType()); if (analyzer == null) { return null; } if (logger.isDebugEnabled()) { logger.debug("Analyzer {} is found for event: {}", analyzer, event); } return analyzer.analyze(analysisContext, event); }
protected S2SPort analyzeS2SPort(ProvenanceEventRecord event, ClusterResolver clusterResolver) { final String transitUri = event.getTransitUri(); final int protocolIndex = transitUri.indexOf(':'); final String protocol = transitUri.substring(0, protocolIndex).toLowerCase(); final String targetHostname; final String targetPortId; switch (protocol) { case "http": case "https": { final Matcher uriMatcher = matchUrl(transitUri, HTTP_URL_REGEX); targetHostname = parseUri(transitUri).getHost(); targetPortId = uriMatcher.group(2); } break; case "nifi": { final Matcher uriMatcher = matchUrl(transitUri, RAW_URL_REGEX); targetHostname = uriMatcher.group(1); targetPortId = getRawProtocolPortId(event); } break; default: throw new IllegalArgumentException("Protocol " + protocol + " is not supported as NiFi S2S transit URL."); } final String clusterName = clusterResolver.fromHostNames(targetHostname); return new S2SPort(clusterName, targetPortId); }
return record.getTransitUri();
@Override public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) { final URI uri = parseUri(event.getTransitUri()); final String clusterName = context.getClusterResolver().fromHostNames(uri.getHost()); final Set<Tuple<String, String>> outputTables = parseTableNames(null, event.getAttribute(ATTR_OUTPUT_TABLES)); if (outputTables.isEmpty()) { return null; } final DataSetRefs refs = new DataSetRefs(event.getComponentId()); outputTables.forEach(tableName -> { final Referenceable ref = createTableRef(clusterName, tableName); refs.addOutput(ref); }); return refs; }
@Override public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) { final String transitUri = event.getTransitUri(); final Matcher uriMatcher = URI_PATTERN.matcher(transitUri); if (!uriMatcher.matches()) { logger.warn("Unexpected transit URI: {}", new Object[]{transitUri}); return null; } final Referenceable ref = new Referenceable(TYPE); final String[] hostNames = splitHostNames(uriMatcher.group(1)); final String clusterName = context.getClusterResolver().fromHostNames(hostNames); final String tableName = uriMatcher.group(2); ref.set(ATTR_NAME, tableName); ref.set(ATTR_QUALIFIED_NAME, toQualifiedName(clusterName, tableName)); // TODO: 'uri' is a mandatory attribute, but what should we set? ref.set(ATTR_URI, transitUri); return singleDataSetRef(event.getComponentId(), event.getEventType(), ref); }
@Override public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) { final Referenceable ref = new Referenceable(TYPE); final URI uri = parseUri(event.getTransitUri()); final String clusterName; try { // use hostname in uri if available for remote path. final String uriHost = uri.getHost(); final String hostname = StringUtils.isEmpty(uriHost) ? InetAddress.getLocalHost().getHostName() : uriHost; clusterName = context.getClusterResolver().fromHostNames(hostname); } catch (UnknownHostException e) { logger.warn("Failed to get localhost name due to " + e, e); return null; } final String path = uri.getPath(); ref.set(ATTR_NAME, path); ref.set(ATTR_PATH, path); ref.set(ATTR_QUALIFIED_NAME, toQualifiedName(clusterName, path)); return singleDataSetRef(event.getComponentId(), event.getEventType(), ref); }
@Override public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) { final Referenceable ref = new Referenceable(TYPE); final String transitUri = event.getTransitUri(); if (transitUri == null) { return null; } final Matcher uriMatcher = URI_PATTERN.matcher(transitUri); if (!uriMatcher.matches()) { logger.warn("Unexpected transit URI: {}", new Object[]{transitUri}); return null; } final String[] hostNames = splitHostNames(uriMatcher.group(1)); final String clusterName = context.getClusterResolver().fromHostNames(hostNames); final String topicName = uriMatcher.group(2); ref.set(ATTR_NAME, topicName); ref.set(ATTR_TOPIC, topicName); ref.set(ATTR_QUALIFIED_NAME, toQualifiedName(clusterName, topicName)); ref.set(ATTR_URI, transitUri); return singleDataSetRef(event.getComponentId(), event.getEventType(), ref); }
@Override public DataSetRefs analyze(AnalysisContext context, ProvenanceEventRecord event) { final Referenceable ref = new Referenceable(TYPE); final URI uri = parseUri(event.getTransitUri()); final String clusterName = context.getClusterResolver().fromHostNames(uri.getHost()); final String path = uri.getPath(); ref.set(ATTR_NAME, path); ref.set(ATTR_PATH, path); ref.set(ATTR_CLUSTER_NAME, clusterName); ref.set(ATTR_QUALIFIED_NAME, toQualifiedName(clusterName, path)); return singleDataSetRef(event.getComponentId(), event.getEventType(), ref); }
private void verifyProvenance(int expectedNumEvents) { List<ProvenanceEventRecord> provEvents = runner.getProvenanceEvents(); Assert.assertEquals(expectedNumEvents, provEvents.size()); for (ProvenanceEventRecord event : provEvents) { Assert.assertEquals(ProvenanceEventType.RECEIVE, event.getEventType()); Assert.assertTrue(event.getTransitUri().startsWith("udp://")); } }
@Test public void testBatching() throws IOException, InterruptedException { runner.setProperty(ListenRELP.MAX_BATCH_SIZE, "5"); final List<RELPFrame> frames = new ArrayList<>(); frames.add(OPEN_FRAME); frames.add(SYSLOG_FRAME); frames.add(SYSLOG_FRAME); frames.add(SYSLOG_FRAME); frames.add(CLOSE_FRAME); // one syslog frame should be transferred since we are batching, but three responses should be sent run(frames, 1, 3, null); final List<ProvenanceEventRecord> events = runner.getProvenanceEvents(); Assert.assertNotNull(events); Assert.assertEquals(1, events.size()); final ProvenanceEventRecord event = events.get(0); Assert.assertEquals(ProvenanceEventType.RECEIVE, event.getEventType()); Assert.assertTrue("transit uri must be set and start with proper protocol", event.getTransitUri().toLowerCase().startsWith("relp")); final List<MockFlowFile> mockFlowFiles = runner.getFlowFilesForRelationship(ListenRELP.REL_SUCCESS); Assert.assertEquals(1, mockFlowFiles.size()); final MockFlowFile mockFlowFile = mockFlowFiles.get(0); Assert.assertEquals(SYSLOG_FRAME.getCommand(), mockFlowFile.getAttribute(ListenRELP.RELPAttributes.COMMAND.key())); Assert.assertTrue(!StringUtils.isBlank(mockFlowFile.getAttribute(ListenRELP.RELPAttributes.PORT.key()))); Assert.assertTrue(!StringUtils.isBlank(mockFlowFile.getAttribute(ListenRELP.RELPAttributes.SENDER.key()))); }
@Test public void testListenRELP() throws IOException, InterruptedException { final List<RELPFrame> frames = new ArrayList<>(); frames.add(OPEN_FRAME); frames.add(SYSLOG_FRAME); frames.add(SYSLOG_FRAME); frames.add(SYSLOG_FRAME); frames.add(CLOSE_FRAME); // three syslog frames should be transferred and three responses should be sent run(frames, 3, 3, null); final List<ProvenanceEventRecord> events = runner.getProvenanceEvents(); Assert.assertNotNull(events); Assert.assertEquals(3, events.size()); final ProvenanceEventRecord event = events.get(0); Assert.assertEquals(ProvenanceEventType.RECEIVE, event.getEventType()); Assert.assertTrue("transit uri must be set and start with proper protocol", event.getTransitUri().toLowerCase().startsWith("relp")); final List<MockFlowFile> mockFlowFiles = runner.getFlowFilesForRelationship(ListenRELP.REL_SUCCESS); Assert.assertEquals(3, mockFlowFiles.size()); final MockFlowFile mockFlowFile = mockFlowFiles.get(0); Assert.assertEquals(String.valueOf(SYSLOG_FRAME.getTxnr()), mockFlowFile.getAttribute(ListenRELP.RELPAttributes.TXNR.key())); Assert.assertEquals(SYSLOG_FRAME.getCommand(), mockFlowFile.getAttribute(ListenRELP.RELPAttributes.COMMAND.key())); Assert.assertTrue(!StringUtils.isBlank(mockFlowFile.getAttribute(ListenRELP.RELPAttributes.PORT.key()))); Assert.assertTrue(!StringUtils.isBlank(mockFlowFile.getAttribute(ListenRELP.RELPAttributes.SENDER.key()))); }
assertTrue(runner.getProvenanceEvents().size() == 1); assertEquals(ProvenanceEventType.SEND, runner.getProvenanceEvents().get(0).getEventType()); assertEquals("https://client@server:8443/test", runner.getProvenanceEvents().get(0).getTransitUri());
@Test public void testValidMessageStaticPropertiesUdp() { final String pri = "34"; final String version = "1"; final String stamp = "2003-10-11T22:14:15.003Z"; final String host = "mymachine.example.com"; final String body = "su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8"; final String expectedMessage = "<" + pri + ">" + version + " " + stamp + " " + host + " " + body; runner.setProperty(PutSyslog.MSG_PRIORITY, pri); runner.setProperty(PutSyslog.MSG_VERSION, version); runner.setProperty(PutSyslog.MSG_TIMESTAMP, stamp); runner.setProperty(PutSyslog.MSG_HOSTNAME, host); runner.setProperty(PutSyslog.MSG_BODY, body); runner.enqueue("incoming data".getBytes(Charset.forName("UTF-8"))); runner.run(); runner.assertAllFlowFilesTransferred(PutSyslog.REL_SUCCESS, 1); Assert.assertEquals(1, sender.messages.size()); Assert.assertEquals(expectedMessage, sender.messages.get(0)); final List<ProvenanceEventRecord> events = runner.getProvenanceEvents(); Assert.assertNotNull(events); Assert.assertEquals(1, events.size()); final ProvenanceEventRecord event = events.get(0); Assert.assertEquals(ProvenanceEventType.SEND, event.getEventType()); Assert.assertEquals("UDP://localhost:12345", event.getTransitUri()); }
Assert.assertTrue("transit uri must be set and start with proper protocol", event.getTransitUri().toLowerCase().startsWith("tcp"));
@Test public void testValidMessageStaticPropertiesTcp() { final String pri = "34"; final String version = "1"; final String stamp = "2003-10-11T22:14:15.003Z"; final String host = "mymachine.example.com"; final String body = "su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8"; final String expectedMessage = "<" + pri + ">" + version + " " + stamp + " " + host + " " + body; runner.setProperty(PutSyslog.PROTOCOL, PutSyslog.TCP_VALUE); runner.setProperty(PutSyslog.MSG_PRIORITY, pri); runner.setProperty(PutSyslog.MSG_VERSION, version); runner.setProperty(PutSyslog.MSG_TIMESTAMP, stamp); runner.setProperty(PutSyslog.MSG_HOSTNAME, host); runner.setProperty(PutSyslog.MSG_BODY, body); runner.enqueue("incoming data".getBytes(Charset.forName("UTF-8"))); runner.run(); runner.assertAllFlowFilesTransferred(PutSyslog.REL_SUCCESS, 1); Assert.assertEquals(1, sender.messages.size()); Assert.assertEquals(expectedMessage, sender.messages.get(0).replace("\n", "")); final List<ProvenanceEventRecord> events = runner.getProvenanceEvents(); Assert.assertNotNull(events); Assert.assertEquals(1, events.size()); final ProvenanceEventRecord event = events.get(0); Assert.assertEquals(ProvenanceEventType.SEND, event.getEventType()); Assert.assertEquals("TCP://localhost:12345", event.getTransitUri()); }
@Test public void testPutProvenanceSendEventTransitUri() throws JMSException { final PutJMS putJMS = spy(new PutJMS()); final TestRunner runnerPut = TestRunners.newTestRunner(putJMS); runnerPut.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER); runnerPut.setProperty(JmsProperties.URL, TEST_URL); runnerPut.setProperty(JmsProperties.DESTINATION_TYPE, TEST_DEST_TYPE); runnerPut.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix()); runnerPut.setProperty(JmsProperties.ATTRIBUTES_TO_JMS_PROPS, "true"); runnerPut.enqueue("putGetMessage".getBytes()); runnerPut.run(); assertEquals(0, runnerPut.getFlowFilesForRelationship(PutJMS.REL_FAILURE).size()); assertEquals(1, runnerPut.getFlowFilesForRelationship(PutJMS.REL_SUCCESS).size()); final List<ProvenanceEventRecord> putProvenanceEvents = runnerPut.getProvenanceEvents(); assertEquals(1, putProvenanceEvents.size()); // Verify the transitUri is the same as that configured in the properties assertEquals(TEST_URL, putProvenanceEvents.get(0).getTransitUri()); final GetJMSQueue getJmsQueue = new GetJMSQueue(); final TestRunner runnerGet = TestRunners.newTestRunner(getJmsQueue); runnerGet.setProperty(JmsProperties.JMS_PROVIDER, TEST_PROVIDER); runnerGet.setProperty(JmsProperties.URL, TEST_URL); runnerGet.setProperty(JmsProperties.DESTINATION_NAME, TEST_DEST_NAME + testQueueSuffix()); runnerGet.setProperty(JmsProperties.ACKNOWLEDGEMENT_MODE, TEST_ACK_MODE); runnerGet.run(); assertEquals(1, runnerGet.getFlowFilesForRelationship(GetJMSQueue.REL_SUCCESS).size()); final List<MockFlowFile> flowFilesGet = runnerGet.getFlowFilesForRelationship(GetJMSQueue.REL_SUCCESS); assertEquals(1, flowFilesGet.size()); final MockFlowFile successFlowFile = flowFilesGet.get(0); successFlowFile.assertContentEquals("putGetMessage"); }
componentId = event.getComponentId(); componentType = event.getComponentType(); transitUri = event.getTransitUri(); sourceSystemFlowFileIdentifier = event.getSourceSystemFlowFileIdentifier(); uuid = event.getFlowFileUuid();