private Map<EndPoint, Set<String>> computeDestinationFields() { if (writeOperations == null) { computeAndValidateFieldLineageInfo(this.operations); } Map<EndPoint, Set<String>> destinationFields = new HashMap<>(); for (WriteOperation write : this.writeOperations) { Set<String> endPointFields = destinationFields.computeIfAbsent(write.getDestination(), k -> new HashSet<>()); for (InputField field : write.getInputs()) { endPointFields.add(field.getName()); } } return destinationFields; }
private Map<EndPoint, Set<String>> computeDestinationFields() { if (writeOperations == null) { computeAndValidateFieldLineageInfo(this.operations); } Map<EndPoint, Set<String>> destinationFields = new HashMap<>(); for (WriteOperation write : this.writeOperations) { Set<String> endPointFields = destinationFields.computeIfAbsent(write.getDestination(), k -> new HashSet<>()); for (InputField field : write.getInputs()) { endPointFields.add(field.getName()); } } return destinationFields; }
private void addMergeOperation(Set<String> stageInputs, Map<String, Operation> processedOperations) { Set<String> sortedInputs = new TreeSet<>(stageInputs); String mergeOperationName = prefixedOperationName(Joiner.on(SEPARATOR).join(sortedInputs), "merge"); String mergeDescription = "Merged stages: " + Joiner.on(",").join(sortedInputs); if (processedOperations.containsKey(mergeOperationName)) { // it is possible that same stages act as an input to multiple stages. // we should still only add single merge operation for them return; } List<InputField> inputFields = new ArrayList<>(); for (String inputStage : sortedInputs) { List<String> parentStages = findParentStages(inputStage); for (String parentStage : parentStages) { Map<String, String> fieldOrigins = stageOutputsWithOrigins.get(parentStage); for (Map.Entry<String, String> fieldOrigin : fieldOrigins.entrySet()) { inputFields.add(InputField.of(fieldOrigin.getValue(), fieldOrigin.getKey())); } } } Set<String> outputs = new LinkedHashSet<>(); for (InputField inputField : inputFields) { outputs.add(inputField.getName()); } TransformOperation merge = new TransformOperation(mergeOperationName, mergeDescription, inputFields, new ArrayList<>(outputs)); processedOperations.put(merge.getName(), merge); }
private Map<EndPointField, Set<EndPointField>> computeIncomingSummary() { if (writeOperations == null) { computeAndValidateFieldLineageInfo(this.operations); } Map<EndPointField, Set<EndPointField>> summary = new HashMap<>(); for (WriteOperation write : writeOperations) { List<InputField> inputs = write.getInputs(); for (InputField input : inputs) { computeIncomingSummaryHelper(new EndPointField(write.getDestination(), input.getName()), operationsMap.get(input.getOrigin()), write, summary); } } return summary; }
private Map<EndPointField, Set<EndPointField>> computeIncomingSummary() { if (writeOperations == null) { computeAndValidateFieldLineageInfo(this.operations); } Map<EndPointField, Set<EndPointField>> summary = new HashMap<>(); for (WriteOperation write : writeOperations) { List<InputField> inputs = write.getInputs(); for (InputField input : inputs) { computeIncomingSummaryHelper(new EndPointField(write.getDestination(), input.getName()), operationsMap.get(input.getOrigin()), write, summary); } } return summary; }
write.getInputs().stream().filter(input -> input.getName().equals(destinationField.getField())) .collect(Collectors.toSet());
write.getInputs().stream().filter(input -> input.getName().equals(destinationField.getField())) .collect(Collectors.toSet());
for (InputField inputField : inputFields) { if (inputField.getOrigin().equals(currentOperation.getName())) { sourceEndPointFields.add(new EndPointField(source, inputField.getName()));
for (InputField inputField : inputFields) { if (inputField.getOrigin().equals(currentOperation.getName())) { sourceEndPointFields.add(new EndPointField(source, inputField.getName()));