private void addSourcesAndOrigins(DocumentWriter writer, List<FieldOrigin> fieldOrigins) { if (notEmpty(fieldOrigins)) { Set<String> sources = Sets.newHashSet(); final List<String> empty = ImmutableList.of(); for (FieldOrigin fieldOrigin : fieldOrigins) { for (Origin origin : listNotNull(fieldOrigin.getOriginsList())) { // DX-3999: fix this in calcite final List<String> path = Optional.fromNullable(origin.getTableList()).or(empty); if (path.isEmpty()) { continue; } NamespaceKey dataset = new NamespaceKey(origin.getTableList()); sources.add(dataset.getRoot()); } } writer.write(DATASET_SOURCES, sources.toArray(new String[0])); } }
private void indexOrigins(final List<FieldOrigin> fieldOriginsList, Set<List<String>> parents, Map<Origin, String> refs) { for (FieldOrigin fieldOrigin : fieldOriginsList) { List<Origin> originsList = listNotNull(fieldOrigin.getOriginsList()); // if size != 1 then the field is derived. if (originsList.size() == 1) { Origin origin = originsList.get(0); // if the field was derived we can not use it // or we would need to be able to invert the expression // it has to be straight unchanged if (!origin.getDerived()) { parents.add(origin.getTableList()); // TODO(Julien): if the same column is referred more than once we could recommend all combinations by making refs a multimap refs.put(origin, fieldOrigin.getName()); } } } }
public static List<String> getSources(DatasetConfig datasetConfig) { final Set<String> sources = Sets.newHashSet(); if (datasetConfig.getType() == DatasetType.VIRTUAL_DATASET) { if (datasetConfig.getVirtualDataset().getFieldOriginsList() != null) { for (FieldOrigin fieldOrigin : datasetConfig.getVirtualDataset().getFieldOriginsList()) { for (Origin origin : listNotNull(fieldOrigin.getOriginsList())) { sources.add(origin.getTableList().get(0)); } } } } else { sources.add(datasetConfig.getFullPathList().get(0)); } return new ArrayList<>(sources); }
private static void updateDerivationAfterLearningOriginsAndAncestors(VirtualDatasetUI newDataset){ // only resolve if we need to, otherwise we should leave the previous derivation alone. (e.g. during a transform) if(newDataset.getDerivation() != Derivation.DERIVED_UNKNOWN){ return; } // if we have don't have one parent, we must have had issues detecting parents of SQL we generated, fallback. if(newDataset.getParentsList() != null && newDataset.getParentsList().size() != 1){ newDataset.setDerivation(Derivation.UNKNOWN); return; } final Set<List<String>> origins = new HashSet<>(); for(FieldOrigin col : listNotNull(newDataset.getFieldOriginsList())) { for(Origin colOrigin : listNotNull(col.getOriginsList())){ origins.add(colOrigin.getTableList()); } } // logic: if we have a single parent and that parent is also the only // table listed in field origins, then we are derived from a physical // dataset. Otherwise, we are a virtual dataset. if(origins.size() == 1 && origins.iterator().next().equals(newDataset.getParentsList().get(0).getDatasetPathList())){ newDataset.setDerivation(Derivation.DERIVED_PHYSICAL); } else { newDataset.setDerivation(Derivation.DERIVED_VIRTUAL); } }
private void validateField(int i, String name, List<String> table, String col) { assertEquals(name, fields.get(i).getName()); Origin oi = fields.get(i).getOriginsList().iterator().next(); assertEquals(col, oi.getColumnName()); assertEquals(table, oi.getTableList()); }
private void validateFieldOrigin(FieldOrigin fieldOrigin, String name, List<String> table, boolean derived, String col) { assertEquals(name, fieldOrigin.getName()); Origin oi = fieldOrigin.getOriginsList().iterator().next(); assertEquals(col, oi.getColumnName()); assertEquals(derived, oi.getDerived()); assertEquals(table, oi.getTableList()); }
for (FieldOrigin fieldOrigin : fieldOrigins) { for (Origin origin : listNotNull(fieldOrigin.getOriginsList())) { List<String> tableList = listNotNull(origin.getTableList()); if (!tableList.isEmpty()) { final NamespaceKey parentDatasetPath = new NamespaceKey(tableList);