public Origin newMessage() { return new Origin(); }
private void addSourcesAndOrigins(DocumentWriter writer, List<FieldOrigin> fieldOrigins) { if (notEmpty(fieldOrigins)) { Set<String> sources = Sets.newHashSet(); final List<String> empty = ImmutableList.of(); for (FieldOrigin fieldOrigin : fieldOrigins) { for (Origin origin : listNotNull(fieldOrigin.getOriginsList())) { // DX-3999: fix this in calcite final List<String> path = Optional.fromNullable(origin.getTableList()).or(empty); if (path.isEmpty()) { continue; } NamespaceKey dataset = new NamespaceKey(origin.getTableList()); sources.add(dataset.getRoot()); } } writer.write(DATASET_SOURCES, sources.toArray(new String[0])); } }
private void indexOrigins(final List<FieldOrigin> fieldOriginsList, Set<List<String>> parents, Map<Origin, String> refs) { for (FieldOrigin fieldOrigin : fieldOriginsList) { List<Origin> originsList = listNotNull(fieldOrigin.getOriginsList()); // if size != 1 then the field is derived. if (originsList.size() == 1) { Origin origin = originsList.get(0); // if the field was derived we can not use it // or we would need to be able to invert the expression // it has to be straight unchanged if (!origin.getDerived()) { parents.add(origin.getTableList()); // TODO(Julien): if the same column is referred more than once we could recommend all combinations by making refs a multimap refs.put(origin, fieldOrigin.getName()); } } } }
String leftColumAlias = refs.get(new Origin(leftColumn, false).setTableList(leftTable)); if (leftColumAlias != null) {
private void validateFieldOrigin(FieldOrigin fieldOrigin, String name, List<String> table, boolean derived, String col) { assertEquals(name, fieldOrigin.getName()); Origin oi = fieldOrigin.getOriginsList().iterator().next(); assertEquals(col, oi.getColumnName()); assertEquals(derived, oi.getDerived()); assertEquals(table, oi.getTableList()); }
private void validateField(int i, String name, List<String> table, String col) { assertEquals(name, fields.get(i).getName()); Origin oi = fields.get(i).getOriginsList().iterator().next(); assertEquals(col, oi.getColumnName()); assertEquals(table, oi.getTableList()); }
public void writeTo(Output output, FieldOrigin message) throws IOException { if(message.name == null) throw new UninitializedMessageException(message); output.writeString(1, message.name, false); if(message.origins != null) { for(Origin origins : message.origins) { if(origins != null) output.writeObject(2, origins, Origin.getSchema(), true); } } }
List<String> table = Origins.getTable(relColumnOrigin); String colName = Origins.getColName(relColumnOrigin); namedOrigins.add(new Origin(colName, relColumnOrigin.isDerived()).setTableList(table));
public void mergeFrom(Input input, FieldOrigin message) throws IOException { for(int number = input.readFieldNumber(this);; number = input.readFieldNumber(this)) { switch(number) { case 0: return; case 1: message.name = input.readString(); break; case 2: if(message.origins == null) message.origins = new ArrayList<Origin>(); message.origins.add(input.mergeObject(null, Origin.getSchema())); break; default: input.handleUnknownField(number, this); } } }
@Test public void testJobParentSearch() throws Exception { Job jobA1 = createJob("A1", asList("space1", "ds1"), "v1", "A", "space1", JobState.COMPLETED, "select * from LocalFS1.\"dac-sample1.json\"", 100L, 110L, QueryType.UI_RUN); jobA1.getJobAttempt().getInfo().setFieldOriginsList(asList( new FieldOrigin("foo") .setOriginsList(asList( new Origin("foo", false) .setTableList(asList("LocalFS1", "dac-sample1.json")) )) )); jobsService.storeJob(jobA1); List<Job> jobsForParent = ImmutableList.copyOf(jobsService.getJobsForParent(new NamespaceKey(asList("LocalFS1", "dac-sample1.json")), Integer.MAX_VALUE)); assertFalse(jobsForParent.isEmpty()); }
public static List<String> getSources(DatasetConfig datasetConfig) { final Set<String> sources = Sets.newHashSet(); if (datasetConfig.getType() == DatasetType.VIRTUAL_DATASET) { if (datasetConfig.getVirtualDataset().getFieldOriginsList() != null) { for (FieldOrigin fieldOrigin : datasetConfig.getVirtualDataset().getFieldOriginsList()) { for (Origin origin : listNotNull(fieldOrigin.getOriginsList())) { sources.add(origin.getTableList().get(0)); } } } } else { sources.add(datasetConfig.getFullPathList().get(0)); } return new ArrayList<>(sources); }
private static void updateDerivationAfterLearningOriginsAndAncestors(VirtualDatasetUI newDataset){ // only resolve if we need to, otherwise we should leave the previous derivation alone. (e.g. during a transform) if(newDataset.getDerivation() != Derivation.DERIVED_UNKNOWN){ return; } // if we have don't have one parent, we must have had issues detecting parents of SQL we generated, fallback. if(newDataset.getParentsList() != null && newDataset.getParentsList().size() != 1){ newDataset.setDerivation(Derivation.UNKNOWN); return; } final Set<List<String>> origins = new HashSet<>(); for(FieldOrigin col : listNotNull(newDataset.getFieldOriginsList())) { for(Origin colOrigin : listNotNull(col.getOriginsList())){ origins.add(colOrigin.getTableList()); } } // logic: if we have a single parent and that parent is also the only // table listed in field origins, then we are derived from a physical // dataset. Otherwise, we are a virtual dataset. if(origins.size() == 1 && origins.iterator().next().equals(newDataset.getParentsList().get(0).getDatasetPathList())){ newDataset.setDerivation(Derivation.DERIVED_PHYSICAL); } else { newDataset.setDerivation(Derivation.DERIVED_VIRTUAL); } }
for (FieldOrigin fieldOrigin : fieldOrigins) { for (Origin origin : listNotNull(fieldOrigin.getOriginsList())) { List<String> tableList = listNotNull(origin.getTableList()); if (!tableList.isEmpty()) { final NamespaceKey parentDatasetPath = new NamespaceKey(tableList);