@Override public NamespaceKey getName() { return delegate.getName(); }
public NamespaceKey getName(){ return delegate.getName(); }
@Override public NamespaceKey apply(SourceTableDefinition input) { return input.getName(); } }).toSet();
@Override public SourceTableDefinition getDataset(NamespaceKey datasetPath, DatasetConfig oldDataset, DatasetRetrievalOptions retrievalOptions) { for (SourceTableDefinition definition : datasets) { if (Objects.equal(datasetPath, definition.getName())) { return definition; } } return null; }
void completeSave(SourceTableDefinition accessor, DatasetConfig oldConfig, NamespaceAttribute... attributes){ try { DatasetConfig newConfig = accessor.getDataset(); if (oldConfig != null) { NamespaceUtils.copyFromOldConfig(oldConfig, newConfig); } completeSave(newConfig, accessor.getSplits(), attributes); }catch(Exception ex){ logger.warn("Failure while retrieving and saving dataset {}.", accessor.getName(), ex); } }
@Override public boolean datasetExists(NamespaceKey key) { for (SourceTableDefinition definition : datasets) { if (definition.getName().equals(key)) { return true; } } return false; }
private boolean refreshSourceNames() throws NamespaceException { final Set<NamespaceKey> foundKeys = Sets.newHashSet(systemUserNamespaceService.getAllDatasets(sourceKey)); boolean changed = false; try { for (SourceTableDefinition accessor : plugin.get() .getDatasets(SYSTEM_USERNAME, msp.getDefaultRetrievalOptions())) { // names only added, never removed. Removal can be done by the full refresh (refreshSource()) if (!foundKeys.contains(accessor.getName()) && accessor.isSaveable()) { try { saver.shallowSave(accessor); changed = true; } catch (ConcurrentModificationException ex) { logger.debug("Concurrent update, ignoring.", ex); } } } // persist last refresh for across restarts. SourceInternalData srcData = sourceDataStore.get(sourceKey); if(srcData == null) { srcData = new SourceInternalData(); srcData.setName(sourceKey.getRoot()); } srcData.setLastNameRefreshDateMs(System.currentTimeMillis()); sourceDataStore.put(sourceKey, srcData); } catch (Exception ex){ logger.warn("Failure while attempting to update metadata for source {}. Terminating update of this source.", sourceKey, ex); } return changed; }
void shallowSave(SourceTableDefinition accessor) throws NamespaceException{ NamespaceKey key = accessor.getName(); DatasetConfig shallow = new DatasetConfig(); shallow.setId(new EntityId().setId(UUID.randomUUID().toString())); shallow.setCreatedAt(System.currentTimeMillis()); shallow.setName(key.getName()); shallow.setFullPathList(key.getPathComponents()); shallow.setType(accessor.getType()); shallow.setSchemaVersion(DatasetHelper.CURRENT_VERSION); systemUserNamespaceService.addOrUpdateDataset(key, shallow); }
return refreshResult; if(knownKeys.add(accessor.getName())){ if(!accessor.isSaveable()){ continue; logger.debug("There was a concurrent exception while trying to refresh a dataset. This generally means another user did something that conflicted with the refresh mechnaism.", ex); removeFoldersOnPathFromOprhanSet(accessor.getName(), orphanedFolders);
String response = table.getName().getSchemaPath(); assertTrue(!(response.contains(".hidden"))); if(response.contains("nothidden")) {
private static SourceTableDefinition newDataset(final String dsPath) { final List<String> path = SqlUtils.parseSchemaPath(dsPath); SourceTableDefinition ret = mock(SourceTableDefinition.class); NamespaceKey datasetName = new NamespaceKey(path); when(ret.getName()).thenReturn(datasetName); BatchSchema schema = BatchSchema.newBuilder() .addField(new Field("string", FieldType.nullable(ArrowType.Utf8.INSTANCE), null)) .build(); DatasetConfig dsConfig = new DatasetConfig() .setName(Util.last(path)) .setFullPathList(path) .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setRecordSchema(ByteString.EMPTY) .setPhysicalDataset( new PhysicalDataset() .setFormatSettings(null)) .setSchemaVersion(DatasetHelper.CURRENT_VERSION) .setRecordSchema(schema.toByteString()) .setReadDefinition(new ReadDefinition()); try { when(ret.getDataset()).thenReturn(dsConfig); } catch (Exception ignored) { } when(ret.getType()).thenReturn(DatasetType.PHYSICAL_DATASET_SOURCE_FILE); when(ret.isSaveable()).thenReturn(true); return ret; }
final NamespaceKey canonicalKey = tableDefinition.getName();