@Override public DatasetConfig apply(Entry<NamespaceKey, NameSpaceContainer> input) { return input.getValue().getDataset(); }}).toSortedList(new Comparator<DatasetConfig>(){ @Override
@Override public DatasetConfig findDatasetByUUID(String uuid) { NameSpaceContainer namespaceContainer = getByIndex(DatasetIndexKeys.DATASET_UUID, uuid); return (namespaceContainer!=null)?namespaceContainer.getDataset():null; }
@Override public DatasetConfig getDataset(NamespaceKey datasetPath) throws NamespaceException { return getEntity(datasetPath, DATASET).getDataset(); }
private static DatasetType getType(NamespaceService service, List<String> namespacePath){ try{ List<NameSpaceContainer> containers = service.getEntities(Collections.singletonList(new NamespaceKey(namespacePath))); if(containers != null && !containers.isEmpty()){ NameSpaceContainer container = containers.get(0); if(container.getType() == Type.DATASET){ return container.getDataset().getType(); } } }catch(Exception ex){ } return null; }
public List<ResourceTreeEntity> listPath(NamespaceKey root, boolean showDatasets) throws NamespaceException, UnsupportedEncodingException { final List<ResourceTreeEntity> resources = Lists.newArrayList(); for (NameSpaceContainer container : namespaceService.get().list(root)) { if (container.getType() == Type.FOLDER) { resources.add(new ResourceTreeEntity(container.getFolder())); } else if (showDatasets && container.getType() == Type.DATASET) { resources.add(new ResourceTreeEntity(container.getDataset())); } } return resources; }
@GET @Path("search") @Produces(MediaType.APPLICATION_JSON) public DatasetSearchUIs searchDatasets(@QueryParam("filter") String filters, @QueryParam("sort") String sortColumn, @QueryParam("order") SortOrder order) throws NamespaceException, DatasetVersionNotFoundException { final DatasetSearchUIs datasets = new DatasetSearchUIs(); for (SearchContainer searchEntity : catalogServiceHelper.searchByQuery(filters)) { if (searchEntity.getNamespaceContainer().getType().equals(NameSpaceContainer.Type.DATASET)) { datasets.add(new DatasetSearchUI(searchEntity.getNamespaceContainer().getDataset(), searchEntity.getCollaborationTag())); } } return datasets; }
@GET @Path("parents") @Produces(APPLICATION_JSON) public List<ParentDatasetUI> getParents() throws DatasetNotFoundException, NamespaceException { final VirtualDatasetUI virtualDatasetUI = datasetService.get(datasetPath, version); final List<ParentDatasetUI> parentDatasetUIs = Lists.newArrayList(); final List<NamespaceKey> parentDatasetPaths = Lists.newArrayList(); for (ParentDataset parentDataset : virtualDatasetUI.getParentsList()) { parentDatasetPaths.add(new NamespaceKey(parentDataset.getDatasetPathList())); } for (NameSpaceContainer nameSpaceContainer : datasetService.getNamespaceService().getEntities(parentDatasetPaths)) { if (nameSpaceContainer != null && nameSpaceContainer.getType() == Type.DATASET) { parentDatasetUIs.add(new ParentDatasetUI(nameSpaceContainer.getFullPathList(), nameSpaceContainer.getDataset().getType())); } } return parentDatasetUIs; }
@Override public void convert(KVStoreProvider.DocumentWriter writer, byte[] id, SearchContainer record) { final NameSpaceContainer namespaceContainer = record.getNamespaceContainer(); final List<String> fullPathList = namespaceContainer.getFullPathList(); writer.write(PATH_UNQUOTED_LC, fullPathList.stream() .map(String::toLowerCase) .toArray(String[]::new)); writer.write(NAME_LC, fullPathList.get(fullPathList.size() - 1).toLowerCase()); if (namespaceContainer.getType() == NameSpaceContainer.Type.DATASET) { String[] columns = NamespaceConverter.getColumnsLowerCase(namespaceContainer.getDataset()); if (columns.length > 0) { writer.write(DATASET_COLUMNS_LC, columns); } } // check if the namespace entity has any tags and index them final CollaborationTag collaborationTag = record.getCollaborationTag(); if (collaborationTag != null && collaborationTag.getTagsList() != null) { // store lowercase and all permutations writer.write(TAGS_LC, collaborationTag.getTagsList().stream() .map(String::toLowerCase).toArray(String[]::new)); } } }
public static String getVersion(NamespaceKey namespaceKey, NamespaceService namespaceService) throws NamespaceException { NameSpaceContainer container = namespaceService.getEntities(Arrays.asList(namespaceKey)).get(0); switch (container.getType()) { case SOURCE: return container.getSource().getTag(); case SPACE: return container.getSpace().getTag(); case HOME: return container.getHome().getTag(); case FOLDER: return container.getFolder().getTag(); case DATASET: return container.getDataset().getTag(); default: throw new RuntimeException("Invalid container type"); } } }
/** * Fetches a single item from the filesystem plugin */ public SchemaEntity get(List<String> path, String userName) { try { final FileStatus status = getFS(userName).getFileStatus(PathUtils.toFSPath(resolveTableNameToValidPath(path))); final Set<List<String>> tableNames = Sets.newHashSet(); final NamespaceService ns = context.getNamespaceService(userName); final NamespaceKey folderNSKey = new NamespaceKey(path); if (ns.exists(folderNSKey)) { for(NameSpaceContainer entity : ns.list(folderNSKey)) { if (entity.getType() == Type.DATASET) { tableNames.add(resolveTableNameToValidPath(entity.getDataset().getFullPathList())); } } } List<String> p = PathUtils.toPathComponents(status.getPath()); return getSchemaEntity(status, tableNames, p); } catch (IOException | NamespaceException e) { throw new RuntimeException(e); } }
@Override public void upgrade(UpgradeContext context) throws Exception { final KVStoreProvider storeProvider = context.getKVStoreProvider(); final KVStore<byte[], NameSpaceContainer> namespace = storeProvider.getStore(NamespaceServiceImpl.NamespaceStoreCreator.class); final KVStore<DatasetSplitId, DatasetSplit> splitsStore = storeProvider.getStore(NamespaceServiceImpl.DatasetSplitCreator.class); int fixedSplitIds = 0; // namespace#find() returns entries ordered by depth, so sources will // be processed before folders, which will be processed before datasets for(Map.Entry<byte[], NameSpaceContainer> entry: namespace.find()) { final NameSpaceContainer container = entry.getValue(); if (container.getType() != NameSpaceContainer.Type.DATASET) { continue; } DatasetConfig config = entry.getValue().getDataset(); if (config.getType() == DatasetType.VIRTUAL_DATASET) { continue; } if (config.getReadDefinition() == null || config.getReadDefinition().getSplitVersion() == null) { continue; } if (!DatasetSplitId.mayRequireNewDatasetId(config)) { // Datasets which do not contain reserved characters are fine continue; } fixSplits(splitsStore, config); } System.out.printf(" Updated %d dataset splits with new ids.%n", fixedSplitIds); }
@Override public void deleteDataset(final NamespaceKey datasetPath, String version) throws NamespaceException { NameSpaceContainer container = deleteEntity(datasetPath, DATASET, version, true); if (container.getDataset().getType() == PHYSICAL_DATASET_SOURCE_FOLDER) { // create a folder so that any existing datasets under the folder are now visible addOrUpdateFolder(datasetPath, new FolderConfig() .setFullPathList(datasetPath.getPathComponents()) .setName(datasetPath.getName()) ); } }
@Override public String getTag(NameSpaceContainer value) { switch (value.getType()) { case DATASET: return value.getDataset().getTag(); case FOLDER: return value.getFolder().getTag(); case HOME: return value.getHome().getTag(); case SOURCE: return value.getSource().getTag(); case SPACE: return value.getSpace().getTag(); default: throw new UnsupportedOperationException("Unknown type: " + value.getType()); } }
@Override public Long getVersion(NameSpaceContainer value) { switch (value.getType()) { case DATASET: return value.getDataset().getVersion(); case FOLDER: return value.getFolder().getVersion(); case HOME: return value.getHome().getVersion(); case SOURCE: return value.getSource().getVersion(); case SPACE: return value.getSpace().getVersion(); default: throw new UnsupportedOperationException("Unknown type: " + value.getType()); } }
@Override public void setTag(NameSpaceContainer value, String version) { switch (value.getType()) { case DATASET: value.getDataset().setTag(version); break; case FOLDER: value.getFolder().setTag(version); break; case HOME: value.getHome().setTag(version); break; case SOURCE: value.getSource().setTag(version); break; case SPACE: value.getSpace().setTag(version); break; default: throw new UnsupportedOperationException("Unknown type: " + value.getType()); } }
@Override public void setVersion(NameSpaceContainer value, Long version) { switch (value.getType()) { case DATASET: value.getDataset().setVersion(version); break; case FOLDER: value.getFolder().setVersion(version); break; case HOME: value.getHome().setVersion(version); break; case SOURCE: value.getSource().setVersion(version); break; case SPACE: value.getSpace().setVersion(version); break; default: throw new UnsupportedOperationException("Unknown type: " + value.getType()); } }
/** helper method that returns the id of the entity in given container */ public static String getId(NameSpaceContainer container) { EntityId entityId; switch (container.getType()) { case SOURCE: entityId = container.getSource().getId(); break; case SPACE: entityId = container.getSpace().getId(); break; case HOME: entityId = container.getHome().getId(); break; case FOLDER: entityId = container.getFolder().getId(); break; case DATASET: entityId = container.getDataset().getId(); break; default: throw new RuntimeException("Invalid container type"); } return entityId != null ? entityId.getId() : null; }
protected void doTraverseAndDeleteChildren(final NameSpaceContainer child) throws NamespaceException { final NamespaceInternalKey childKey = new NamespaceInternalKey(new NamespaceKey(child.getFullPathList()), keyNormalization); traverseAndDeleteChildren(childKey, child); switch (child.getType()) { case FOLDER: namespace.delete(childKey.getKey(), child.getFolder().getTag()); break; case DATASET: namespace.delete(childKey.getKey(), child.getDataset().getTag()); break; default: // Only leaf level or intermediate namespace container types are expected here. throw new RuntimeException("Unexpected namespace container type: " + child.getType()); } }
/** helper method that sets the given id in given container */ static void setId(NameSpaceContainer container, String id) { switch (container.getType()) { case SOURCE: container.getSource().setId(new EntityId(id)); return; case SPACE: container.getSpace().setId(new EntityId(id)); return; case HOME: container.getHome().setId(new EntityId(id)); return; case FOLDER: container.getFolder().setId(new EntityId(id)); return; case DATASET: container.getDataset().setId(new EntityId(id)); return; default: throw new RuntimeException("Invalid container type"); } }
protected DatasetConfig doRenameDataset(NamespaceKey oldDatasetPath, NamespaceKey newDatasetPath, List<NameSpaceContainer> oldDatasetParentEntitiesOnPath, List<NameSpaceContainer> newDatasetParentEntitiesOnPath) throws NamespaceException { final String newDatasetName = newDatasetPath.getName(); final NamespaceInternalKey oldKey = new NamespaceInternalKey(oldDatasetPath, keyNormalization); final DatasetConfig datasetConfig = getEntity(oldDatasetPath, DATASET).getDataset(); if (isPhysicalDataset(datasetConfig.getType())) { throw UserException.validationError() .message("Failed to rename %s to %s. Renames on physical datasets are not allowed.", oldDatasetPath, newDatasetPath) .build(logger); } if (datasetConfig.getType() == DatasetType.PHYSICAL_DATASET_HOME_FILE && !newDatasetPath.getPathComponents().get(0).startsWith("@")) { throw UserException.validationError() .message("Failed to rename %s to %s. You cannot move a uploaded file in your home directory to a space.", oldDatasetPath, newDatasetPath) .build(logger); } datasetConfig.setName(newDatasetName); datasetConfig.setFullPathList(newDatasetPath.getPathComponents()); datasetConfig.setCreatedAt(System.currentTimeMillis()); NamespaceEntity newValue = NamespaceEntity.toEntity(DATASET, newDatasetPath, datasetConfig, keyNormalization); // in case of upgrade we may have a version here from previous versions, so clear out datasetConfig.setVersion(null); datasetConfig.setTag(null); namespace.put(newValue.getPathKey().getKey(), newValue.getContainer()); namespace.delete(oldKey.getKey()); return datasetConfig; }