private void loadIfNecessary() { if(datasetPointer != null){ return; } SplitsPointer splitsPointer; if(datasetConfig.getReadDefinition() != null) { splitsPointer = DatasetSplitsPointer.of(ns, datasetConfig); } else { try{ final DatasetConfig newDatasetConfig = datasetAccessor.getDataset(); newDatasetConfig.setId(datasetConfig.getId()); newDatasetConfig.setTag(datasetConfig.getTag()); List<DatasetSplit> splits = datasetAccessor.getSplits(); ns.addOrUpdateDataset(getName(), newDatasetConfig, splits); datasetConfig = newDatasetConfig; splitsPointer = MaterializedSplitsPointer.of(splits, splits.size()); } catch (Exception e) { throw new RuntimeException(e); } } datasetPointer = new TableMetadataImpl(pluginId, datasetConfig, user, splitsPointer); }
.thenReturn( new DatasetConfig() .setTag("0") );
private DatasetConfig saveDataset(List<String> path, DatasetType type, Function<DatasetConfig, DatasetConfig> transformer) throws NamespaceException { final NamespaceKey key = new NamespaceKey(path); final byte[] binaryKey = NamespaceServiceImpl.getKey(key); final Optional<DatasetConfig> oldDataset = Optional.ofNullable(namespaceStore.get(binaryKey)).map(NameSpaceContainer::getDataset); final DatasetConfig datasetConfig = transformer.apply(new DatasetConfig() .setId(oldDataset.map(DatasetConfig::getId).orElse(new EntityId().setId(UUID.randomUUID().toString()))) .setName(path.get(path.size() - 1)) .setFullPathList(path) .setType(type) .setTag(oldDataset.map(DatasetConfig::getTag).orElse(null)) .setOwner("dremio")); final NameSpaceContainer container = new NameSpaceContainer() .setType(NameSpaceContainer.Type.DATASET) .setFullPathList(path) .setDataset(datasetConfig); namespaceStore.put(binaryKey, container); return datasetConfig; }
@Override public void setTag(NameSpaceContainer value, String version) { switch (value.getType()) { case DATASET: value.getDataset().setTag(version); break; case FOLDER: value.getFolder().setTag(version); break; case HOME: value.getHome().setTag(version); break; case SOURCE: value.getSource().setTag(version); break; case SPACE: value.getSpace().setTag(version); break; default: throw new UnsupportedOperationException("Unknown type: " + value.getType()); } }
datasetConfig.setTag(currentConfig.getTag());
public static DatasetConfig toDatasetConfig(PhysicalDatasetConfig physicalDatasetConfig, String owner) { final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setOwner(owner); datasetConfig.setFullPathList(physicalDatasetConfig.getFullPathList()); datasetConfig.setName(physicalDatasetConfig.getName()); datasetConfig.setType(physicalDatasetConfig.getType()); datasetConfig.setTag(physicalDatasetConfig.getTag()); datasetConfig.setPhysicalDataset(new com.dremio.service.namespace.dataset.proto.PhysicalDataset().setFormatSettings( physicalDatasetConfig.getFormatSettings())); return datasetConfig; }
dataset.setTag(existingDatasetConfig.getTag());
config.setFullPathList(dataset.getPath()); config.setName(dataset.getPath().get(dataset.getPath().size() - 1)); config.setTag(dataset.getTag()); config.setCreatedAt(dataset.getCreatedAt()); VirtualDataset virtualDataset = new VirtualDataset();
@Test public void testDeleteDataset() throws Exception { Dataset dataset = new Dataset( "dataset-id", Dataset.DatasetType.VIRTUAL_DATASET, Arrays.asList("source", "path"), null, 0L, "1", null, "sql", null, null, null ); NameSpaceContainer namespaceContainer = new NameSpaceContainer(); namespaceContainer.setType(NameSpaceContainer.Type.DATASET); DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setId(new EntityId(dataset.getId())); datasetConfig.setType(VIRTUAL_DATASET); datasetConfig.setFullPathList(dataset.getPath()); datasetConfig.setTag(dataset.getTag()); namespaceContainer.setDataset(datasetConfig); when(namespaceService.getEntityById(dataset.getId())).thenReturn(namespaceContainer); DremioTable dremioTable = mock(DremioTable.class); when(dremioTable.getDatasetConfig()).thenReturn(datasetConfig); when(catalog.getTable(any(String.class))).thenReturn(dremioTable); catalogServiceHelper.deleteCatalogItem(dataset.getId(), "1"); verify(namespaceService, times(1)).deleteDataset(new NamespaceKey(dataset.getPath()), datasetConfig.getTag()); }
protected DatasetConfig doRenameDataset(NamespaceKey oldDatasetPath, NamespaceKey newDatasetPath, List<NameSpaceContainer> oldDatasetParentEntitiesOnPath, List<NameSpaceContainer> newDatasetParentEntitiesOnPath) throws NamespaceException { final String newDatasetName = newDatasetPath.getName(); final NamespaceInternalKey oldKey = new NamespaceInternalKey(oldDatasetPath, keyNormalization); final DatasetConfig datasetConfig = getEntity(oldDatasetPath, DATASET).getDataset(); if (isPhysicalDataset(datasetConfig.getType())) { throw UserException.validationError() .message("Failed to rename %s to %s. Renames on physical datasets are not allowed.", oldDatasetPath, newDatasetPath) .build(logger); } if (datasetConfig.getType() == DatasetType.PHYSICAL_DATASET_HOME_FILE && !newDatasetPath.getPathComponents().get(0).startsWith("@")) { throw UserException.validationError() .message("Failed to rename %s to %s. You cannot move a uploaded file in your home directory to a space.", oldDatasetPath, newDatasetPath) .build(logger); } datasetConfig.setName(newDatasetName); datasetConfig.setFullPathList(newDatasetPath.getPathComponents()); datasetConfig.setCreatedAt(System.currentTimeMillis()); NamespaceEntity newValue = NamespaceEntity.toEntity(DATASET, newDatasetPath, datasetConfig, keyNormalization); // in case of upgrade we may have a version here from previous versions, so clear out datasetConfig.setVersion(null); datasetConfig.setTag(null); namespace.put(newValue.getPathKey().getKey(), newValue.getContainer()); namespace.delete(oldKey.getKey()); return datasetConfig; }
public static DatasetConfig toDatasetConfig(FileConfig fileConfig, DatasetType datasetType, String owner, EntityId id) { final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setOwner(owner); datasetConfig.setFullPathList(fileConfig.getFullPathList()); datasetConfig.setName(fileConfig.getName()); datasetConfig.setOwner(fileConfig.getOwner()); datasetConfig.setTag(fileConfig.getTag()); datasetConfig.setType(datasetType); datasetConfig.setCreatedAt(fileConfig.getCtime()); datasetConfig.setId(id); datasetConfig.setPhysicalDataset(new com.dremio.service.namespace.dataset.proto.PhysicalDataset().setFormatSettings(fileConfig)); return datasetConfig; }
config.setFullPathList(dataset.getPath()); config.setName(dataset.getPath().get(dataset.getPath().size() - 1)); config.setTag(dataset.getTag()); config.setCreatedAt(dataset.getCreatedAt()); VirtualDataset virtualDataset = new VirtualDataset();
private static String addJsonTable(String tableName, String... jsonData) throws Exception { final File file = temp.newFile(tableName); final String dataFile = file.getAbsolutePath(); //TODO write each record in a separate file, so we can cause a union type for example try (PrintWriter writer = new PrintWriter(file)) { for (String record : jsonData) { writer.println(record); } } final DatasetPath path = new DatasetPath(ImmutableList.of("dfs", dataFile)); final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setTag(null) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.JSON)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); return dataFile; } }
/** * Carry over few properties from old dataset config to new one * @param oldConfig old dataset config from namespace * @param newConfig new dataset config thats about to be saved in namespace */ public static void copyFromOldConfig(DatasetConfig oldConfig, DatasetConfig newConfig) { if(oldConfig == null) { return; } newConfig.setId(oldConfig.getId()); newConfig.setTag(oldConfig.getTag()); newConfig.setCreatedAt(oldConfig.getCreatedAt()); newConfig.setType(oldConfig.getType()); newConfig.setFullPathList(oldConfig.getFullPathList()); newConfig.setOwner(oldConfig.getOwner()); // make sure to copy the acceleration settings from old to new config // newConfig may contain upgrade fileFormat physical settings if (oldConfig.getPhysicalDataset() != null) { if (newConfig.getPhysicalDataset() == null) { newConfig.setPhysicalDataset(new PhysicalDataset()); } } }
private String createPhysicalDS(NamespaceService ns, String path, DatasetType datasetType) throws NamespaceException{ DatasetConfig datasetConfig = new DatasetConfig(); PhysicalDatasetPath physicalDatasetPath = new PhysicalDatasetPath(path); datasetConfig.setType(datasetType); datasetConfig.setFullPathList(physicalDatasetPath.toPathList()); datasetConfig.setName(physicalDatasetPath.getLeaf().getName()); datasetConfig.setCreatedAt(System.currentTimeMillis()); datasetConfig.setTag(null); datasetConfig.setOwner("test_user"); datasetConfig.setPhysicalDataset(new PhysicalDataset()); ns.addOrUpdateDataset(physicalDatasetPath.toNamespaceKey(), datasetConfig); return datasetConfig.getTag(); }
private String createPhysicalDSInHome(NamespaceService ns, String path, DatasetType datasetType) throws NamespaceException{ DatasetConfig datasetConfig = new DatasetConfig(); FilePath filePath = new FilePath(path); datasetConfig.setType(datasetType); datasetConfig.setFullPathList(filePath.toPathList()); datasetConfig.setName(filePath.getFileName().toString()); datasetConfig.setCreatedAt(System.currentTimeMillis()); datasetConfig.setTag(null); datasetConfig.setOwner("test_user"); datasetConfig.setPhysicalDataset(new PhysicalDataset()); ns.addOrUpdateDataset(filePath.toNamespaceKey(), datasetConfig); return datasetConfig.getTag(); }
@Test public void testUpdatePDSWithChangedTypeShouldFail() throws Exception { Dataset dataset = new Dataset( "dataset-id", Dataset.DatasetType.PHYSICAL_DATASET, Arrays.asList("source", "path"), null, 0L, "1", null, null, null, null, null ); DatasetConfig config = new DatasetConfig(); config.setId(new EntityId(dataset.getId())); config.setFullPathList(dataset.getPath()); config.setName(dataset.getPath().get(dataset.getPath().size() - 1)); config.setTag(dataset.getTag()); config.setCreatedAt(dataset.getCreatedAt()); VirtualDataset virtualDataset = new VirtualDataset(); virtualDataset.setSql(dataset.getSql()); virtualDataset.setContextList(dataset.getSqlContext()); config.setType(VIRTUAL_DATASET); config.setVirtualDataset(virtualDataset); when(namespaceService.findDatasetByUUID(dataset.getId())).thenReturn(config); thrown.expect(IllegalArgumentException.class); catalogServiceHelper.updateCatalogItem(dataset, dataset.getId()); }
public void addJson(DatasetPath path, DatasetPath vdsPath) throws Exception { final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setTag(null) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.JSON)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); createDatasetFromParentAndSave(vdsPath, path.toPathString()); }
protected static DatasetConfig addJson(DatasetPath path) throws Exception { final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setTag(null) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.JSON)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); return nsService.getDataset(path.toNamespaceKey()); }
public static VirtualDatasetVersion toVirtualDatasetVersion(VirtualDatasetUI virtualDatasetUI) { final VirtualDatasetVersion vvds = new VirtualDatasetVersion(); final DatasetConfig datasetConfig = new DatasetConfig(); final VirtualDataset virtualDataset = toVirtualDataset(virtualDatasetUI); vvds.setLastTransform(virtualDatasetUI.getLastTransform()); vvds.setState(virtualDatasetUI.getState()); vvds.setPreviousVersion(virtualDatasetUI.getPreviousVersion()); vvds.setNamed(virtualDatasetUI.getIsNamed()); vvds.setDerivation(virtualDatasetUI.getDerivation()); datasetConfig.setName(virtualDatasetUI.getName()); datasetConfig.setOwner(virtualDatasetUI.getOwner()); datasetConfig.setType(DatasetType.VIRTUAL_DATASET); datasetConfig.setCreatedAt(virtualDatasetUI.getCreatedAt()); datasetConfig.setFullPathList(virtualDatasetUI.getFullPathList()); datasetConfig.setTag(virtualDatasetUI.getSavedTag()); datasetConfig.setVirtualDataset(virtualDataset); datasetConfig.setRecordSchema(virtualDatasetUI.getRecordSchema()); if (virtualDatasetUI.getId() != null) { datasetConfig.setId(new EntityId(virtualDatasetUI.getId())); } vvds.setDataset(datasetConfig); return vvds; }