void shallowSave(SourceTableDefinition accessor) throws NamespaceException{ NamespaceKey key = accessor.getName(); DatasetConfig shallow = new DatasetConfig(); shallow.setId(new EntityId().setId(UUID.randomUUID().toString())); shallow.setCreatedAt(System.currentTimeMillis()); shallow.setName(key.getName()); shallow.setFullPathList(key.getPathComponents()); shallow.setType(accessor.getType()); shallow.setSchemaVersion(DatasetHelper.CURRENT_VERSION); systemUserNamespaceService.addOrUpdateDataset(key, shallow); }
dataset.setCreatedAt(System.currentTimeMillis());
dacSample1.setCreatedAt(System.currentTimeMillis()); dacSample1.setFullPathList(dacSampleAllTypes.toPathList()); dacSample1.setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE); dacSample1.setCreatedAt(System.currentTimeMillis()); dacSample1.setFullPathList(dacSample1Path.toPathList()); dacSample1.setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE); dacSample2.setCreatedAt(System.currentTimeMillis()); dacSample2.setFullPathList(dacSample2Path.toPathList()); dacSample2.setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE);
config.setName(dataset.getPath().get(dataset.getPath().size() - 1)); config.setTag(dataset.getTag()); config.setCreatedAt(dataset.getCreatedAt()); PhysicalDataset physicalDataset = new PhysicalDataset(); physicalDataset.setFormatSettings(new JsonFileConfig().asFileConfig());
config.setName(dataset.getPath().get(dataset.getPath().size() - 1)); config.setTag(dataset.getTag()); config.setCreatedAt(dataset.getCreatedAt()); VirtualDataset virtualDataset = new VirtualDataset(); virtualDataset.setSql(dataset.getSql());
protected DatasetConfig doRenameDataset(NamespaceKey oldDatasetPath, NamespaceKey newDatasetPath, List<NameSpaceContainer> oldDatasetParentEntitiesOnPath, List<NameSpaceContainer> newDatasetParentEntitiesOnPath) throws NamespaceException { final String newDatasetName = newDatasetPath.getName(); final NamespaceInternalKey oldKey = new NamespaceInternalKey(oldDatasetPath, keyNormalization); final DatasetConfig datasetConfig = getEntity(oldDatasetPath, DATASET).getDataset(); if (isPhysicalDataset(datasetConfig.getType())) { throw UserException.validationError() .message("Failed to rename %s to %s. Renames on physical datasets are not allowed.", oldDatasetPath, newDatasetPath) .build(logger); } if (datasetConfig.getType() == DatasetType.PHYSICAL_DATASET_HOME_FILE && !newDatasetPath.getPathComponents().get(0).startsWith("@")) { throw UserException.validationError() .message("Failed to rename %s to %s. You cannot move a uploaded file in your home directory to a space.", oldDatasetPath, newDatasetPath) .build(logger); } datasetConfig.setName(newDatasetName); datasetConfig.setFullPathList(newDatasetPath.getPathComponents()); datasetConfig.setCreatedAt(System.currentTimeMillis()); NamespaceEntity newValue = NamespaceEntity.toEntity(DATASET, newDatasetPath, datasetConfig, keyNormalization); // in case of upgrade we may have a version here from previous versions, so clear out datasetConfig.setVersion(null); datasetConfig.setTag(null); namespace.put(newValue.getPathKey().getKey(), newValue.getContainer()); namespace.delete(oldKey.getKey()); return datasetConfig; }
public static DatasetConfig toDatasetConfig(FileConfig fileConfig, DatasetType datasetType, String owner, EntityId id) { final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setOwner(owner); datasetConfig.setFullPathList(fileConfig.getFullPathList()); datasetConfig.setName(fileConfig.getName()); datasetConfig.setOwner(fileConfig.getOwner()); datasetConfig.setTag(fileConfig.getTag()); datasetConfig.setType(datasetType); datasetConfig.setCreatedAt(fileConfig.getCtime()); datasetConfig.setId(id); datasetConfig.setPhysicalDataset(new com.dremio.service.namespace.dataset.proto.PhysicalDataset().setFormatSettings(fileConfig)); return datasetConfig; }
config.setName(dataset.getPath().get(dataset.getPath().size() - 1)); config.setTag(dataset.getTag()); config.setCreatedAt(dataset.getCreatedAt()); VirtualDataset virtualDataset = new VirtualDataset(); virtualDataset.setSql(dataset.getSql());
private static String addJsonTable(String tableName, String... jsonData) throws Exception { final File file = temp.newFile(tableName); final String dataFile = file.getAbsolutePath(); //TODO write each record in a separate file, so we can cause a union type for example try (PrintWriter writer = new PrintWriter(file)) { for (String record : jsonData) { writer.println(record); } } final DatasetPath path = new DatasetPath(ImmutableList.of("dfs", dataFile)); final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setTag(null) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.JSON)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); return dataFile; } }
/** * Carry over few properties from old dataset config to new one * @param oldConfig old dataset config from namespace * @param newConfig new dataset config thats about to be saved in namespace */ public static void copyFromOldConfig(DatasetConfig oldConfig, DatasetConfig newConfig) { if(oldConfig == null) { return; } newConfig.setId(oldConfig.getId()); newConfig.setTag(oldConfig.getTag()); newConfig.setCreatedAt(oldConfig.getCreatedAt()); newConfig.setType(oldConfig.getType()); newConfig.setFullPathList(oldConfig.getFullPathList()); newConfig.setOwner(oldConfig.getOwner()); // make sure to copy the acceleration settings from old to new config // newConfig may contain upgrade fileFormat physical settings if (oldConfig.getPhysicalDataset() != null) { if (newConfig.getPhysicalDataset() == null) { newConfig.setPhysicalDataset(new PhysicalDataset()); } } }
private String createPhysicalDSInHome(NamespaceService ns, String path, DatasetType datasetType) throws NamespaceException{ DatasetConfig datasetConfig = new DatasetConfig(); FilePath filePath = new FilePath(path); datasetConfig.setType(datasetType); datasetConfig.setFullPathList(filePath.toPathList()); datasetConfig.setName(filePath.getFileName().toString()); datasetConfig.setCreatedAt(System.currentTimeMillis()); datasetConfig.setTag(null); datasetConfig.setOwner("test_user"); datasetConfig.setPhysicalDataset(new PhysicalDataset()); ns.addOrUpdateDataset(filePath.toNamespaceKey(), datasetConfig); return datasetConfig.getTag(); }
private String createPhysicalDS(NamespaceService ns, String path, DatasetType datasetType) throws NamespaceException{ DatasetConfig datasetConfig = new DatasetConfig(); PhysicalDatasetPath physicalDatasetPath = new PhysicalDatasetPath(path); datasetConfig.setType(datasetType); datasetConfig.setFullPathList(physicalDatasetPath.toPathList()); datasetConfig.setName(physicalDatasetPath.getLeaf().getName()); datasetConfig.setCreatedAt(System.currentTimeMillis()); datasetConfig.setTag(null); datasetConfig.setOwner("test_user"); datasetConfig.setPhysicalDataset(new PhysicalDataset()); ns.addOrUpdateDataset(physicalDatasetPath.toNamespaceKey(), datasetConfig); return datasetConfig.getTag(); }
@Test public void testUpdatePDSWithChangedTypeShouldFail() throws Exception { Dataset dataset = new Dataset( "dataset-id", Dataset.DatasetType.PHYSICAL_DATASET, Arrays.asList("source", "path"), null, 0L, "1", null, null, null, null, null ); DatasetConfig config = new DatasetConfig(); config.setId(new EntityId(dataset.getId())); config.setFullPathList(dataset.getPath()); config.setName(dataset.getPath().get(dataset.getPath().size() - 1)); config.setTag(dataset.getTag()); config.setCreatedAt(dataset.getCreatedAt()); VirtualDataset virtualDataset = new VirtualDataset(); virtualDataset.setSql(dataset.getSql()); virtualDataset.setContextList(dataset.getSqlContext()); config.setType(VIRTUAL_DATASET); config.setVirtualDataset(virtualDataset); when(namespaceService.findDatasetByUUID(dataset.getId())).thenReturn(config); thrown.expect(IllegalArgumentException.class); catalogServiceHelper.updateCatalogItem(dataset, dataset.getId()); }
public void addJson(DatasetPath path, DatasetPath vdsPath) throws Exception { final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setTag(null) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.JSON)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); createDatasetFromParentAndSave(vdsPath, path.toPathString()); }
protected static DatasetConfig addJson(DatasetPath path) throws Exception { final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setTag(null) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.JSON)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); return nsService.getDataset(path.toNamespaceKey()); }
protected DatasetConfig addDataSet(DatasetPath path) throws Exception { final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FOLDER) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.PARQUET).setCtime(1L).setOwner (DEFAULT_USERNAME)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); return nsService.getDataset(path.toNamespaceKey()); } }
public static VirtualDatasetVersion toVirtualDatasetVersion(VirtualDatasetUI virtualDatasetUI) { final VirtualDatasetVersion vvds = new VirtualDatasetVersion(); final DatasetConfig datasetConfig = new DatasetConfig(); final VirtualDataset virtualDataset = toVirtualDataset(virtualDatasetUI); vvds.setLastTransform(virtualDatasetUI.getLastTransform()); vvds.setState(virtualDatasetUI.getState()); vvds.setPreviousVersion(virtualDatasetUI.getPreviousVersion()); vvds.setNamed(virtualDatasetUI.getIsNamed()); vvds.setDerivation(virtualDatasetUI.getDerivation()); datasetConfig.setName(virtualDatasetUI.getName()); datasetConfig.setOwner(virtualDatasetUI.getOwner()); datasetConfig.setType(DatasetType.VIRTUAL_DATASET); datasetConfig.setCreatedAt(virtualDatasetUI.getCreatedAt()); datasetConfig.setFullPathList(virtualDatasetUI.getFullPathList()); datasetConfig.setTag(virtualDatasetUI.getSavedTag()); datasetConfig.setVirtualDataset(virtualDataset); datasetConfig.setRecordSchema(virtualDatasetUI.getRecordSchema()); if (virtualDatasetUI.getId() != null) { datasetConfig.setId(new EntityId(virtualDatasetUI.getId())); } vvds.setDataset(datasetConfig); return vvds; }