public DatasetConfig newMessage() { return new DatasetConfig(); }
void shallowSave(SourceTableDefinition accessor) throws NamespaceException{ NamespaceKey key = accessor.getName(); DatasetConfig shallow = new DatasetConfig(); shallow.setId(new EntityId().setId(UUID.randomUUID().toString())); shallow.setCreatedAt(System.currentTimeMillis()); shallow.setName(key.getName()); shallow.setFullPathList(key.getPathComponents()); shallow.setType(accessor.getType()); shallow.setSchemaVersion(DatasetHelper.CURRENT_VERSION); systemUserNamespaceService.addOrUpdateDataset(key, shallow); }
public static void addFile(NamespaceService ns, List<String> path) throws Exception { NamespaceKey filePath = new NamespaceKey(path); final boolean isHome = path.get(0).startsWith("@"); final DatasetConfig ds = new DatasetConfig() .setType(isHome ? DatasetType.PHYSICAL_DATASET_HOME_FILE : DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig())); ns.addOrUpdateDataset(filePath, ds); }
private DatasetConfig saveDataset(List<String> path, DatasetType type, Function<DatasetConfig, DatasetConfig> transformer) throws NamespaceException { final NamespaceKey key = new NamespaceKey(path); final byte[] binaryKey = NamespaceServiceImpl.getKey(key); final Optional<DatasetConfig> oldDataset = Optional.ofNullable(namespaceStore.get(binaryKey)).map(NameSpaceContainer::getDataset); final DatasetConfig datasetConfig = transformer.apply(new DatasetConfig() .setId(oldDataset.map(DatasetConfig::getId).orElse(new EntityId().setId(UUID.randomUUID().toString()))) .setName(path.get(path.size() - 1)) .setFullPathList(path) .setType(type) .setTag(oldDataset.map(DatasetConfig::getTag).orElse(null)) .setOwner("dremio")); final NameSpaceContainer container = new NameSpaceContainer() .setType(NameSpaceContainer.Type.DATASET) .setFullPathList(path) .setDataset(datasetConfig); namespaceStore.put(binaryKey, container); return datasetConfig; }
@Test public void testSanitizing() throws Exception { QlikAppMessageBodyGenerator generator = new QlikAppMessageBodyGenerator(); VirtualDataset dataset = new VirtualDataset() .setSqlFieldsList(Arrays.asList(new ViewFieldType("testdetail2", "STRUCTURED"))); DatasetConfig datasetConfig = new DatasetConfig() .setName("evil /!@ *-=+{}<>,~ characters") .setType(DatasetType.VIRTUAL_DATASET) .setFullPathList(DatasetTool.TMP_DATASET_PATH.toPathList()) .setVirtualDataset(dataset); MultivaluedMap<String, Object> httpHeaders = new MultivaluedHashMap<>(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); assertTrue(generator.isWriteable(datasetConfig.getClass(), null, null, WebServer.MediaType.TEXT_PLAIN_QLIK_APP_TYPE)); generator.writeTo(datasetConfig, DatasetConfig.class, null, new Annotation[] {}, WebServer.MediaType.TEXT_PLAIN_QLIK_APP_TYPE, httpHeaders, baos); String script = new String(baos.toByteArray(), UTF_8); assertTrue(script.contains("evil_characters: DIRECT QUERY")); } }
public static DatasetConfig toDatasetConfig(PhysicalDatasetConfig physicalDatasetConfig, String owner) { final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setOwner(owner); datasetConfig.setFullPathList(physicalDatasetConfig.getFullPathList()); datasetConfig.setName(physicalDatasetConfig.getName()); datasetConfig.setType(physicalDatasetConfig.getType()); datasetConfig.setTag(physicalDatasetConfig.getTag()); datasetConfig.setPhysicalDataset(new com.dremio.service.namespace.dataset.proto.PhysicalDataset().setFormatSettings( physicalDatasetConfig.getFormatSettings())); return datasetConfig; }
public static DatasetConfig toDatasetConfig(FileConfig fileConfig, DatasetType datasetType, String owner, EntityId id) { final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setOwner(owner); datasetConfig.setFullPathList(fileConfig.getFullPathList()); datasetConfig.setName(fileConfig.getName()); datasetConfig.setOwner(fileConfig.getOwner()); datasetConfig.setTag(fileConfig.getTag()); datasetConfig.setType(datasetType); datasetConfig.setCreatedAt(fileConfig.getCtime()); datasetConfig.setId(id); datasetConfig.setPhysicalDataset(new com.dremio.service.namespace.dataset.proto.PhysicalDataset().setFormatSettings(fileConfig)); return datasetConfig; }
public void addPhysicalDataset(final DatasetPath path, final DatasetType type) throws Exception { NamespaceKey datasetPath = path.toNamespaceKey(); final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setName(datasetPath.getName()); datasetConfig.setType(type); datasetConfig.setPhysicalDataset(new PhysicalDataset()); getNamespaceService().tryCreatePhysicalDataset(datasetPath, datasetConfig); }
public void addPhysicalDataset(final DatasetPath path, final DatasetType type) throws Exception { NamespaceKey datasetPath = path.toNamespaceKey(); final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setName(datasetPath.getName()); datasetConfig.setType(type); datasetConfig.setPhysicalDataset(new PhysicalDataset()); getNamespaceService().tryCreatePhysicalDataset(datasetPath, datasetConfig); }
private void createSpaceAndVDS(NamespaceKey spacePath, List<String> vdsPath) throws NamespaceException { // create space SpaceConfig spaceConfig = new SpaceConfig(); spaceConfig.setName(spacePath.getRoot()); newNamespaceService().addOrUpdateSpace(spacePath, spaceConfig); // create vds VirtualDataset virtualDataset = new VirtualDataset(); virtualDataset.setSql("select * from sys.version"); DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setName(vdsPath.get(vdsPath.size() - 1)); datasetConfig.setFullPathList(vdsPath); datasetConfig.setType(DatasetType.VIRTUAL_DATASET); datasetConfig.setVirtualDataset(virtualDataset); getSabotContext().getViewCreator(SystemUser.SYSTEM_USERNAME).createView(vdsPath, "select * from sys.version", null); } }
private String createPhysicalDSInHome(NamespaceService ns, String path, DatasetType datasetType) throws NamespaceException{ DatasetConfig datasetConfig = new DatasetConfig(); FilePath filePath = new FilePath(path); datasetConfig.setType(datasetType); datasetConfig.setFullPathList(filePath.toPathList()); datasetConfig.setName(filePath.getFileName().toString()); datasetConfig.setCreatedAt(System.currentTimeMillis()); datasetConfig.setTag(null); datasetConfig.setOwner("test_user"); datasetConfig.setPhysicalDataset(new PhysicalDataset()); ns.addOrUpdateDataset(filePath.toNamespaceKey(), datasetConfig); return datasetConfig.getTag(); }
private String createPhysicalDS(NamespaceService ns, String path, DatasetType datasetType) throws NamespaceException{ DatasetConfig datasetConfig = new DatasetConfig(); PhysicalDatasetPath physicalDatasetPath = new PhysicalDatasetPath(path); datasetConfig.setType(datasetType); datasetConfig.setFullPathList(physicalDatasetPath.toPathList()); datasetConfig.setName(physicalDatasetPath.getLeaf().getName()); datasetConfig.setCreatedAt(System.currentTimeMillis()); datasetConfig.setTag(null); datasetConfig.setOwner("test_user"); datasetConfig.setPhysicalDataset(new PhysicalDataset()); ns.addOrUpdateDataset(physicalDatasetPath.toNamespaceKey(), datasetConfig); return datasetConfig.getTag(); }
public static void addPhysicalDS(NamespaceService ns, String filePath, DatasetType type, byte[] datasetSchema) throws Exception { NamespaceKey datasetPath = new NamespaceKey(PathUtils.parseFullPath(filePath)); final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setName(datasetPath.getName()); datasetConfig.setType(type); final PhysicalDataset physicalDataset = new PhysicalDataset(); if (datasetSchema != null) { datasetConfig.setRecordSchema(io.protostuff.ByteString.copyFrom(datasetSchema)); } datasetConfig.setSchemaVersion(DatasetHelper.CURRENT_VERSION); datasetConfig.setPhysicalDataset(physicalDataset); ns.tryCreatePhysicalDataset(datasetPath, datasetConfig); }
public static void addDS(NamespaceService ns, String name) throws Exception { final NamespaceKey dsPath = new NamespaceKey(PathUtils.parseFullPath(name)); final DatasetConfig ds = new DatasetConfig(); final VirtualDataset vds = new VirtualDataset(); vds.setVersion(DatasetVersion.newVersion()); ds.setType(DatasetType.VIRTUAL_DATASET); ds.setVirtualDataset(vds); ds.setFullPathList(dsPath.getPathComponents()); ds.setName(dsPath.getName()); ns.addOrUpdateDataset(dsPath, ds); }
public void addJson(DatasetPath path, DatasetPath vdsPath) throws Exception { final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setTag(null) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.JSON)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); createDatasetFromParentAndSave(vdsPath, path.toPathString()); }
protected static DatasetConfig addJson(DatasetPath path) throws Exception { final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setTag(null) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.JSON)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); return nsService.getDataset(path.toNamespaceKey()); }
protected DatasetConfig addDataSet(DatasetPath path) throws Exception { final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FOLDER) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.PARQUET).setCtime(1L).setOwner (DEFAULT_USERNAME)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); return nsService.getDataset(path.toNamespaceKey()); } }
@Test public void testIdWithPercentageFromConfig() throws Exception { DatasetConfig datasetConfig = new DatasetConfig() .setId(new EntityId().setId("ds1%test")) .setReadDefinition(new ReadDefinition().setSplitVersion(0L)); DatasetSplitId split1 = DatasetSplitId.of(datasetConfig, new DatasetSplit().setSplitKey("s1"), 0L); DatasetSplitId split2 = DatasetSplitId.of(datasetConfig, new DatasetSplit().setSplitKey("s2"), 0L); DatasetSplitId split3 = DatasetSplitId.of(datasetConfig, new DatasetSplit().setSplitKey("s3"), 0L); assertEquals("ds1%25test_0_s1", split1.getSplitId()); assertEquals("ds1%25test_0_s2", split2.getSplitId()); assertEquals("ds1%25test_0_s3", split3.getSplitId()); }
@Test public void testIdFromConfig() throws Exception { DatasetConfig datasetConfig = new DatasetConfig() .setId(new EntityId().setId("ds1")) .setReadDefinition(new ReadDefinition().setSplitVersion(0L)); DatasetSplitId split1 = DatasetSplitId.of(datasetConfig, new DatasetSplit().setSplitKey("s1"), 0L); DatasetSplitId split2 = DatasetSplitId.of(datasetConfig, new DatasetSplit().setSplitKey("s2"), 0L); DatasetSplitId split3 = DatasetSplitId.of(datasetConfig, new DatasetSplit().setSplitKey("s3"), 0L); assertEquals("ds1_0_s1", split1.getSplitId()); assertEquals("ds1_0_s2", split2.getSplitId()); assertEquals("ds1_0_s3", split3.getSplitId()); }
@Test public void testIdWithUnderscoreFromConfig() throws Exception { DatasetConfig datasetConfig = new DatasetConfig() .setId(new EntityId().setId("ds1_test")) .setReadDefinition(new ReadDefinition().setSplitVersion(0L)); DatasetSplitId split1 = DatasetSplitId.of(datasetConfig, new DatasetSplit().setSplitKey("s1"), 0L); DatasetSplitId split2 = DatasetSplitId.of(datasetConfig, new DatasetSplit().setSplitKey("s2"), 0L); DatasetSplitId split3 = DatasetSplitId.of(datasetConfig, new DatasetSplit().setSplitKey("s3"), 0L); assertEquals("ds1%5Ftest_0_s1", split1.getSplitId()); assertEquals("ds1%5Ftest_0_s2", split2.getSplitId()); assertEquals("ds1%5Ftest_0_s3", split3.getSplitId()); }