public EntityId newMessage() { return new EntityId(); }
@Override public EntityId revert(byte[] v) { return new EntityId(StringSerializer.INSTANCE.revert(v)); }
@Override public EntityId fromJson(String v) throws IOException { return new EntityId(StringSerializer.INSTANCE.fromJson(v)); }
@Override public DatasetConfig getDataset() throws Exception { if (datasetConfig != null) { return datasetConfig; } datasetConfig = buildDataset(); readDefinition = buildMetadata(); datasetConfig.setReadDefinition(readDefinition); if(datasetConfig.getId() == null){ datasetConfig.setId(new EntityId(UUID.randomUUID().toString())); } return datasetConfig; }
void shallowSave(SourceTableDefinition accessor) throws NamespaceException{ NamespaceKey key = accessor.getName(); DatasetConfig shallow = new DatasetConfig(); shallow.setId(new EntityId().setId(UUID.randomUUID().toString())); shallow.setCreatedAt(System.currentTimeMillis()); shallow.setName(key.getName()); shallow.setFullPathList(key.getPathComponents()); shallow.setType(accessor.getType()); shallow.setSchemaVersion(DatasetHelper.CURRENT_VERSION); systemUserNamespaceService.addOrUpdateDataset(key, shallow); }
public static FolderConfig getFolderConfig(Folder folder) { FolderConfig config = new FolderConfig(); config.setId(new EntityId(folder.getId())); config.setFullPathList(folder.getPath()); config.setName(Iterables.getLast(folder.getPath())); if (folder.getTag() != null) { config.setTag(folder.getTag()); } return config; }
private DatasetConfig saveDataset(List<String> path, DatasetType type, Function<DatasetConfig, DatasetConfig> transformer) throws NamespaceException { final NamespaceKey key = new NamespaceKey(path); final byte[] binaryKey = NamespaceServiceImpl.getKey(key); final Optional<DatasetConfig> oldDataset = Optional.ofNullable(namespaceStore.get(binaryKey)).map(NameSpaceContainer::getDataset); final DatasetConfig datasetConfig = transformer.apply(new DatasetConfig() .setId(oldDataset.map(DatasetConfig::getId).orElse(new EntityId().setId(UUID.randomUUID().toString()))) .setName(path.get(path.size() - 1)) .setFullPathList(path) .setType(type) .setTag(oldDataset.map(DatasetConfig::getTag).orElse(null)) .setOwner("dremio")); final NameSpaceContainer container = new NameSpaceContainer() .setType(NameSpaceContainer.Type.DATASET) .setFullPathList(path) .setDataset(datasetConfig); namespaceStore.put(binaryKey, container); return datasetConfig; }
@JsonIgnore public SourceConfig asSourceConfig() { SourceConfig c = new SourceConfig(); c.setType(config.getType()); c.setName(name); c.setCtime(ctime); c.setImg(img); c.setConfig(config.toBytesString()); c.setDescription(description); c.setTag(tag); c.setAccelerationRefreshPeriod(accelerationRefreshPeriod); c.setAccelerationGracePeriod(accelerationGracePeriod); c.setAccelerationNeverExpire(Boolean.TRUE.equals(accelerationNeverExpire)); c.setAccelerationNeverRefresh(Boolean.TRUE.equals(accelerationNeverRefresh)); c.setMetadataPolicy(metadataPolicy.asMetadataPolicy()); c.setId(new EntityId(getId())); return c; }
public static SpaceConfig getSpaceConfig(Space space) { SpaceConfig config = new SpaceConfig(); config.setName(space.getName()); config.setId(new EntityId(space.getId())); if (space.getTag() != null) { config.setTag(space.getTag()); } config.setCtime(space.getCreatedAt()); return config; }
private void addHome(String name) throws Exception { final HomeConfig home = new HomeConfig() .setId(new EntityId().setId(UUID.randomUUID().toString())) .setOwner(name); final NameSpaceContainer container = new NameSpaceContainer() .setType(NameSpaceContainer.Type.HOME) .setFullPathList(Arrays.asList(name)) .setHome(home); namespaceStore.put(NamespaceServiceImpl.getKey(new NamespaceKey("@" + name)), container); }
private void newS3Source(KVStore<byte[], NameSpaceContainer> namespace, String path, S3PluginConfig s3PluginConfig) { final List<String> fullPathList = Arrays.asList(path); final SourceConfig config = new SourceConfig() .setId(new EntityId(UUID.randomUUID().toString())) .setName(path) .setConnectionConf(s3PluginConfig); namespace.put(NamespaceServiceImpl.getKey(new NamespaceKey(fullPathList)), new NameSpaceContainer() .setFullPathList(fullPathList) .setType(NameSpaceContainer.Type.SOURCE) .setSource(config)); } }
@Override public DatasetConfig getDataset() throws Exception { final DatasetConfig dataset; if(oldDataset == null) { dataset = new DatasetConfig() .setFullPathList(key.getPathComponents()) .setId(new EntityId(UUID.randomUUID().toString())) .setType(DatasetType.PHYSICAL_DATASET); } else { dataset = oldDataset; } return dataset .setName(key.getName()) .setReadDefinition(new ReadDefinition() .setScanStats(new ScanStats().setRecordCount(100l) .setScanFactor(ScanCostFactor.OTHER.getFactor()))) .setOwner(SystemUser.SYSTEM_USERNAME) .setPhysicalDataset(new PhysicalDataset()) .setRecordSchema(getSchema().toByteString()) .setSchemaVersion(DatasetHelper.CURRENT_VERSION); }
public static SpaceConfig addOrUpdateSpace(NamespaceService service, SpacePath spacePath, Space space) throws NamespaceException, UserNotFoundException { SpaceConfig spaceConfig = new SpaceConfig() .setId(space.getId() != null ? new EntityId(space.getId()) : null) .setName(space.getName()) .setDescription(space.getDescription()) .setTag(space.getVersion()); service.addOrUpdateSpace(spacePath.toNamespaceKey(), spaceConfig); return service.getSpace(spacePath.toNamespaceKey()); }
private void addSource(String name) throws Exception { final SourceConfig source = new SourceConfig() .setId(new EntityId().setId(UUID.randomUUID().toString())) .setName(name) .setType("test") .setCtime(100L) .setAccelerationRefreshPeriod(REFRESH_PERIOD_MS) .setAccelerationGracePeriod(GRACE_PERIOD_MS) .setMetadataPolicy(new MetadataPolicy().setDatasetDefinitionExpireAfterMs(TimeUnit.DAYS.toMillis(1))); final NameSpaceContainer container = new NameSpaceContainer() .setType(NameSpaceContainer.Type.SOURCE) .setFullPathList(Arrays.asList(name)) .setSource(source); namespaceStore.put(NamespaceServiceImpl.getKey(new NamespaceKey(name)), container); }
public SourceConfig toSourceConfig() { SourceConfig sourceConfig = new SourceConfig(); sourceConfig.setId(new EntityId(getId())); String tag = getTag(); if (tag != null) { sourceConfig.setTag(tag); } sourceConfig.setType(getConfig().getType()); sourceConfig.setConfig(getConfig().toBytesString()); sourceConfig.setName(getName()); sourceConfig.setDescription(getDescription()); sourceConfig.setCtime(this.createdAt); sourceConfig.setMetadataPolicy(getMetadataPolicy().toMetadataPolicy()); sourceConfig.setAccelerationGracePeriod(getAccelerationGracePeriodMs()); sourceConfig.setAccelerationRefreshPeriod(getAccelerationRefreshPeriodMs()); sourceConfig.setAccelerationNeverExpire(isAccelerationNeverExpire()); sourceConfig.setAccelerationNeverRefresh(isAccelerationNeverRefresh()); return sourceConfig; }
@Test public void testRemovingSensitiveFields() throws Exception { SourceConfig config = new SourceConfig(); config.setName("Foopy"); config.setId(new EntityId("id")); config.setTag("0"); config.setAccelerationGracePeriod(0L); config.setAccelerationRefreshPeriod(0L); APrivateSource priv = new APrivateSource(); priv.password = "hello"; config.setConnectionConf(priv); SourceResource sourceResource = new SourceResource(newSourceService(), null); SourceResource.SourceDeprecated source = sourceResource.fromSourceConfig(config); APrivateSource newConfig = (APrivateSource) source.getConfig(); // make sure the sensitive fields have been removed assertEquals(newConfig.password, ConnectionConf.USE_EXISTING_SECRET_VALUE); }
@Before public void setup() throws Exception { { SampleDataPopulator.addDefaultFirstUser(l(UserService.class), new NamespaceServiceImpl(l(KVStoreProvider.class))); final HDFSConf hdfsConfig = new HDFSConf(); hdfsConfig.hostname = host; hdfsConfig.port = port; SourceConfig source = new SourceConfig(); source.setName(SOURCE_NAME); source.setMetadataPolicy(CatalogService.DEFAULT_METADATA_POLICY_WITH_AUTO_PROMOTE); source.setConnectionConf(hdfsConfig); source.setId(new EntityId(SOURCE_ID)); source.setDescription(SOURCE_DESC); ((CatalogServiceImpl)l(CatalogService.class)).getSystemUserCatalog().createSource(source); } }
@Test public void testIdWithPercentageFromConfig() throws Exception { DatasetConfig datasetConfig = new DatasetConfig() .setId(new EntityId().setId("ds1%test")) .setReadDefinition(new ReadDefinition().setSplitVersion(0L)); DatasetSplitId split1 = DatasetSplitId.of(datasetConfig, new DatasetSplit().setSplitKey("s1"), 0L); DatasetSplitId split2 = DatasetSplitId.of(datasetConfig, new DatasetSplit().setSplitKey("s2"), 0L); DatasetSplitId split3 = DatasetSplitId.of(datasetConfig, new DatasetSplit().setSplitKey("s3"), 0L); assertEquals("ds1%25test_0_s1", split1.getSplitId()); assertEquals("ds1%25test_0_s2", split2.getSplitId()); assertEquals("ds1%25test_0_s3", split3.getSplitId()); }
@Test public void testIdFromConfig() throws Exception { DatasetConfig datasetConfig = new DatasetConfig() .setId(new EntityId().setId("ds1")) .setReadDefinition(new ReadDefinition().setSplitVersion(0L)); DatasetSplitId split1 = DatasetSplitId.of(datasetConfig, new DatasetSplit().setSplitKey("s1"), 0L); DatasetSplitId split2 = DatasetSplitId.of(datasetConfig, new DatasetSplit().setSplitKey("s2"), 0L); DatasetSplitId split3 = DatasetSplitId.of(datasetConfig, new DatasetSplit().setSplitKey("s3"), 0L); assertEquals("ds1_0_s1", split1.getSplitId()); assertEquals("ds1_0_s2", split2.getSplitId()); assertEquals("ds1_0_s3", split3.getSplitId()); }
@Test public void testIdWithUnderscoreFromConfig() throws Exception { DatasetConfig datasetConfig = new DatasetConfig() .setId(new EntityId().setId("ds1_test")) .setReadDefinition(new ReadDefinition().setSplitVersion(0L)); DatasetSplitId split1 = DatasetSplitId.of(datasetConfig, new DatasetSplit().setSplitKey("s1"), 0L); DatasetSplitId split2 = DatasetSplitId.of(datasetConfig, new DatasetSplit().setSplitKey("s2"), 0L); DatasetSplitId split3 = DatasetSplitId.of(datasetConfig, new DatasetSplit().setSplitKey("s3"), 0L); assertEquals("ds1%5Ftest_0_s1", split1.getSplitId()); assertEquals("ds1%5Ftest_0_s2", split2.getSplitId()); assertEquals("ds1%5Ftest_0_s3", split3.getSplitId()); }