public static DatasetSplitId of(DatasetConfig config, DatasetSplit split, long splitVersion) { Preconditions.checkArgument(splitVersion > -1); EntityId datasetId = config.getId(); String splitKey = split.getSplitKey(); return of(datasetId, splitVersion, splitKey); }
@VisibleForTesting public Iterable<ReflectionGoal> getReflectionGoals(final NamespaceKey path, final String reflectionName) { try { DatasetConfig config = namespaceService.get().getDataset(path); return FluentIterable.from(getReflectionsByDatasetId(config.getId().getId())).filter(new Predicate<ReflectionGoal>() { @Override public boolean apply(ReflectionGoal input) { return reflectionName.equals(input.getName()); }}); }catch(NamespaceException ex) { throw Throwables.propagate(ex); } }
/** * Check if split id for this dataset may need new split id * * See DX-13336 for details * * @param config the dataset config * @return true if this dataset might be using legacy/invalid datasetId. */ public static boolean mayRequireNewDatasetId(DatasetConfig config) { return RESERVED_DATASET_ID_CHARACTERS.matchesAnyOf(config.getId().getId()); }
@Override public Iterable<ReflectionGoal> getReflectionsByDatasetPath(NamespaceKey path) { try { DatasetConfig config = namespaceService.get().getDataset(path); return getReflectionsByDatasetId(config.getId().getId()); }catch(NamespaceException ex) { throw Throwables.propagate(ex); } }
@Override public String computeDigest(){ return String.format("%s|%s|%s", splits.computeDigest(), plugin.getName(), config.getId().getId()); }
public static SearchQuery getSplitsQuery(DatasetConfig datasetConfig) { Preconditions.checkNotNull(datasetConfig.getReadDefinition()); long splitVersion = Preconditions.checkNotNull(datasetConfig.getReadDefinition().getSplitVersion()); return getSplitsQuery(datasetConfig.getId(), splitVersion); }
/** * Create a range for the current split version of the given dataset * * @param datasetConfig the dataset config * @return a range which would contain all split ids for this dataset and its current split version */ public static Range<DatasetSplitId> getCurrentSplitRange(DatasetConfig datasetConfig){ final long splitVersion = datasetConfig.getReadDefinition().getSplitVersion(); return getSplitRange(datasetConfig.getId(), splitVersion); }
@Override public DatasetConfig getDataset() throws Exception { if (datasetConfig != null) { return datasetConfig; } datasetConfig = buildDataset(); readDefinition = buildMetadata(); datasetConfig.setReadDefinition(readDefinition); if(datasetConfig.getId() == null){ datasetConfig.setId(new EntityId(UUID.randomUUID().toString())); } return datasetConfig; }
public static SplitsPointer of(NamespaceService namespaceService, DatasetConfig datasetConfig) { final EntityId datasetId = Preconditions.checkNotNull(datasetConfig.getId()); final ReadDefinition readDefinition = Preconditions.checkNotNull(datasetConfig.getReadDefinition()); final long splitVersion = readDefinition.getSplitVersion(); int splitsCount = namespaceService.getSplitCount(new FindByCondition().setCondition(DatasetSplitId.getSplitsQuery(datasetConfig))); return new DatasetSplitsPointer(namespaceService, datasetId, splitVersion, splitsCount); }
/** * UNSAFE! Use {@code DatasetSplitId#getSplitRange(EntityId, long)} instead */ public static FindByRange<DatasetSplitId> unsafeGetSplitsRange(DatasetConfig config) { final long splitVersion = config.getReadDefinition().getSplitVersion(); final long nextSplitVersion = splitVersion + 1; final String datasetId = config.getId().getId(); // Unsafe way of constructing dataset split id!!! final DatasetSplitId start = new DatasetSplitId(SPLIT_ID_JOINER.join(datasetId, splitVersion, ""), datasetId, splitVersion, ""); final DatasetSplitId end = new DatasetSplitId(SPLIT_ID_JOINER.join(datasetId, nextSplitVersion, ""), datasetId, splitVersion, ""); return new FindByRange<DatasetSplitId>() .setStart(start, true) .setEnd(end, false); } /**
@Override public Iterable<ExternalReflection> getExternalReflectionByDatasetPath(List<String> datasetPath) { DatasetConfig datasetConfig; try { datasetConfig = namespaceService.get().getDataset(new NamespaceKey(datasetPath)); } catch (NamespaceException e) { throw UserException.validationError(e).build(logger); } if (datasetConfig == null) { throw UserException.validationError().message(String.format("Dataset %s not found", quotedCompound(datasetPath))).build(logger); } return externalReflectionStore.findByDatasetId(datasetConfig.getId().getId()); }
public static DatasetSplitId of(DatasetConfig config, String key) { return DatasetSplitId.ofUnsafe(config.getId(), config.getReadDefinition().getSplitVersion(), key); }
private List<String> getPartitionColumnsForDataSet(String folder) throws Exception { URL stream = (TestParquetPartitionColumns.class.getClassLoader() .getResource(folder)); File fileDir = new File(stream.getFile()); DatasetPath parquet = new DatasetPath(ImmutableList.of("dfs", fileDir.getAbsolutePath())); DatasetConfig config = addDataSet(parquet); DremioTable table = p(CatalogService.class).get().getCatalog(SchemaConfig.newBuilder (DEFAULT_USERNAME) .build()) .getTable(config.getId().getId()); return table.getDatasetConfig().getReadDefinition() .getPartitionColumnsList(); }
public static CatalogItem fromDatasetConfig(DatasetConfig datasetConfig, CollaborationTag tags) { DatasetSubType datasetType = DatasetSubType.PROMOTED; if (datasetConfig.getType() == DatasetType.PHYSICAL_DATASET) { datasetType = DatasetSubType.DIRECT; } else if (datasetConfig.getType() == DatasetType.VIRTUAL_DATASET) { datasetType = DatasetSubType.VIRTUAL; } return new Builder() .setId(datasetConfig.getId().getId()) .setPath(Lists.newArrayList(datasetConfig.getFullPathList())) .setTag(String.valueOf(datasetConfig.getTag())) .setType(CatalogItemType.DATASET) .setDatasetType(datasetType) .setTags(tags) .build(); }
public VirtualDatasetUI get(DatasetPath path, DatasetVersion version) throws DatasetNotFoundException, NamespaceException { try { final DatasetConfig datasetConfig = namespaceService.getDataset(path.toNamespaceKey()); final VirtualDatasetVersion virtualDatasetVersion = datasetVersions.get(new VersionDatasetKey(path, version)); if (virtualDatasetVersion == null) { throw new DatasetNotFoundException(path, format("Missing version %s.", version.toString())); } final VirtualDatasetUI virtualDatasetUI = toVirtualDatasetUI(virtualDatasetVersion) .setId(datasetConfig.getId().getId()) .setSavedTag(datasetConfig.getTag()); return virtualDatasetUI; } catch (NamespaceNotFoundException e) { throw new DatasetNotFoundException(path, format("Some path not found while looking for dataset %s, version %s.", path.toPathString(), version.toString()), e); } }
public static PhysicalDatasetConfig toPhysicalDatasetConfig(DatasetConfig datasetConfig) { checkNotNull(datasetConfig.getPhysicalDataset()); final com.dremio.service.namespace.dataset.proto.PhysicalDataset physicalDataset = datasetConfig.getPhysicalDataset(); final PhysicalDatasetConfig physicalDatasetConfig = new PhysicalDatasetConfig(); physicalDatasetConfig.setFormatSettings(physicalDataset.getFormatSettings()); physicalDatasetConfig.setFullPathList(datasetConfig.getFullPathList()); physicalDatasetConfig.setType(datasetConfig.getType()); physicalDatasetConfig.setName(datasetConfig.getName()); physicalDatasetConfig.setTag(datasetConfig.getTag()); physicalDatasetConfig.setId(datasetConfig.getId().getId()); return physicalDatasetConfig; }
protected static void requestRefresh(NamespaceKey datasetKey) throws NamespaceException { DatasetConfig dataset = getNamespaceService().getDataset(datasetKey); getReflectionService().requestRefresh(dataset.getId().getId()); }
public void put(VirtualDatasetUI ds, NamespaceAttribute... attributes) throws DatasetNotFoundException, NamespaceException { DatasetPath path = new DatasetPath(ds.getFullPathList()); validatePath(path); validate(path, ds); DatasetConfig datasetConfig = toVirtualDatasetVersion(ds).getDataset(); namespaceService.addOrUpdateDataset(path.toNamespaceKey(), datasetConfig, attributes); ds.setId(datasetConfig.getId().getId()); ds.setSavedTag(datasetConfig.getTag()); // Update this version of dataset with new occ version of dataset config from namespace. putVersion(ds); }
private void createVDS(String name, List<String> tags, String sql) throws Exception { if (sql == null) { sql = "select * from sys.version"; } List<String> path = Arrays.asList("searchSpace", name); NamespaceKey namespaceKey = new NamespaceKey(path); ViewCreatorFactory vcf = l(ViewCreatorFactory.class); vcf.get(DEFAULT_USERNAME).createView(path, sql, Collections.emptyList()); DatasetConfig dataset = newNamespaceService().getDataset(namespaceKey); if (tags != null) { Tags tagsEntity = new Tags(tags, null); CollaborationHelper collaborationHelper = getCollaborationHelper(); collaborationHelper.setTags(dataset.getId().getId(), tagsEntity); } }
private Reflection createReflection() { // create a reflection List<ReflectionField> displayFields = new ArrayList<>(); DremioTable table = newCatalogService().getCatalog(SchemaConfig.newBuilder(SystemUser.SYSTEM_USERNAME).build()).getTable(datasetId); for (int i = 0; i < table.getSchema().getFieldCount(); i++) { Field field = table.getSchema().getColumn(i); displayFields.add(new ReflectionField(field.getName())); } return Reflection.newRawReflection(null, "My Raw", null, null, null, table.getDatasetConfig().getId().getId(), null, null, true, null, displayFields, null, null, null, PartitionDistributionStrategy.CONSOLIDATED); }