@Override public IDataSource<DataSourceId> getDataSource() { try { DataSourceId sourceId = new DataSourceId(dataverse, dataset); return metadataProvider.lookupSourceInMetadata(sourceId); } catch (Exception me) { return null; } }
public static Pair<String, String> getDatasetInfo(AbstractDataSourceOperator op) throws AlgebricksException { DataSourceId srcId = (DataSourceId) op.getDataSource().getId(); return new Pair<>(srcId.getDataverseName(), srcId.getDatasourceName()); }
@Override public String toString() { return id.toString(); }
String outputType, String locations, LogicalVariable recordVar, IOptimizationContext context, List<LogicalVariable> pkVars) throws AlgebricksException { Dataset dataset = metadataProvider.findDataset(aqlId.getDataverseName(), targetDataset); ARecordType feedOutputType = (ARecordType) metadataProvider.findType(aqlId.getDataverseName(), outputType); Feed sourceFeed = metadataProvider.findFeed(aqlId.getDataverseName(), sourceFeedName); FeedConnection feedConnection = metadataProvider.findFeedConnection(aqlId.getDataverseName(), sourceFeedName, targetDataset); ARecordType metaType = null; throw new AlgebricksException("Feed to a dataset with metadata doesn't have meta type specified"); String dataverseName = aqlId.getDataverseName(); if (metaTypeName.contains(".")) { dataverseName = metaTypeName.substring(0, metaTypeName.indexOf('.'));
List<LogicalVariable> additionalNonFilterFields, RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec) throws AlgebricksException { String datasetName = dataSource.getId().getDatasourceName(); Dataset dataset = findDataset(dataSource.getId().getDataverseName(), datasetName); if (dataset == null) { throw new AlgebricksException( "Unknown dataset " + datasetName + " in dataverse " + dataSource.getId().getDataverseName());
public StorageComponentsDatasource(INodeDomain domain, int datasetId) throws AlgebricksException { super(new DataSourceId(StorageComponentsRewriter.STORAGE_COMPONENTS.getNamespace(), StorageComponentsRewriter.STORAGE_COMPONENTS.getName()), domain); this.datasetId = datasetId; }
new FeedConnectionId(getId().getDataverseName(), getId().getDatasourceName(), getTargetDataset()); FeedCollectOperatorDescriptor feedCollector = new FeedCollectOperatorDescriptor(jobSpec, feedConnectionId, feedOutputType, feedDesc, feedPolicy.getProperties(), getLocation());
public DatasetResourcesDatasource(INodeDomain domain, int datasetId) throws AlgebricksException { super(new DataSourceId(DatasetResourcesRewriter.DATASET_RESOURCES.getNamespace(), DatasetResourcesRewriter.DATASET_RESOURCES.getName()), domain); this.datasetId = datasetId; }
public static DataSource lookupSourceInMetadata(IClusterStateManager clusterStateManager, MetadataTransactionContext mdTxnCtx, DataSourceId aqlId) throws AlgebricksException { Dataset dataset = findDataset(mdTxnCtx, aqlId.getDataverseName(), aqlId.getDatasourceName()); if (dataset == null) { throw new AlgebricksException("Datasource with id " + aqlId + " was not found."); } IAType itemType = findType(mdTxnCtx, dataset.getItemTypeDataverseName(), dataset.getItemTypeName()); IAType metaItemType = findType(mdTxnCtx, dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName()); INodeDomain domain = findNodeDomain(clusterStateManager, mdTxnCtx, dataset.getNodeGroupName()); byte datasourceType = dataset.getDatasetType().equals(DatasetType.EXTERNAL) ? DataSource.Type.EXTERNAL_DATASET : DataSource.Type.INTERNAL_DATASET; return new DatasetDataSource(aqlId, dataset, itemType, metaItemType, datasourceType, dataset.getDatasetDetails(), domain); } }
public LoadableDataSource(Dataset targetDataset, IAType itemType, IAType metaItemType, String adapter, Map<String, String> properties) throws AlgebricksException, IOException { super(new DataSourceId("loadable_dv", "loadable_ds"), itemType, metaItemType, Type.LOADABLE, null); this.targetDataset = targetDataset; this.adapter = adapter; this.adapterProperties = properties; partitioningKeys = targetDataset.getPrimaryKeys(); ARecordType recType = (ARecordType) itemType; isPKAutoGenerated = ((InternalDatasetDetails) targetDataset.getDatasetDetails()).isAutogenerated(); if (isPKAutoGenerated) { // Since the key is auto-generated, we need to use another // record type (possibly nested) which has all fields except the PK recType = getStrippedPKType(new LinkedList<>(partitioningKeys.get(0)), recType); } schemaTypes = new IAType[] { recType }; }
LogicalVariable payload, List<LogicalVariable> additionalNonKeyFields, JobGenContext context, JobSpecification spec) throws AlgebricksException { String dataverseName = dataSource.getId().getDataverseName(); String datasetName = dataSource.getId().getDatasourceName(); Dataset dataset = MetadataManagerUtil.findExistingDataset(mdTxnCtx, dataverseName, datasetName); int numKeys = keys.size();
String outputType = ConstantExpressionUtil.getStringArgument(f, 5); MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider(); DataSourceId asid = new DataSourceId(dataverse, getTargetFeed); String policyName = (String) metadataProvider.getConfig().get(FeedActivityDetails.FEED_POLICY_NAME); FeedPolicyEntity policy = metadataProvider.findFeedPolicy(dataverse, policyName);
DataSourceId asid = ((IDataSource<DataSourceId>) scan.getDataSource()).getId(); Dataset dataset = mp.findDataset(asid.getDataverseName(), asid.getDatasourceName()); if (dataset == null) { throw new CompilationException(ErrorCode.UNKNOWN_DATASET_IN_DATAVERSE, scan.getSourceLocation(), asid.getDatasourceName(), asid.getDataverseName());
DataSourceId dataSourceId = new DataSourceId(dataset.getDataverseName(), dataset.getDatasetName()); unnestOp.setPhysicalOperator(new ExternalDataLookupPOperator(dataSourceId, dataset, recordType, primaryKeyVars, false, retainInput, retainNull));
String dataverseName = dataSourceIndex.getDataSource().getId().getDataverseName(); String datasetName = dataSourceIndex.getDataSource().getId().getDatasourceName();
datasetName, dataverseName); DataSourceId asid = new DataSourceId(dataverseName, datasetName); List<LogicalVariable> variables = new ArrayList<>(); if (dataset.getDatasetType() == DatasetType.INTERNAL) {
List<LogicalVariable> additionalNonFilteringFields) throws AlgebricksException { String datasetName = dataSource.getId().getDatasourceName(); Dataset dataset = MetadataManagerUtil.findExistingDataset(mdTxnCtx, dataSource.getId().getDataverseName(), datasetName); int numKeys = keys.size(); int numFilterFields = DatasetUtil.getFilterField(dataset) == null ? 0 : 1;
private DatasetDataSource validateDatasetInfo(MetadataProvider metadataProvider, String dataverseName, String datasetName, SourceLocation sourceLoc) throws AlgebricksException { Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName); if (dataset == null) { throw new CompilationException(ErrorCode.UNKNOWN_DATASET_IN_DATAVERSE, sourceLoc, datasetName, dataverseName); } if (dataset.getDatasetType() == DatasetType.EXTERNAL) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "Cannot write output to an external dataset."); } DataSourceId sourceId = new DataSourceId(dataverseName, datasetName); IAType itemType = metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName()); IAType metaItemType = metadataProvider.findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName()); INodeDomain domain = metadataProvider.findNodeDomain(dataset.getNodeGroupName()); return new DatasetDataSource(sourceId, dataset, itemType, metaItemType, DataSource.Type.INTERNAL_DATASET, dataset.getDatasetDetails(), domain); }
Dataset dataset = mp.findDataset(asid.getDataverseName(), asid.getDatasourceName()); if (dataset == null) { throw new CompilationException(ErrorCode.UNKNOWN_DATASET_IN_DATAVERSE, scan.getSourceLocation(), asid.getDatasourceName(), asid.getDataverseName());
MetadataProvider mp = (MetadataProvider) context.getMetadataProvider(); DataSourceId dataSourceId = new DataSourceId(jobGenParams.getDataverseName(), jobGenParams.getDatasetName()); Dataset dataset = mp.findDataset(jobGenParams.getDataverseName(), jobGenParams.getDatasetName());