public StoragePluginId getStoragePluginId() { return dataset.getStoragePluginId(); }
public HBaseScanPrel( RelOptCluster cluster, RelTraitSet traitSet, RelOptTable table, TableMetadata dataset, List<SchemaPath> projectedColumns, double observedRowcountAdjustment, byte[] startRow, byte[] stopRow, byte[] filter) { super(cluster, traitSet, table, dataset.getStoragePluginId(), dataset, projectedColumns, observedRowcountAdjustment); this.filter = filter; this.startRow = startRow; this.stopRow = stopRow; }
public InfoSchemaScanPrel( RelOptCluster cluster, RelTraitSet traitSet, RelOptTable table, TableMetadata dataset, SearchQuery query, List<SchemaPath> projectedColumns, double observedRowcountAdjustment ) { super(cluster, traitSet, table, dataset.getStoragePluginId(), dataset, projectedColumns, observedRowcountAdjustment); this.pluginId = dataset.getStoragePluginId(); this.table = Preconditions.checkNotNull(InfoSchemaStoragePlugin.TABLE_MAP.get(dataset.getName().getName().toLowerCase()), "Unexpected system table."); this.query = query; }
@VisibleForTesting public SystemScanPrel( RelOptCluster cluster, RelTraitSet traitSet, RelOptTable table, TableMetadata dataset, List<SchemaPath> projectedColumns, double observedRowcountAdjustment, RelDataType rowType ) { super(cluster, traitSet, table, dataset.getStoragePluginId(), dataset, projectedColumns, observedRowcountAdjustment); this.systemTable = Preconditions.checkNotNull(SystemStoragePlugin.TABLE_MAP.get(dataset.getName().getName().toLowerCase()), "Unexpected system table."); this.executorCount = PrelUtil.getPlannerSettings(cluster).getExecutorCount(); this.rowType = rowType; this.pluginId = dataset.getStoragePluginId(); }
public AbstractFileGroupScan(TableMetadata dataset, List<SchemaPath> columns) { super(dataset, columns); this.affinity = dataset.getStoragePluginId().getCapabilities().getCapability(SourceCapabilities.REQUIRES_HARD_AFFINITY) ? DistributionAffinity.HARD : DistributionAffinity.SOFT; }
@Override public DistributionAffinity getDistributionAffinity() { return this.tableMetadata.getStoragePluginId().getCapabilities().getCapability(SourceCapabilities.REQUIRES_HARD_AFFINITY) ? DistributionAffinity.HARD : DistributionAffinity.SOFT; }
public ElasticIntermediateScanPrel( RelOptCluster cluster, RelTraitSet traitSet, RelOptTable table, TableMetadata dataset, List<SchemaPath> projectedColumns, double observedRowcountAdjustment) { super(cluster, traits(cluster, table.getRowCount(), dataset.getSplitCount(), traitSet), table, dataset.getStoragePluginId(), dataset, projectedColumns, observedRowcountAdjustment); }
@Override public SubScan getSpecificScan(List<SplitWork> work) throws ExecutionSetupException { List<HBaseSubScanSpec> splitWork = FluentIterable.from(work).transform(new Function<SplitWork, HBaseSubScanSpec>(){ @Override public HBaseSubScanSpec apply(SplitWork input) { return toSubScan(input.getSplit()); }}).toList(); return new HBaseSubScan( getDataset().getStoragePluginId(), getUserName(), splitWork, getColumns(), getSchema(), Iterables.getOnlyElement(getReferencedTables()) ); }
@Override public ScanCrel toRel(ToRelContext toRelContext, RelOptTable relOptTable) { return new ScanCrel(toRelContext.getCluster(), toRelContext.getCluster().traitSetOf(Convention.NONE), dataset.getStoragePluginId(), dataset, null, 1.0d, true); }
@Override public SubScan getSpecificScan(List<SplitWork> work) throws ExecutionSetupException { List<DatasetSplit> splits = new ArrayList<>(work.size()); BatchSchema schema = getDataset().getSchema(); for(SplitWork split : work){ splits.add(split.getSplit()); } boolean storageImpersonationEnabled = dataset.getStoragePluginId().getCapabilities().getCapability(SourceCapabilities.STORAGE_IMPERSONATION); String userName = storageImpersonationEnabled ? getUserName() : ImpersonationUtil.getProcessUserName(); final ReadDefinition readDefinition = dataset.getReadDefinition(); return new HiveSubScan(splits, userName, schema, dataset.getName().getPathComponents(), filter, dataset.getStoragePluginId(), columns, readDefinition.getPartitionColumnsList()); }
@Override public SubScan getSpecificScan(List<SplitWork> work) throws ExecutionSetupException { List<DatasetSplit> splitWork = FluentIterable.from(work).transform(new Function<SplitWork, DatasetSplit>(){ @Override public DatasetSplit apply(SplitWork input) { return input.getSplit(); }}).toList(); return new ElasticsearchSubScan( getUserName(), getDataset().getStoragePluginId(), spec, splitWork, getColumns(), Iterables.getOnlyElement(getReferencedTables()), getSchema(), getDataset().getReadDefinition().getExtendedProperty() ); }
private Prel newScan(RelDataType rowType, double rowCount, double splitRatio) throws Exception { TableMetadata metadata = Mockito.mock(TableMetadata.class); when(metadata.getName()).thenReturn(new NamespaceKey(ImmutableList.of("sys", "version"))); when(metadata.getSchema()).thenReturn(SystemTable.VERSION.getSchema()); when(metadata.getSplitRatio()).thenReturn(splitRatio); StoragePluginId pluginId = new StoragePluginId(new SourceConfig().setConfig(new SystemPluginConf().toBytesString()), new SystemPluginConf(), SourceCapabilities.NONE); when(metadata.getStoragePluginId()).thenReturn(pluginId); List<SchemaPath> columns = FluentIterable.from(SystemTable.VERSION.getSchema()).transform(input -> SchemaPath.getSimplePath(input.getName())).toList(); final RelOptTable relOptTable = Mockito.mock(RelOptTable.class); when(relOptTable.getRowCount()).thenReturn(rowCount); return new SystemScanPrel(cluster, traits, relOptTable, metadata, columns, 1.0d, rowType); }
private Prel newHardScan(RelDataType rowType) { TableMetadata metadata = Mockito.mock(TableMetadata.class); when(metadata.getName()).thenReturn(new NamespaceKey(ImmutableList.of("sys", "memory"))); when(metadata.getSchema()).thenReturn(SystemTable.MEMORY.getSchema()); StoragePluginId pluginId = new StoragePluginId(new SourceConfig().setConfig(new SystemPluginConf().toBytesString()), new SystemPluginConf(), SourceCapabilities.NONE); when(metadata.getStoragePluginId()).thenReturn(pluginId); List<SchemaPath> columns = FluentIterable.from(SystemTable.MEMORY.getSchema()).transform(new Function<Field, SchemaPath>(){ @Override public SchemaPath apply(Field input) { return SchemaPath.getSimplePath(input.getName()); }}).toList(); return new SystemScanPrel(cluster, traits, Mockito.mock(RelOptTable.class), metadata, columns, 1.0d, rowType); }
private Prel newSoftScan(RelDataType rowType) { TableMetadata metadata = Mockito.mock(TableMetadata.class); when(metadata.getName()).thenReturn(new NamespaceKey(ImmutableList.of("sys", "version"))); when(metadata.getSchema()).thenReturn(SystemTable.VERSION.getSchema()); StoragePluginId pluginId = new StoragePluginId(new SourceConfig().setConfig(new SystemPluginConf().toBytesString()), new SystemPluginConf(), SourceCapabilities.NONE); when(metadata.getStoragePluginId()).thenReturn(pluginId); List<SchemaPath> columns = FluentIterable.from(SystemTable.VERSION.getSchema()).transform(new Function<Field, SchemaPath>(){ @Override public SchemaPath apply(Field input) { return SchemaPath.getSimplePath(input.getName()); }}).toList(); return new SystemScanPrel(cluster, traits, Mockito.mock(RelOptTable.class), metadata, columns, 1.0d, rowType); }
@Override public SubScan getSpecificScan(List<SplitWork> work) throws ExecutionSetupException { final BatchSchema schema = cachedRelDataType == null ? getDataset().getSchema(): BatchSchema.fromCalciteRowType(cachedRelDataType); // Create an abridged version of the splits to save network bytes. List<DatasetSplit> splits = work.stream().map( workSplit -> ProtostuffUtil.copy(workSplit.getSplit()) .setExtendedProperty(convertToScanXAttr(workSplit.getSplit().getExtendedProperty())) ).collect(Collectors.toList()); return new ParquetSubScan(dataset.getFormatSettings(), splits, getUserName(), schema, getDataset().getName().getPathComponents(), filter == null ? null : filter.getConditions(), dataset.getStoragePluginId(), columns, dataset.getReadDefinition().getPartitionColumnsList(), globalDictionaryEncodedColumns, dataset.getReadDefinition().getExtendedProperty()); }
@Override public SubScan getSpecificScan(List<SplitWork> work) throws ExecutionSetupException { final List<DatasetSplit> splits = new ArrayList<>(work.size()); final BatchSchema schema = getDataset().getSchema(); for(SplitWork split : work){ splits.add(split.getSplit()); } return new EasySubScan( getDataset().getFormatSettings(), splits, getUserName(), schema, getDataset().getName().getPathComponents(), dataset.getStoragePluginId(), columns, getDataset().getReadDefinition().getPartitionColumnsList(), getDataset().getReadDefinition().getExtendedProperty()); }
@Override public RelNode convert(RelNode rel) { FilesystemScanDrel drel = (FilesystemScanDrel) rel; // TODO: this singleton check should be removed once DX-7175 is fixed boolean singleton = !drel.getTableMetadata().getStoragePluginId().getCapabilities().getCapability(SourceCapabilities.REQUIRES_HARD_AFFINITY) && drel.getTableMetadata().getSplitCount() == 1; return new EasyScanPrel(drel.getCluster(), drel.getTraitSet().plus(Prel.PHYSICAL).plus(singleton ? DistributionTrait.SINGLETON : DistributionTrait.ANY), drel.getTable(), drel.getPluginId(), drel.getTableMetadata(), drel.getProjectedColumns(), drel.getObservedRowcountAdjustment()); }