private String getTableName() { return key.getName(); }
@Override public boolean datasetExists(NamespaceKey key) { return key.size() == 2 && TABLES.contains(key.getName().toLowerCase()); }
@Override public boolean datasetExists(NamespaceKey key) { return key.size() == 2 && TABLES.contains(key.getName().toLowerCase()); }
public static TableName getTableName(NamespaceKey key) { switch(key.size()) { case 2: return TableName.valueOf(key.getName()); case 3: final List<String> paths = key.getPathComponents(); return TableName.valueOf(paths.get(1), paths.get(2)); default: throw new IllegalStateException("Unexpected key length: " + key); } } }
@Override public SourceTableDefinition getDataset(NamespaceKey datasetPath, DatasetConfig oldDataset, DatasetRetrievalOptions ignored) throws Exception { if(datasetPath.size() != 2) { return null; } InfoSchemaTable table = TABLE_MAP.get(datasetPath.getName().toLowerCase()); if(table != null) { return table.asTableDefinition(oldDataset); } return null; }
@Override public SourceTableDefinition getDataset(NamespaceKey datasetPath, DatasetConfig oldDataset, DatasetRetrievalOptions ignored) throws Exception { if(datasetPath.size() != 2) { return null; } SystemTable table = TABLE_MAP.get(datasetPath.getName().toLowerCase()); if(table != null) { return table.asTableDefinition(oldDataset); } return null; }
public InfoSchemaScanPrel( RelOptCluster cluster, RelTraitSet traitSet, RelOptTable table, TableMetadata dataset, SearchQuery query, List<SchemaPath> projectedColumns, double observedRowcountAdjustment ) { super(cluster, traitSet, table, dataset.getStoragePluginId(), dataset, projectedColumns, observedRowcountAdjustment); this.pluginId = dataset.getStoragePluginId(); this.table = Preconditions.checkNotNull(InfoSchemaStoragePlugin.TABLE_MAP.get(dataset.getName().getName().toLowerCase()), "Unexpected system table."); this.query = query; }
@Override public boolean containerExists(NamespaceKey key) { if(key.size() != 2) { return false; } if (key.getName().equals(HBASE_SYSTEM_NAMESPACE)) { // DX-10110: do not allow access to the system namespace of HBase itself. Causes confusion when multiple 'use hbase' statements are processed return false; } try(Admin admin = connection.getConnection().getAdmin()) { NamespaceDescriptor descriptor = admin.getNamespaceDescriptor(key.getName()); return descriptor != null; } catch (IOException e) { logger.warn("Failure while checking for HBase Namespace {}.", key, e); } return false; }
@VisibleForTesting public SystemScanPrel( RelOptCluster cluster, RelTraitSet traitSet, RelOptTable table, TableMetadata dataset, List<SchemaPath> projectedColumns, double observedRowcountAdjustment, RelDataType rowType ) { super(cluster, traitSet, table, dataset.getStoragePluginId(), dataset, projectedColumns, observedRowcountAdjustment); this.systemTable = Preconditions.checkNotNull(SystemStoragePlugin.TABLE_MAP.get(dataset.getName().getName().toLowerCase()), "Unexpected system table."); this.executorCount = PrelUtil.getPlannerSettings(cluster).getExecutorCount(); this.rowType = rowType; this.pluginId = dataset.getStoragePluginId(); }
private static FileFormat asFormat(NamespaceKey key, Path path, boolean isFolder) { String name = path.getName(); if(name.endsWith(".zip")) { name = name.substring(0, name.length() - 4); } if(name.endsWith(".gz")) { name = name.substring(0, name.length() - 3); } final FileConfig config = new FileConfig() .setCtime(System.currentTimeMillis()) .setFullPathList(key.getPathComponents()) .setName(key.getName()) .setType(FileFormat.getFileFormatType(Collections.singletonList(FilenameUtils.getExtension(name)))) .setTag(null); return isFolder ? FileFormat.getForFolder(config) : FileFormat.getForFile(config); }
void shallowSave(SourceTableDefinition accessor) throws NamespaceException{ NamespaceKey key = accessor.getName(); DatasetConfig shallow = new DatasetConfig(); shallow.setId(new EntityId().setId(UUID.randomUUID().toString())); shallow.setCreatedAt(System.currentTimeMillis()); shallow.setName(key.getName()); shallow.setFullPathList(key.getPathComponents()); shallow.setType(accessor.getType()); shallow.setSchemaVersion(DatasetHelper.CURRENT_VERSION); systemUserNamespaceService.addOrUpdateDataset(key, shallow); }
@Override public void deleteDataset(final NamespaceKey datasetPath, String version) throws NamespaceException { NameSpaceContainer container = deleteEntity(datasetPath, DATASET, version, true); if (container.getDataset().getType() == PHYSICAL_DATASET_SOURCE_FOLDER) { // create a folder so that any existing datasets under the folder are now visible addOrUpdateFolder(datasetPath, new FolderConfig() .setFullPathList(datasetPath.getPathComponents()) .setName(datasetPath.getName()) ); } }
@Override public DatasetConfig getDataset() throws Exception { final DatasetConfig dataset; if(oldDataset == null) { dataset = new DatasetConfig() .setFullPathList(key.getPathComponents()) .setId(new EntityId(UUID.randomUUID().toString())) .setType(DatasetType.PHYSICAL_DATASET); } else { dataset = oldDataset; } return dataset .setName(key.getName()) .setReadDefinition(new ReadDefinition() .setScanStats(new ScanStats().setRecordCount(100l) .setScanFactor(ScanCostFactor.OTHER.getFactor()))) .setOwner(SystemUser.SYSTEM_USERNAME) .setPhysicalDataset(new PhysicalDataset()) .setRecordSchema(getSchema().toByteString()) .setSchemaVersion(DatasetHelper.CURRENT_VERSION); }
@Override public DatasetConfig getDataset() { final DatasetConfig dataset; if(oldDataset == null) { dataset = new DatasetConfig() .setFullPathList(key.getPathComponents()) .setId(new EntityId(UUID.randomUUID().toString())) .setType(DatasetType.PHYSICAL_DATASET); } else { dataset = oldDataset; } return dataset .setName(key.getName()) .setReadDefinition(new ReadDefinition() .setScanStats(new ScanStats().setRecordCount(100l) .setScanFactor(ScanCostFactor.OTHER.getFactor()))) .setOwner(SystemUser.SYSTEM_USERNAME) .setPhysicalDataset(new PhysicalDataset()) .setRecordSchema(getSchema().toByteString()) .setSchemaVersion(DatasetHelper.CURRENT_VERSION); }
public static void addFolder(NamespaceService ns, String name) throws Exception { final FolderConfig folder = new FolderConfig(); final NamespaceKey folderPath = new NamespaceKey(PathUtils.parseFullPath(name)); folder.setName(folderPath.getName()); folder.setFullPathList(folderPath.getPathComponents()); ns.addOrUpdateFolder(folderPath, folder); }
protected DatasetConfig doRenameDataset(NamespaceKey oldDatasetPath, NamespaceKey newDatasetPath, List<NameSpaceContainer> oldDatasetParentEntitiesOnPath, List<NameSpaceContainer> newDatasetParentEntitiesOnPath) throws NamespaceException { final String newDatasetName = newDatasetPath.getName(); final NamespaceInternalKey oldKey = new NamespaceInternalKey(oldDatasetPath, keyNormalization); final DatasetConfig datasetConfig = getEntity(oldDatasetPath, DATASET).getDataset(); if (isPhysicalDataset(datasetConfig.getType())) { throw UserException.validationError() .message("Failed to rename %s to %s. Renames on physical datasets are not allowed.", oldDatasetPath, newDatasetPath) .build(logger); } if (datasetConfig.getType() == DatasetType.PHYSICAL_DATASET_HOME_FILE && !newDatasetPath.getPathComponents().get(0).startsWith("@")) { throw UserException.validationError() .message("Failed to rename %s to %s. You cannot move a uploaded file in your home directory to a space.", oldDatasetPath, newDatasetPath) .build(logger); } datasetConfig.setName(newDatasetName); datasetConfig.setFullPathList(newDatasetPath.getPathComponents()); datasetConfig.setCreatedAt(System.currentTimeMillis()); NamespaceEntity newValue = NamespaceEntity.toEntity(DATASET, newDatasetPath, datasetConfig, keyNormalization); // in case of upgrade we may have a version here from previous versions, so clear out datasetConfig.setVersion(null); datasetConfig.setTag(null); namespace.put(newValue.getPathKey().getKey(), newValue.getContainer()); namespace.delete(oldKey.getKey()); return datasetConfig; }
public void addPhysicalDataset(final DatasetPath path, final DatasetType type) throws Exception { NamespaceKey datasetPath = path.toNamespaceKey(); final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setName(datasetPath.getName()); datasetConfig.setType(type); datasetConfig.setPhysicalDataset(new PhysicalDataset()); getNamespaceService().tryCreatePhysicalDataset(datasetPath, datasetConfig); }
public void addPhysicalDataset(final DatasetPath path, final DatasetType type) throws Exception { NamespaceKey datasetPath = path.toNamespaceKey(); final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setName(datasetPath.getName()); datasetConfig.setType(type); datasetConfig.setPhysicalDataset(new PhysicalDataset()); getNamespaceService().tryCreatePhysicalDataset(datasetPath, datasetConfig); }
public static void addPhysicalDS(NamespaceService ns, String filePath, DatasetType type, byte[] datasetSchema) throws Exception { NamespaceKey datasetPath = new NamespaceKey(PathUtils.parseFullPath(filePath)); final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setName(datasetPath.getName()); datasetConfig.setType(type); final PhysicalDataset physicalDataset = new PhysicalDataset(); if (datasetSchema != null) { datasetConfig.setRecordSchema(io.protostuff.ByteString.copyFrom(datasetSchema)); } datasetConfig.setSchemaVersion(DatasetHelper.CURRENT_VERSION); datasetConfig.setPhysicalDataset(physicalDataset); ns.tryCreatePhysicalDataset(datasetPath, datasetConfig); }
public static void addDS(NamespaceService ns, String name) throws Exception { final NamespaceKey dsPath = new NamespaceKey(PathUtils.parseFullPath(name)); final DatasetConfig ds = new DatasetConfig(); final VirtualDataset vds = new VirtualDataset(); vds.setVersion(DatasetVersion.newVersion()); ds.setType(DatasetType.VIRTUAL_DATASET); ds.setVirtualDataset(vds); ds.setFullPathList(dsPath.getPathComponents()); ds.setName(dsPath.getName()); ns.addOrUpdateDataset(dsPath, ds); }