String partition_values_to_partition_filter(final TableDto tableDto, final List<String> partVals) throws MetaException { if (partVals.size() > tableDto.getPartition_keys().size()) { throw new MetaException("Too many partition values for " + tableDto.getName()); final List<FieldDto> fields = tableDto.getFields(); final List<String> partitionFilters = Lists.newArrayListWithCapacity(fields.size()); for (int i = 0, partitionIdx = 0; i < fields.size(); i++) {
@Override public TableDto hiveToMetacatTable(final QualifiedName name, final Table table) { final TableDto dto = new TableDto(); dto.setSerde(toStorageDto(table.getSd(), table.getOwner())); dto.setAudit(new AuditDto()); dto.setName(name); if (table.isSetCreateTime()) { dto.getAudit().setCreatedDate(epochSecondsToDate(table.getCreateTime())); } dto.setMetadata(table.getParameters()); final List<FieldSchema> nonPartitionColumns = table.getSd().getCols(); final List<FieldSchema> partitionColumns = table.getPartitionKeys(); final List<FieldDto> allFields = Lists.newArrayListWithCapacity(nonPartitionColumns.size() + partitionColumns.size()); nonPartitionColumns.stream() .map(field -> this.hiveToMetacatField(field, false)) .forEachOrdered(allFields::add); partitionColumns.stream() .map(field -> this.hiveToMetacatField(field, true)) .forEachOrdered(allFields::add); dto.setFields(allFields); return dto; }
protected ConnectorTableMetadata fromTableDto(final TableDto table, final TypeManager typeManager) { return new ConnectorTableDetailMetadata( toSchemaTableName(table.getName()), columnsFromTable(table, typeManager), table.getSerde() == null ? null : table.getSerde().getOwner(), fromStorageDto(table.getSerde()), table.getMetadata(), fromAuditDto(table.getAudit()) ); }
@VisibleForTesting private boolean isTableInfoProvided(final TableDto tableDto, final TableDto oldTableDto) { boolean result = false; if ((tableDto.getFields() != null && !tableDto.getFields().isEmpty()) || isSerdeInfoProvided(tableDto, oldTableDto) || (tableDto.getMetadata() != null && !tableDto.getMetadata().isEmpty()) || tableDto.getAudit() != null) { result = true; } return result; }
private void updateEntitiesWithSameUri(final String metadataType, final TableDto dto, final String userName) { if (dto.isDataExternal()) { final List<String> ids = es.getTableIdsByUri(metadataType, dto.getDataUri()) .stream().filter(s -> !s.equals(dto.getName().toString())).collect(Collectors.toList()); if (!ids.isEmpty()) { log.info("ElasticSearch table updates({}) with same uri {} (Table:{})", ids.size(), dto.getDataUri(), dto.getName()); final ObjectNode node = metacatJsonLocator.emptyObjectNode(); node.set(ElasticSearchDoc.Field.DATA_METADATA, dto.getDataMetadata()); node.put(ElasticSearchDoc.Field.USER, userName); node.put(ElasticSearchDoc.Field.TIMESTAMP, java.time.Instant.now().toEpochMilli()); es.updates(ElasticSearchDoc.Type.table.name(), ids, node); } } }
/** * {@inheritDoc} */ @Override public TableDto copy(final TableDto tableDto, final QualifiedName targetName) { final QualifiedName databaseName = QualifiedName.ofDatabase(targetName.getCatalogName(), targetName.getDatabaseName()); if (!databaseService.exists(databaseName)) { final DatabaseDto databaseDto = new DatabaseDto(); databaseDto.setName(databaseName); databaseService.create(databaseName, databaseDto); } final TableDto targetTableDto = new TableDto(); targetTableDto.setName(targetName); targetTableDto.setFields(tableDto.getFields()); targetTableDto.setPartition_keys(tableDto.getPartition_keys()); final StorageDto storageDto = tableDto.getSerde(); if (storageDto != null) { final StorageDto targetStorageDto = new StorageDto(); targetStorageDto.setInputFormat(storageDto.getInputFormat()); targetStorageDto.setOwner(storageDto.getOwner()); targetStorageDto.setOutputFormat(storageDto.getOutputFormat()); targetStorageDto.setParameters(storageDto.getParameters()); targetStorageDto.setUri(storageDto.getUri()); targetStorageDto.setSerializationLib(storageDto.getSerializationLib()); targetTableDto.setSerde(targetStorageDto); } create(targetName, targetTableDto); return targetTableDto; }
@Override public TableDto toTableDto(QualifiedName name, String type, TableMetadata ptm) { if ( name == null && type == null && ptm == null ) { return null; } TableDto tableDto = new TableDto(); if ( name != null ) { tableDto.setName( name ); } if ( ptm != null ) { tableDto.setMetadata( toMetadata( ptm.getMetadata() ) ); tableDto.setPartition_keys( toPartitionKeys( ptm.getColumns() ) ); tableDto.setSerde( toStorageDto( ptm.getMetadata() ) ); tableDto.setAudit( toAuditDto( ptm.getMetadata() ) ); tableDto.setFields( toFieldDtos( ptm.getColumns() ) ); } return tableDto; }
final StorageDescriptor sd = result.getSd(); if (tableDto != null) { if (sd.getSerdeInfo() != null && tableDto.getSerde() != null && Strings.isNullOrEmpty( sd.getSerdeInfo().getSerializationLib())) { sd.getSerdeInfo().setSerializationLib(tableDto.getSerde().getSerializationLib()); final List<FieldDto> fields = tableDto.getFields(); if (fields == null) { sd.setCols(Collections.emptyList());
/** * {@inheritDoc} */ @Override public PartitionDto hiveToMetacatPartition(final TableDto tableDto, final Partition partition) { final QualifiedName tableName = tableDto.getName(); final QualifiedName partitionName = QualifiedName.ofPartition(tableName.getCatalogName(), tableName.getDatabaseName(), tableName.getTableName(), getNameFromPartVals(tableDto, partition.getValues())); final PartitionDto result = new PartitionDto(); String owner = ""; if (tableDto.getSerde() != null) { owner = tableDto.getSerde().getOwner(); } //not setting Serde to view if (null == tableDto.getView() || Strings.isNullOrEmpty(tableDto.getView().getViewOriginalText())) { result.setSerde(toStorageDto(partition.getSd(), owner)); } result.setMetadata(partition.getParameters()); final AuditDto auditDto = new AuditDto(); auditDto.setCreatedDate(epochSecondsToDate(partition.getCreateTime())); auditDto.setLastModifiedDate(epochSecondsToDate(partition.getLastAccessTime())); result.setAudit(auditDto); result.setName(partitionName); return result; }
private PartitionDto getPartitionDtoByName(final TableDto tableDto, final String partName) throws TException { final GetPartitionsRequestDto dto = new GetPartitionsRequestDto(null, ImmutableList.of(partName), true, false); final List<PartitionDto> partitionDtos = partV1.getPartitionsForRequest( catalogName, tableDto.getName().getDatabaseName(), tableDto.getName().getTableName(), null, null, null, null, false, dto); if (partitionDtos == null || partitionDtos.isEmpty()) { throw new NoSuchObjectException("Partition (" + partName + ") not found on " + tableDto.getName()); } else if (partitionDtos.size() != 1) { // I don't think this is even possible throw new NoSuchObjectException("Partition (" + partName + ") matched extra on " + tableDto.getName()); } return partitionDtos.get(0); }
private List<Partition> addPartitionsCore(final String dbName, final String tblName, final List<Partition> parts, final boolean ifNotExists) throws TException { log.debug("Ignoring {} since metacat save partitions will do an update if it already exists", ifNotExists); final TableDto tableDto = v1.getTable(catalogName, dbName, tblName, true, false, false); final List<String> partitionKeys = tableDto.getPartition_keys(); if (partitionKeys == null || partitionKeys.isEmpty()) { throw new MetaException("Unable to add partition to unpartitioned table: " + tableDto.getName()); } final PartitionsSaveRequestDto partitionsSaveRequestDto = new PartitionsSaveRequestDto(); final List<PartitionDto> converted = Lists.newArrayListWithCapacity(parts.size()); for (Partition partition : parts) { converted.add(hiveConverters.hiveToMetacatPartition(tableDto, partition)); } partitionsSaveRequestDto.setPartitions(converted); partV1.savePartitions(catalogName, dbName, tblName, partitionsSaveRequestDto); return parts; }
final TableDto t = new TableDto(); t.setName(name); return t; }); tagService.delete(name, false); } else { if (config.canSoftDeleteDataMetadata() && tableDto.isDataExternal()) { userMetadataService.softDeleteDataMetadata(metacatRequestContext.getUserName(), Lists.newArrayList(tableDto.getDataUri()));
final TableDto dto = new TableDto(); dto.setName(name); dto.setDefinitionMetadata(partitionsSaveRequestDto.getDefinitionMetadata()); dto.setDataMetadata(partitionsSaveRequestDto.getDataMetadata()); this.v1.updateTable(catalogName, databaseName, tableName, dto); final TableDto dto = new TableDto(); dto.setName(name); dto.setDefinitionMetadata(partitionsSaveRequestDto.getDefinitionMetadata()); dto.setDataMetadata(partitionsSaveRequestDto.getDataMetadata()); this.v1.updateMView(catalogName, databaseName, tableName, viewName, dto);
/** * {@inheritDoc} */ @Override public String getNameFromPartVals(final TableDto tableDto, final List<String> partVals) { final List<String> partitionKeys = tableDto.getPartition_keys(); if (partitionKeys.size() != partVals.size()) { throw new IllegalArgumentException("Not the same number of partition columns and partition values"); } return FileUtils.makePartName(partitionKeys, partVals, ""); }
/** * Save all tables to index it in elastic search. * * @param databaseName database name * @param dtos table dtos * @return future */ private ListenableFuture<Void> indexTableDtos(final QualifiedName databaseName, final List<Optional<TableDto>> dtos) { return esService.submit(() -> { final List<ElasticSearchDoc> docs = dtos.stream().filter(dto -> dto != null && dto.isPresent()).map( tableDtoOptional -> { final TableDto dto = tableDtoOptional.get(); final String userName = dto.getAudit() != null ? dto.getAudit().getCreatedBy() : "admin"; return new ElasticSearchDoc(dto.getName().toString(), dto, userName, false, refreshMarkerText); }).collect(Collectors.toList()); log.info("Saving tables for database: {}", databaseName); elasticSearchUtil.save(ElasticSearchDoc.Type.table.name(), docs); return null; }); }
final TableDto dto = new TableDto(); dto.setName(name); this.eventBus.post(new MetacatDeleteMViewPostEvent(name, metacatRequestContext, this, dto)); } else if (name.isTableDefinition()) { final TableDto dto = new TableDto(); dto.setName(name); this.eventBus.post(new MetacatDeleteTablePostEvent(name, metacatRequestContext, this, dto, false));
@Override public PartitionDto hiveToMetacatPartition(final TableDto tableDto, final Partition partition) { final QualifiedName tableName = tableDto.getName(); final QualifiedName partitionName = QualifiedName.ofPartition(tableName.getCatalogName(), tableName.getDatabaseName(), tableName.getTableName(), getNameFromPartVals(tableDto, partition.getValues())); final PartitionDto result = new PartitionDto(); String owner = ""; if (tableDto.getSerde() != null) { owner = tableDto.getSerde().getOwner(); } result.setSerde(toStorageDto(partition.getSd(), owner)); result.setMetadata(partition.getParameters()); final AuditDto auditDto = new AuditDto(); auditDto.setCreatedDate(epochSecondsToDate(partition.getCreateTime())); auditDto.setLastModifiedDate(epochSecondsToDate(partition.getLastAccessTime())); result.setAudit(auditDto); result.setName(partitionName); return result; }