private List<HivePartition> getOrComputePartitions(HiveTableLayoutHandle layoutHandle, ConnectorSession session, ConnectorTableHandle tableHandle) { if (layoutHandle.getPartitions().isPresent()) { return layoutHandle.getPartitions().get(); } else { TupleDomain<ColumnHandle> promisedPredicate = layoutHandle.getPromisedPredicate(); Predicate<Map<ColumnHandle, NullableValue>> predicate = convertToPredicate(promisedPredicate); List<ConnectorTableLayoutResult> tableLayoutResults = getTableLayouts(session, tableHandle, new Constraint<>(promisedPredicate, predicate), Optional.empty()); return ((HiveTableLayoutHandle) Iterables.getOnlyElement(tableLayoutResults).getTableLayout().getHandle()).getPartitions().get(); } }
List<HiveColumnHandle> columnHandles = getColumnHandles(tableMetadata, ImmutableSet.copyOf(partitionedBy), typeTranslator); HiveStorageFormat hiveStorageFormat = getHiveStorageFormat(tableMetadata.getProperties()); Map<String, String> tableProperties = getEmptyTableProperties(tableMetadata, !partitionedBy.isEmpty(), new HdfsContext(session, schemaName, tableName)); .map(column -> new Column(column.getName(), column.getHiveType(), column.getComment())) .collect(toList()); checkPartitionTypesSupported(partitionColumns); targetPath = getExternalPath(new HdfsContext(session, schemaName, tableName), externalLocation); Table table = buildTableObject( session.getQueryId(), schemaName, external, prestoVersion); PrincipalPrivileges principalPrivileges = buildInitialPrivilegeSet(table.getOwner()); HiveBasicStatistics basicStatistics = table.getPartitionColumns().isEmpty() ? createZeroStatistics() : createEmptyStatistics(); metastore.createTable(
@SuppressWarnings("TryWithIdenticalCatches") @Override public Map<SchemaTableName, List<ColumnMetadata>> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix) { requireNonNull(prefix, "prefix is null"); ImmutableMap.Builder<SchemaTableName, List<ColumnMetadata>> columns = ImmutableMap.builder(); for (SchemaTableName tableName : listTables(session, prefix)) { try { columns.put(tableName, getTableMetadata(tableName).getColumns()); } catch (HiveViewNotSupportedException e) { // view is not supported } catch (TableNotFoundException e) { // table disappeared during listing operation } } return columns.build(); }
@Override public TableStatistics getTableStatistics(ConnectorSession session, ConnectorTableHandle tableHandle, Constraint<ColumnHandle> constraint) { if (!isStatisticsEnabled(session)) { return TableStatistics.empty(); } Map<String, ColumnHandle> columns = getColumnHandles(session, tableHandle) .entrySet().stream() .filter(entry -> !((HiveColumnHandle) entry.getValue()).isHidden()) .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); Map<String, Type> columnTypes = columns.entrySet().stream() .collect(toImmutableMap(Map.Entry::getKey, entry -> getColumnMetadata(session, tableHandle, entry.getValue()).getType())); List<HivePartition> partitions = getPartitionsAsList(tableHandle, constraint); return hiveStatisticsProvider.getTableStatistics(session, ((HiveTableHandle) tableHandle).getSchemaTableName(), columns, columnTypes, partitions); }
.build(); ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata(tableName, columns, properties, tableOwner); HiveOutputTableHandle outputHandle = metadata.beginCreateTable(SESSION, tableMetadata, Optional.empty()); metadata.finishCreateTable(SESSION, outputHandle, fragments); List<ColumnHandle> columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(SESSION, tableHandle).values()); tableMetadata = metadata.getTableMetadata(SESSION, getTableHandle(metadata, tableName)); assertEquals(tableMetadata.getOwner(), tableOwner); assertEquals(tableMetadata.getColumns(), columns); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(SESSION, tableHandle, new Constraint<>(TupleDomain.all(), bindings -> true), Optional.empty()); HiveTableLayoutHandle layoutHandle = (HiveTableLayoutHandle) getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); assertEquals(layoutHandle.getPartitions().get().size(), 1);
@Override public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain<Integer> constraint) { TupleDomain<ColumnHandle> targetTupleDomain = constraint.transform(fieldIdToColumnHandle::get); Predicate<Map<ColumnHandle, NullableValue>> targetPredicate = convertToPredicate(targetTupleDomain); Constraint<ColumnHandle> targetConstraint = new Constraint<>(targetTupleDomain, targetPredicate); Iterable<List<Object>> records = () -> stream(partitionManager.getPartitions(metastore, sourceTableHandle, targetConstraint).getPartitions()) .map(hivePartition -> (List<Object>) IntStream.range(0, partitionColumns.size()) .mapToObj(fieldIdToColumnHandle::get) .map(columnHandle -> hivePartition.getKeys().get(columnHandle).getValue()) .collect(toList())) .iterator(); return new InMemoryRecordSet(partitionColumnTypes, records).cursor(); } };
@Override public HiveOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional<ConnectorNewTableLayout> layout) verifyJvmTimeZone(); String tableName = schemaTableName.getTableName(); Map<String, String> tableProperties = getEmptyTableProperties(tableMetadata, !partitionedBy.isEmpty(), new HdfsContext(session, schemaName, tableName)); List<HiveColumnHandle> columnHandles = getColumnHandles(tableMetadata, ImmutableSet.copyOf(partitionedBy), typeTranslator); HiveStorageFormat partitionStorageFormat = isRespectTableFormat(session) ? tableStorageFormat : getHiveStorageFormat(session); .map(column -> new Column(column.getName(), column.getHiveType(), column.getComment())) .collect(toList()); checkPartitionTypesSupported(partitionColumns);
@Override public HiveOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional<ConnectorNewTableLayout> layout) { checkNoRollback(); verifyJvmTimeZone(); checkArgument(!isNullOrEmpty(tableMetadata.getOwner()), "Table owner is null or empty"); HiveStorageFormat hiveStorageFormat = getHiveStorageFormat(tableMetadata.getProperties()); List<String> partitionedBy = getPartitionedBy(tableMetadata.getProperties()); Map<String, String> additionalTableParameters = tableParameterCodec.encode(tableMetadata.getProperties()); // get the root directory for the database SchemaTableName schemaTableName = tableMetadata.getTable(); String schemaName = schemaTableName.getSchemaName(); String tableName = schemaTableName.getTableName(); List<HiveColumnHandle> columnHandles = getColumnHandles(connectorId, tableMetadata, ImmutableSet.copyOf(partitionedBy)); HiveOutputTableHandle result = new HiveOutputTableHandle( connectorId, schemaName, tableName, columnHandles, session.getQueryId(), locationService.forNewTable(session.getQueryId(), schemaName, tableName), hiveStorageFormat, partitionedBy, tableMetadata.getOwner(), additionalTableParameters); setRollback(() -> rollbackCreateTable(result)); return result; }
@Override public HiveInsertTableHandle beginInsert(ConnectorSession session, ConnectorTableHandle tableHandle) checkNoRollback(); verifyJvmTimeZone(); hiveStorageFormat = extractHiveStorageFormat(table.get()); hiveStorageFormat); setRollback(() -> rollbackInsert(result)); return result;
@Override public void rollback() { checkState(!closed); closed = true; HiveMetadata metadata = (HiveMetadata) transactionManager.remove(transactionHandle); checkArgument(metadata != null, "no such transaction: %s", transactionHandle); metadata.rollback(); }
@Test public void testGetRecordsS3() throws Exception { HiveTransactionHandle transaction = new HiveTransactionHandle(); HiveMetadata metadata = metadataFactory.create(); ConnectorTableHandle table = getTableHandle(metadata, tableS3); List<ColumnHandle> columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(SESSION, table).values()); Map<String, Integer> columnIndex = indexColumns(columnHandles); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(SESSION, table, new Constraint<>(TupleDomain.all(), bindings -> true), Optional.empty()); HiveTableLayoutHandle layoutHandle = (HiveTableLayoutHandle) getOnlyElement(tableLayoutResults).getTableLayout().getHandle(); assertEquals(layoutHandle.getPartitions().get().size(), 1); ConnectorSplitSource splitSource = splitManager.getSplits(transaction, SESSION, layoutHandle); long sum = 0; for (ConnectorSplit split : getAllSplits(splitSource)) { try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction, SESSION, split, columnHandles)) { MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, getTypes(columnHandles)); for (MaterializedRow row : result) { sum += (Long) row.getField(columnIndex.get("t_bigint")); } } } assertEquals(sum, 78300); }
Function<HiveColumnHandle, ColumnMetadata> metadataGetter = columnMetadataGetter(table.get(), typeManager); ImmutableList.Builder<ColumnMetadata> columns = ImmutableList.builder(); for (HiveColumnHandle columnHandle : hiveColumnHandles(table.get())) { HiveStorageFormat format = extractHiveStorageFormat(table.get()); properties.put(STORAGE_FORMAT_PROPERTY, format);
@Singleton @Provides public Function<HiveTransactionHandle, SemiTransactionalHiveMetastore> createMetastoreGetter(HiveTransactionManager transactionManager) { return transactionHandle -> ((HiveMetadata) transactionManager.get(transactionHandle)).getMetastore(); } }
@Override public HiveInsertTableHandle beginInsert(ConnectorSession session, ConnectorTableHandle tableHandle) verifyJvmTimeZone(); .collect(toList()); HiveStorageFormat tableStorageFormat = extractHiveStorageFormat(table.get()); LocationHandle locationHandle = locationService.forExistingTable(metastore, session, table.get()); HiveInsertTableHandle result = new HiveInsertTableHandle(
List<HivePartition> partitions = hiveLayoutHandle.getPartitions().get(); TupleDomain<ColumnHandle> predicate = createPredicate(partitionColumns, partitions);
@Override public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle tableHandle) { requireNonNull(tableHandle, "tableHandle is null"); SchemaTableName tableName = schemaTableName(tableHandle); return getTableMetadata(tableName); }
@Override public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata) { checkArgument(!isNullOrEmpty(tableMetadata.getOwner()), "Table owner is null or empty"); SchemaTableName schemaTableName = tableMetadata.getTable(); String schemaName = schemaTableName.getSchemaName(); String tableName = schemaTableName.getTableName(); List<String> partitionedBy = getPartitionedBy(tableMetadata.getProperties()); List<HiveColumnHandle> columnHandles = getColumnHandles(connectorId, tableMetadata, ImmutableSet.copyOf(partitionedBy)); HiveStorageFormat hiveStorageFormat = getHiveStorageFormat(tableMetadata.getProperties()); Map<String, String> additionalTableParameters = tableParameterCodec.encode(tableMetadata.getProperties()); LocationHandle locationHandle = locationService.forNewTable(session.getQueryId(), schemaName, tableName); Path targetPath = locationService.targetPathRoot(locationHandle); createDirectory(hdfsEnvironment, targetPath); createTable(schemaName, tableName, tableMetadata.getOwner(), columnHandles, hiveStorageFormat, partitionedBy, additionalTableParameters, targetPath); }
@Override public HiveMetadata get() { SemiTransactionalHiveMetastore metastore = new SemiTransactionalHiveMetastore( hdfsEnvironment, CachingHiveMetastore.memoizeMetastore(this.metastore, perTransactionCacheMaximumSize), // per-transaction cache renameExecution, skipDeletionForAlter); return new HiveMetadata( metastore, hdfsEnvironment, partitionManager, timeZone, allowCorruptWritesForTesting, writesToNonManagedTablesEnabled, createsOfNonManagedTablesEnabled, typeManager, locationService, tableParameterCodec, partitionUpdateCodec, typeTranslator, prestoVersion, new MetastoreHiveStatisticsProvider(metastore), maxPartitions); } }
static class HiveTransaction implements Transaction { private final HiveTransactionManager transactionManager; private final ConnectorTransactionHandle transactionHandle; private boolean closed; public HiveTransaction(HiveTransactionManager transactionManager, HiveMetadata hiveMetadata) { this.transactionManager = requireNonNull(transactionManager, "transactionManager is null"); this.transactionHandle = new HiveTransactionHandle(); transactionManager.put(transactionHandle, hiveMetadata); getMetastore().testOnlyThrowOnCleanupFailures(); } @Override public ConnectorMetadata getMetadata() { return transactionManager.get(transactionHandle); } @Override public SemiTransactionalHiveMetastore getMetastore(String schema) { return getMetastore(); } private SemiTransactionalHiveMetastore getMetastore() { return ((HiveMetadata) transactionManager.get(transactionHandle)).getMetastore();
@Override public void rollback(ConnectorTransactionHandle transaction) { HiveMetadata metadata = transactions.remove(transaction); checkArgument(metadata != null, "no such transaction: %s", transaction); try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) { metadata.rollback(); } }