this.maxOpenSortFiles = maxOpenSortFiles; this.immutablePartitions = immutablePartitions; this.insertExistingPartitionsBehavior = HiveSessionProperties.getInsertExistingPartitionsBehavior(session); if (immutablePartitions) { checkArgument(insertExistingPartitionsBehavior != InsertExistingPartitionsBehavior.APPEND, "insertExistingPartitionsBehavior cannot be APPEND");
private void verifyPartitionedBucketedTable(HiveStorageFormat storageFormat, String tableName) { TableMetadata tableMetadata = getTableMetadata(catalog, TPCH_SCHEMA, tableName); assertEquals(tableMetadata.getMetadata().getProperties().get(STORAGE_FORMAT_PROPERTY), storageFormat); assertEquals(tableMetadata.getMetadata().getProperties().get(PARTITIONED_BY_PROPERTY), ImmutableList.of("orderstatus")); assertEquals(tableMetadata.getMetadata().getProperties().get(BUCKETED_BY_PROPERTY), ImmutableList.of("custkey", "custkey2")); assertEquals(tableMetadata.getMetadata().getProperties().get(BUCKET_COUNT_PROPERTY), 11); List<?> partitions = getPartitions(tableName); assertEquals(partitions.size(), 3); assertQuery("SELECT * from " + tableName, "SELECT custkey, custkey, comment, orderstatus FROM orders"); for (int i = 1; i <= 30; i++) { assertQuery( format("SELECT * from " + tableName + " where custkey = %d and custkey2 = %d", i, i), format("SELECT custkey, custkey, comment, orderstatus FROM orders where custkey = %d", i)); } assertThatThrownBy(() -> assertUpdate("INSERT INTO " + tableName + " VALUES (1, 1, 'comment', 'O')", 1)) .hasMessage(getExpectedErrorMessageForInsertExistingBucketedTable( getInsertExistingPartitionsBehavior(getConnectorSession(getSession())), "orderstatus=O")); }
private void testInsertPartitionedBucketedTableFewRows(Session session, HiveStorageFormat storageFormat) { String tableName = "test_insert_partitioned_bucketed_table_few_rows"; assertUpdate(session, "" + "CREATE TABLE " + tableName + " (" + " bucket_key varchar," + " col varchar," + " partition_key varchar)" + "WITH (" + "format = '" + storageFormat + "', " + "partitioned_by = ARRAY[ 'partition_key' ], " + "bucketed_by = ARRAY[ 'bucket_key' ], " + "bucket_count = 11)"); assertUpdate( // make sure that we will get one file per bucket regardless of writer count configured getParallelWriteSession(), "INSERT INTO " + tableName + " " + "VALUES " + " (VARCHAR 'a', VARCHAR 'b', VARCHAR 'c'), " + " ('aa', 'bb', 'cc'), " + " ('aaa', 'bbb', 'ccc')", 3); verifyPartitionedBucketedTableAsFewRows(storageFormat, tableName); assertThatThrownBy(() -> assertUpdate(session, "INSERT INTO test_insert_partitioned_bucketed_table_few_rows VALUES ('a0', 'b0', 'c')", 1)) .hasMessage(getExpectedErrorMessageForInsertExistingBucketedTable( getInsertExistingPartitionsBehavior(getConnectorSession(session)), "partition_key=c")); assertUpdate(session, "DROP TABLE test_insert_partitioned_bucketed_table_few_rows"); assertFalse(getQueryRunner().tableExists(session, tableName)); }
getInsertExistingPartitionsBehavior(getConnectorSession(session)), "partition_key=c"));
private void verifyPartitionedBucketedTable(HiveStorageFormat storageFormat, String tableName) { TableMetadata tableMetadata = getTableMetadata(catalog, TPCH_SCHEMA, tableName); assertEquals(tableMetadata.getMetadata().getProperties().get(STORAGE_FORMAT_PROPERTY), storageFormat); assertEquals(tableMetadata.getMetadata().getProperties().get(PARTITIONED_BY_PROPERTY), ImmutableList.of("orderstatus")); assertEquals(tableMetadata.getMetadata().getProperties().get(BUCKETED_BY_PROPERTY), ImmutableList.of("custkey", "custkey2")); assertEquals(tableMetadata.getMetadata().getProperties().get(BUCKET_COUNT_PROPERTY), 11); List<?> partitions = getPartitions(tableName); assertEquals(partitions.size(), 3); assertQuery("SELECT * from " + tableName, "SELECT custkey, custkey, comment, orderstatus FROM orders"); for (int i = 1; i <= 30; i++) { assertQuery( format("SELECT * from " + tableName + " where custkey = %d and custkey2 = %d", i, i), format("SELECT custkey, custkey, comment, orderstatus FROM orders where custkey = %d", i)); } assertThatThrownBy(() -> assertUpdate("INSERT INTO " + tableName + " VALUES (1, 1, 'comment', 'O')", 1)) .hasMessage(getExpectedErrorMessageForInsertExistingBucketedTable( getInsertExistingPartitionsBehavior(getConnectorSession(getSession())), "orderstatus=O")); }
private void testInsertPartitionedBucketedTableFewRows(Session session, HiveStorageFormat storageFormat) { String tableName = "test_insert_partitioned_bucketed_table_few_rows"; assertUpdate(session, "" + "CREATE TABLE " + tableName + " (" + " bucket_key varchar," + " col varchar," + " partition_key varchar)" + "WITH (" + "format = '" + storageFormat + "', " + "partitioned_by = ARRAY[ 'partition_key' ], " + "bucketed_by = ARRAY[ 'bucket_key' ], " + "bucket_count = 11)"); assertUpdate( // make sure that we will get one file per bucket regardless of writer count configured getParallelWriteSession(), "INSERT INTO " + tableName + " " + "VALUES " + " (VARCHAR 'a', VARCHAR 'b', VARCHAR 'c'), " + " ('aa', 'bb', 'cc'), " + " ('aaa', 'bbb', 'ccc')", 3); verifyPartitionedBucketedTableAsFewRows(storageFormat, tableName); assertThatThrownBy(() -> assertUpdate(session, "INSERT INTO test_insert_partitioned_bucketed_table_few_rows VALUES ('a0', 'b0', 'c')", 1)) .hasMessage(getExpectedErrorMessageForInsertExistingBucketedTable( getInsertExistingPartitionsBehavior(getConnectorSession(session)), "partition_key=c")); assertUpdate(session, "DROP TABLE test_insert_partitioned_bucketed_table_few_rows"); assertFalse(getQueryRunner().tableExists(session, tableName)); }
getInsertExistingPartitionsBehavior(getConnectorSession(session)), "partition_key=c"));