builder.compressionType(properties.getProperty(COMPRESSION_TYPE_FIELD_NAME));
builder.compressionType(compressionType);
builder.compressionType(compressionType);
.schema(schema) .format(datasetDefinition.getFormat()) .compressionType(datasetDefinition.getCompressionType()) .build(); .schema(schema) .format(datasetDefinition.getFormat()) .compressionType(datasetDefinition.getCompressionType()) .partitionStrategy(datasetDefinition.getPartitionStrategy()); if (datasetDefinition.getWriterCacheSize() != null) {
.schema(USER_SCHEMA) .format(format) .compressionType(compressionType) .location(testDirectory) .partitionStrategy(partitionStrategy)
.schema(USER_NULLABLE_SCHEMA) .format(format) .compressionType(compressionType) .location(testDirectory) .partitionStrategy(partitionStrategy)
.schema(USER_SCHEMA) .format(format) .compressionType(compressionType) .location(testDirectory) .partitionStrategy(partitionStrategy) .schema(USER_SCHEMA) .format(format) .compressionType(compressionType) .location(newTestDirectory) .partitionStrategy(partitionStrategy)
.schema(USER_SCHEMA) .format(format) .compressionType(compressionType) .location(testDirectory) .partitionStrategy(partitionStrategy)
.schema(USER_SCHEMA) .format(format) .compressionType(compressionType) .location(testDirectory) .partitionStrategy(partitionStrategy)
.schema(USER_SCHEMA) .format(format) .compressionType(compressionType) .location(testDirectory) .partitionStrategy(partitionStrategy)
@Test @SuppressWarnings("deprecation") public void testWriteToSubpartition() throws IOException { PartitionStrategy partitionStrategy = new PartitionStrategy.Builder() .hash("username", "username_part", 2).hash("email", 3).build(); FileSystemDataset<Record> ds = new FileSystemDataset.Builder<Record>() .namespace("ns") .name("partitioned-users") .configuration(getConfiguration()) .descriptor(new DatasetDescriptor.Builder() .schema(USER_SCHEMA) .format(format) .compressionType(compressionType) .location(testDirectory) .partitionStrategy(partitionStrategy) .build()) .type(Record.class) .build(); PartitionKey key = new PartitionKey(1); FileSystemDataset<Record> userPartition = (FileSystemDataset<Record>) ds.getPartition(key, true); Assert.assertEquals(key, userPartition.getPartitionKey()); writeTestUsers(userPartition, 1); Assert.assertTrue("Partitioned directory exists", fileSystem.exists(new Path(testDirectory, "username_part=1/email_hash=2"))); Assert.assertEquals(1, readTestUsersInPartition(ds, key, "email_hash")); }
@Test public void testWriteAndRead() throws IOException { FileSystemDataset<Record> ds = new FileSystemDataset.Builder<Record>() .namespace("ns") .name("test") .configuration(getConfiguration()) .descriptor(new DatasetDescriptor.Builder() .schemaUri(USER_SCHEMA_URL) .format(format) .compressionType(compressionType) .location(testDirectory) .build()) .type(Record.class) .build(); Assert.assertFalse("Dataset is not partitioned", ds.getDescriptor() .isPartitioned()); writeTestUsers(ds, 10); checkTestUsers(ds, 10); }
@Test public void testPathIterator_Directory() { FileSystemDataset<Record> ds = new FileSystemDataset.Builder<Record>() .namespace("ns") .name("users") .configuration(getConfiguration()) .descriptor(new DatasetDescriptor.Builder() .schema(USER_SCHEMA) .format(format) .compressionType(compressionType) .location(testDirectory) .build()) .type(Record.class) .build(); List<Path> dirPaths = Lists.newArrayList(ds.dirIterator()); Assert.assertEquals("dirIterator for non-partitioned dataset should yield a single path.", 1, dirPaths.size()); Assert.assertEquals("dirIterator should yield absolute paths.", testDirectory, dirPaths.get(0)); }
/** * Configure the dataset's compression format (optional). If not set, * default to {@link CompressionType#Snappy}. * * @param compressionTypeName the name of the compression format * * @return This builder for method chaining * * @since 0.17.0 */ public Builder compressionType(String compressionTypeName) { return compressionType(CompressionType.forName(compressionTypeName)); }