/** * Create the partition key. * * @throws IllegalStateException if no fields have been added, * or the partitioning is known and not all fields have been added. */ public PartitionKey build() { if (fields.isEmpty()) { throw new IllegalStateException("Partition key cannot be empty."); } if (partitioning != null && !partitioning.getFields().keySet().equals(fields.keySet())) { throw new IllegalStateException(String.format( "Partition key is incomplete: It only contains fields %s, but the partitioning requires %s", fields.keySet(), partitioning.getFields().keySet())); } return new PartitionKey(fields); } }
public static String getOutputPath(PartitionKey key, Partitioning partitioning) { validatePartitionKey(key, partitioning); StringBuilder builder = new StringBuilder(); String sep = ""; for (String fieldName : partitioning.getFields().keySet()) { builder.append(sep).append(key.getField(fieldName).toString()); sep = "/"; } return builder.toString(); }
public static String getOutputPath(PartitionKey key, Partitioning partitioning) { validatePartitionKey(key, partitioning); StringBuilder builder = new StringBuilder(); String sep = ""; for (String fieldName : partitioning.getFields().keySet()) { builder.append(sep).append(key.getField(fieldName).toString()); sep = "/"; } return builder.toString(); }
/** * Logs a warning if the partition filter contains a field that is not part of the partitioning. */ private void warnIfInvalidPartitionFilter(PartitionFilter filter, Partitioning partitioning) { if (null == filter) { return; } for (Map.Entry<String, PartitionFilter.Condition<? extends Comparable>> entry : filter.getConditions().entrySet()) { if (!partitioning.getFields().containsKey(entry.getKey())) { LOG.warn("Partition filter cannot match any partitions in dataset '{}' because it contains field '{}' " + "that is not a valid partitioning field", getName(), entry.getKey()); } } }
/** * Sets the base path for the file dataset. */ public Builder setPartitioning(Partitioning partitioning) { StringBuilder builder = new StringBuilder(); String sep = ""; for (String key : partitioning.getFields().keySet()) { builder.append(sep).append(key); sep = ","; } add(PARTITIONING_FIELDS, builder.toString()); for (Map.Entry<String, Partitioning.FieldType> entry : partitioning.getFields().entrySet()) { add(PARTITIONING_FIELD_PREFIX + entry.getKey(), entry.getValue().name()); } return this; }
/** * Logs a warning if the partition filter contains a field that is not part of the partitioning. */ private void warnIfInvalidPartitionFilter(PartitionFilter filter, Partitioning partitioning) { if (null == filter) { return; } for (Map.Entry<String, PartitionFilter.Condition<? extends Comparable>> entry : filter.getConditions().entrySet()) { if (!partitioning.getFields().containsKey(entry.getKey())) { LOG.warn("Partition filter cannot match any partitions in dataset '{}' because it contains field '{}' " + "that is not a valid partitioning field", getName(), entry.getKey()); } } }
private PartitionKey getPartitionKey(Partitioning partitioning, String relativePath) { List<String> pathParts = Arrays.asList(relativePath.split(Path.SEPARATOR)); if (pathParts.size() != partitioning.getFields().size()) { throw new IllegalArgumentException( String.format("relativePath '%s' does not have same number of components as partitioning '%s", relativePath, partitioning)); } PartitionKey.Builder builder = PartitionKey.builder(); int i = 0; for (Map.Entry<String, Partitioning.FieldType> entry : partitioning.getFields().entrySet()) { String keyName = entry.getKey(); Comparable keyValue = entry.getValue().parse(pathParts.get(i)); builder.addField(keyName, keyValue); i++; } return builder.build(); }
private PartitionKey getPartitionKey(Partitioning partitioning, String relativePath) { List<String> pathParts = Arrays.asList(relativePath.split(Path.SEPARATOR)); if (pathParts.size() != partitioning.getFields().size()) { throw new IllegalArgumentException( String.format("relativePath '%s' does not have same number of components as partitioning '%s", relativePath, partitioning)); } PartitionKey.Builder builder = PartitionKey.builder(); int i = 0; for (Map.Entry<String, Partitioning.FieldType> entry : partitioning.getFields().entrySet()) { String keyName = entry.getKey(); Comparable keyValue = entry.getValue().parse(pathParts.get(i)); builder.addField(keyName, keyValue); i++; } return builder.build(); }
/** * Validates the partition key against the partitioning. */ private static void validatePartitionKey(PartitionKey key, Partitioning partitioning) { if (!partitioning.getFields().keySet().equals(key.getFields().keySet())) { throw new IllegalArgumentException(String.format( "Partition key is invalid: It contains fields %s, but the partitioning requires %s", key.getFields().keySet(), partitioning.getFields().keySet())); } for (Map.Entry<String, FieldType> entry : partitioning.getFields().entrySet()) { String fieldName = entry.getKey(); FieldType fieldType = entry.getValue(); Comparable fieldValue = key.getField(fieldName); if (fieldValue == null) { throw new IllegalArgumentException( String.format("Incomplete partition key: value for field '%s' is missing", fieldName)); } try { fieldType.validate(fieldValue); } catch (IllegalArgumentException e) { throw new IllegalArgumentException(String.format( "Invalid partition key: Value for field '%s' is incompatible with the partitioning: %s", fieldName, e.getMessage())); } } }
if (!partitioning.getFields().containsKey(name)) { throw new IllegalArgumentException(String.format("Field '%s' is an unknown field in partitioning %s", name, partitioning)); partitioning.getFields().get(name).validate(value); } catch (IllegalArgumentException e) { throw new IllegalArgumentException(String.format(
/** * Validates the partition key against the partitioning. */ private static void validatePartitionKey(PartitionKey key, Partitioning partitioning) { if (!partitioning.getFields().keySet().equals(key.getFields().keySet())) { throw new IllegalArgumentException(String.format( "Partition key is invalid: It contains fields %s, but the partitioning requires %s", key.getFields().keySet(), partitioning.getFields().keySet())); } for (Map.Entry<String, FieldType> entry : partitioning.getFields().entrySet()) { String fieldName = entry.getKey(); FieldType fieldType = entry.getValue(); Comparable fieldValue = key.getField(fieldName); if (fieldValue == null) { throw new IllegalArgumentException( String.format("Incomplete partition key: value for field '%s' is missing", fieldName)); } try { fieldType.validate(fieldValue); } catch (IllegalArgumentException e) { throw new IllegalArgumentException(String.format( "Invalid partition key: Value for field '%s' is incompatible with the partitioning: %s", fieldName, e.getMessage())); } } }
/** * Validates the partition key against the partitioning and gererates the row key for that partition key. */ @VisibleForTesting static byte[] generateRowKey(PartitionKey key, Partitioning partitioning) { validatePartitionKey(key, partitioning); // validate partition key, convert values, and compute size of output Map<String, FieldType> partitionFields = partitioning.getFields(); int totalSize = partitionFields.size() - 1; // one \0 between each of the fields ArrayList<byte[]> values = Lists.newArrayListWithCapacity(partitionFields.size()); for (Map.Entry<String, FieldType> entry : partitionFields.entrySet()) { String fieldName = entry.getKey(); FieldType fieldType = entry.getValue(); Comparable fieldValue = key.getField(fieldName); byte[] bytes = FieldTypes.toBytes(fieldValue, fieldType); totalSize += bytes.length; values.add(bytes); } byte[] rowKey = new byte[totalSize]; int offset = 0; for (byte[] bytes : values) { System.arraycopy(bytes, 0, rowKey, offset, bytes.length); offset += bytes.length + 1; // this leaves a \0 byte after the value } return rowKey; }
/** * Validates the partition key against the partitioning and gererates the row key for that partition key. */ @VisibleForTesting static byte[] generateRowKey(PartitionKey key, Partitioning partitioning) { validatePartitionKey(key, partitioning); // validate partition key, convert values, and compute size of output Map<String, FieldType> partitionFields = partitioning.getFields(); int totalSize = partitionFields.size() - 1; // one \0 between each of the fields ArrayList<byte[]> values = Lists.newArrayListWithCapacity(partitionFields.size()); for (Map.Entry<String, FieldType> entry : partitionFields.entrySet()) { String fieldName = entry.getKey(); FieldType fieldType = entry.getValue(); Comparable fieldValue = key.getField(fieldName); byte[] bytes = FieldTypes.toBytes(fieldValue, fieldType); totalSize += bytes.length; values.add(bytes); } byte[] rowKey = new byte[totalSize]; int offset = 0; for (byte[] bytes : values) { System.arraycopy(bytes, 0, rowKey, offset, bytes.length); offset += bytes.length + 1; // this leaves a \0 byte after the value } return rowKey; }
/** * @return the partition key of the output partition to be written; or null if no partition key was found * * @param arguments the runtime arguments for a partitioned dataset * @param partitioning the declared partitioning for the dataset, needed for proper interpretation of values */ @Nullable public static PartitionKey getOutputPartitionKey(Map<String, String> arguments, Partitioning partitioning) { // extract the arguments that describe the output partition key Map<String, String> keyArguments = FileSetProperties.propertiesWithPrefix(arguments, OUTPUT_PARTITION_KEY_PREFIX); if (keyArguments.isEmpty()) { return null; // there is no output partition key } // there is a partition key; now it is required to match the partitioning PartitionKey.Builder builder = PartitionKey.builder(); for (Map.Entry<String, FieldType> entry : partitioning.getFields().entrySet()) { String fieldName = entry.getKey(); FieldType fieldType = entry.getValue(); String stringValue = keyArguments.get(fieldName); Comparable fieldValue = convertFieldValue("key", "value", fieldName, fieldType, stringValue, false); builder.addField(fieldName, fieldValue); } return builder.build(); }
@VisibleForTesting static List<PartitionFilter> partitionFiltersForTimeRange(long startTime, long endTime) { // unsatisfiable range if (startTime >= endTime) { return Collections.emptyList(); } PartitionKey keyLower = startTime <= 0 ? null : partitionKeyForTime(startTime); PartitionKey keyUpper = endTime == Long.MAX_VALUE ? null : partitionKeyForTime(endTime); // no bounds -> no filter if (keyLower == null && keyUpper == null) { return Collections.singletonList(null); // no filter needed to select all time } List<PartitionFilter> filters = Lists.newArrayList(); String[] allFields = PARTITIONING.getFields().keySet().toArray(new String[PARTITIONING.getFields().size()]); // if there is no lower bound, we only need the filters for the upper bound if (keyLower == null) { addUpperFilters(allFields, 0, keyUpper, filters, initialSupplier()); return filters; } // if there is no upper bound, we only need the filters for the lower bound if (keyUpper == null) { addLowerFilters(allFields, 0, keyLower, filters, initialSupplier()); return filters; } return filtersFor(allFields, 0, keyLower, keyUpper, filters, initialSupplier()); }
@VisibleForTesting static List<PartitionFilter> partitionFiltersForTimeRange(long startTime, long endTime) { // unsatisfiable range if (startTime >= endTime) { return Collections.emptyList(); } PartitionKey keyLower = startTime <= 0 ? null : partitionKeyForTime(startTime); PartitionKey keyUpper = endTime == Long.MAX_VALUE ? null : partitionKeyForTime(endTime); // no bounds -> no filter if (keyLower == null && keyUpper == null) { return Collections.singletonList(null); // no filter needed to select all time } List<PartitionFilter> filters = Lists.newArrayList(); String[] allFields = PARTITIONING.getFields().keySet().toArray(new String[PARTITIONING.getFields().size()]); // if there is no lower bound, we only need the filters for the upper bound if (keyLower == null) { addUpperFilters(allFields, 0, keyUpper, filters, initialSupplier()); return filters; } // if there is no upper bound, we only need the filters for the lower bound if (keyUpper == null) { addLowerFilters(allFields, 0, keyLower, filters, initialSupplier()); return filters; } return filtersFor(allFields, 0, keyLower, keyUpper, filters, initialSupplier()); }
int offset = 0; boolean first = true; for (Map.Entry<String, FieldType> entry : partitioning.getFields().entrySet()) { String fieldName = entry.getKey(); FieldType fieldType = entry.getValue();
Preconditions.checkNotNull(oldPartitioning, "Existing dataset has no partitioning"); Preconditions.checkNotNull(newPartitioning, "New properties do not contain partitioning"); if (!Iterators.elementsEqual(oldPartitioning.getFields().entrySet().iterator(), newPartitioning.getFields().entrySet().iterator())) { throw new IncompatibleUpdateException(String.format( "Partitioning cannot be changed. Existing: %s, new: %s", oldPartitioning, newPartitioning));
@Test public void testFieldOrder() { Partitioning partitioning = Partitioning.builder() .addIntField("1") .addLongField("2") .addStringField("3") .build(); Iterator<Map.Entry<String, FieldType>> iterator = partitioning.getFields().entrySet().iterator(); Assert.assertEquals("1", iterator.next().getKey()); Assert.assertEquals("2", iterator.next().getKey()); Assert.assertEquals("3", iterator.next().getKey()); Assert.assertFalse(iterator.hasNext()); // the previous order may have been preserved by chance. Now try the reverse order partitioning = Partitioning.builder() .addIntField("3") .addLongField("2") .addStringField("1") .build(); iterator = partitioning.getFields().entrySet().iterator(); Assert.assertEquals("3", iterator.next().getKey()); Assert.assertEquals("2", iterator.next().getKey()); Assert.assertEquals("1", iterator.next().getKey()); Assert.assertFalse(iterator.hasNext()); }
@Test public void testBuilderGetters() { Partitioning partitioning = Partitioning.builder() .addField("a", FieldType.STRING) .addField("b", FieldType.INT) .addField("c", FieldType.LONG) .addStringField("d") .addIntField("e") .addLongField("f") .build(); Assert.assertEquals(FieldType.STRING, partitioning.getFieldType("a")); Assert.assertEquals(FieldType.INT, partitioning.getFieldType("b")); Assert.assertEquals(FieldType.LONG, partitioning.getFieldType("c")); Assert.assertEquals(FieldType.STRING, partitioning.getFieldType("d")); Assert.assertEquals(FieldType.INT, partitioning.getFieldType("e")); Assert.assertEquals(FieldType.LONG, partitioning.getFieldType("f")); Assert.assertNull(partitioning.getFieldType("x")); Assert.assertEquals(partitioning.getFields().keySet(), ImmutableSet.of("a", "b", "c", "d", "e", "f")); }