public StorageKey reuseFor(List<String> dirs, PathConversion conversion) { return reuseFor(new Path(PATH_JOINER.join(dirs)), conversion); }
public static <T> String dirnameForValue(FieldPartitioner<?, T> field, T value) { return PART_JOIN.join(field.getName(), valueToString(field, value)); }
/** * Precondition-style validation that a {@link Schema} is compatible. * * @param schema an avro {@code Schema} */ public static void checkSchema(Schema schema) { Preconditions.checkNotNull(schema, "Schema cannot be null"); List<String> incompatible = getIncompatibleNames(schema); ValidationException.check(incompatible.isEmpty(), "Field names are not alphanumeric (plus '_'): %s", Joiner.on(", ").join(incompatible)); }
private static String constructPath(Map<String, String> match, String pattern) { LinkedList<String> patternParts = Lists.newLinkedList( PATH_SPLITTER.split(pattern)); List<String> pathParts = Lists .newArrayListWithExpectedSize(patternParts.size()); for (String part : patternParts) { if (!part.isEmpty()) { // only supports simple matches, like matching if (VAR_START.matches(part.charAt(0))) { String name = part.substring(1); if (match.containsKey(name)) { pathParts.add(match.remove(name)); } else if (!part.startsWith("*")) { // required variable is missing throw new IllegalArgumentException( "Missing required option: " + name); } } else { // it is a required URI part pathParts.add(part); } } else { // preserve additional slashes in paths pathParts.add(part); } } return PATH_JOINER.join(pathParts); }
@Override public String toNormalizedString(Schema schema) { // we want the string this returns to be consistent // ensure the set is sorted before joining it. // ToString first allows us to consistently sort even when // the value may not be comparable Iterable<String> strings = Iterables.transform(set, new ToString<T>(schema)); return Joiner.on(',').join(Sets.newTreeSet(strings)); }
/** * Returns the nested {@link Schema} for the given field name. * * @param schema a record Schema * @param name a String field name * @return the nested Schema for the field */ public static Schema fieldSchema(Schema schema, String name) { Schema nested = unwrapNullable(schema); List<String> levels = Lists.newArrayList(); for (String level : NAME_SPLITTER.split(name)) { levels.add(level); ValidationException.check(Schema.Type.RECORD == schema.getType(), "Cannot get schema for %s: %s is not a record schema: %s", name, NAME_JOINER.join(levels), nested.toString(true)); Schema.Field field = nested.getField(level); ValidationException.check(field != null, "Cannot get schema for %s: %s is not a field", name, NAME_JOINER.join(levels)); nested = unwrapNullable(field.schema()); } return nested; }
private static void addPropertiesForDescriptor(Table table, DatasetDescriptor descriptor) { // copy custom properties to the table if (!descriptor.listProperties().isEmpty()) { for (String property : descriptor.listProperties()) { // no need to check the reserved list, those are not set on descriptors table.getParameters().put(property, descriptor.getProperty(property)); } // set which properties are custom and should be set on descriptors table.getParameters().put(CUSTOM_PROPERTIES_PROPERTY_NAME, NAME_JOINER.join(descriptor.listProperties())); } }
@VisibleForTesting public static String[] parseHostsAndPort(String zkQuorum) { List<String> hosts = Lists.newArrayList(); String port = null; for (String hostPort : Splitter.on(',').split(zkQuorum)) { Iterator<String> split = Splitter.on(':').split(hostPort).iterator(); hosts.add(split.next()); if (split.hasNext()) { String p = split.next(); if (port == null) { port = p; } else if (!port.equals(p)) { throw new IllegalArgumentException("Mismatched ports in " + zkQuorum); } } } return new String[] { Joiner.on(',').join(hosts), port }; } }
private static void addPropertiesForDescriptor(Table table, DatasetDescriptor descriptor) { // copy custom properties to the table if (!descriptor.listProperties().isEmpty()) { for (String property : descriptor.listProperties()) { // no need to check the reserved list, those are not set on descriptors table.getParameters().put(property, descriptor.getProperty(property)); } // set which properties are custom and should be set on descriptors table.getParameters().put(CUSTOM_PROPERTIES_PROPERTY_NAME, NAME_JOINER.join(descriptor.listProperties())); } }
@VisibleForTesting public static String[] parseHostsAndPort(String zkQuorum) { List<String> hosts = Lists.newArrayList(); String port = null; for (String hostPort : Splitter.on(',').split(zkQuorum)) { Iterator<String> split = Splitter.on(':').split(hostPort).iterator(); hosts.add(split.next()); if (split.hasNext()) { String p = split.next(); if (port == null) { port = p; } else if (!port.equals(p)) { throw new IllegalArgumentException("Mismatched ports in " + zkQuorum); } } } return new String[] { Joiner.on(',').join(hosts), port }; } }
final String joined = PATH_JOINER.join(parts); if (!joined.isEmpty() || matchedAfterGlob || matchedBeforeGlob) { storage.put(globPattern.substring(1), joined);
public static Pair<DatasetRepository, Map<String, String>> lookupDatasetUri(URI datasetUri) { String scheme = datasetUri.getScheme(); List<String> schemeMatches = Lists.newArrayList(); for (URIPattern pattern : DATASET_PATTERNS.keySet()) { Map<String, String> match = pattern.getMatch(datasetUri); if (match != null) { return Pair.of(DATASET_PATTERNS.get(pattern).getFromOptions(match), match); } else if (pattern.getScheme() != null && pattern.getScheme().equals(scheme)) { schemeMatches.add(pattern.getPatternString()); } } String message = "Unknown dataset URI pattern: dataset:" + datasetUri; if (schemeMatches.isEmpty()) { // no known patterns for the scheme, maybe jars are missing message += "\nCheck that JARs for " + scheme + " datasets are on the classpath"; } else { // show the known patterns in case it's a simple error message += "\nKnown patterns for " + scheme + ":\n dataset:" + Joiner.on("\n dataset:").join(schemeMatches); } throw new DatasetNotFoundException(message); }
} else { recordSchema = Schema.createRecord( DOT.join(recordLevels), null, null, false);