private static PartitionSpec identitySpec(Schema schema, String... partitionNames) { return identitySpec(schema, Lists.newArrayList(partitionNames)); }
private static PartitionSpec identitySpec(Schema schema, Collection<Column> columns) { List<String> names = Lists.newArrayList(); for (Column column : columns) { if (column.isPartition()) { names.add(column.name()); } } return identitySpec(schema, names); }
/** * Returns a {@link PartitionSpec} for the given table. * <p> * This creates a partition spec for an existing table by looking up the table's schema and * creating a spec with identity partitions for each partition column. * * @param spark a Spark session * @param name a table name and (optional) database * @return a PartitionSpec for the table, if found * @throws AnalysisException if thrown by the Spark catalog */ public static PartitionSpec specForTable(SparkSession spark, String name) throws AnalysisException { List<String> parts = Lists.newArrayList(Splitter.on('.').limit(2).split(name)); String db = parts.size() == 1 ? "default" : parts.get(0); String table = parts.get(parts.size() == 1 ? 0 : 1); return identitySpec( schemaForTable(spark, name), spark.catalog().listColumns(db, table).collectAsList()); }