/** * Gets the sort columns. * * @return the sort columns */ public List<Order> getSortCols() { return this.sd.getSortCols(); }
/** * Gets the sort columns. * * @return the sort columns */ public List<Order> getSortCols() { return this.sd.getSortCols(); }
public List<Order> getSortCols() { return tPartition.getSd().getSortCols(); }
public List<Order> getSortCols() { return tTable.getSd().getSortCols(); }
public List<Order> getSortCols() { return tPartition.getSd().getSortCols(); }
public List<Order> getSortCols() { return tTable.getSd().getSortCols(); }
private void checkSorted(Table newTable) throws MetaException { if(!TxnUtils.isAcidTable(newTable)) { return; } StorageDescriptor sd = newTable.getSd(); if (sd.getSortCols() != null && sd.getSortCols().size() > 0) { throw new MetaException("Table " + Warehouse.getQualifiedName(newTable) + " cannot support full ACID functionality since it is sorted."); } }
public static Optional<HiveBucketProperty> fromStorageDescriptor(StorageDescriptor storageDescriptor, String tablePartitionName) { boolean bucketColsSet = storageDescriptor.isSetBucketCols() && !storageDescriptor.getBucketCols().isEmpty(); boolean numBucketsSet = storageDescriptor.isSetNumBuckets() && storageDescriptor.getNumBuckets() > 0; if (!numBucketsSet) { // In Hive, a table is considered as not bucketed when its bucketCols is set but its numBucket is not set. return Optional.empty(); } if (!bucketColsSet) { throw new PrestoException(HIVE_INVALID_METADATA, "Table/partition metadata has 'numBuckets' set, but 'bucketCols' is not set: " + tablePartitionName); } List<SortingColumn> sortedBy = ImmutableList.of(); if (storageDescriptor.isSetSortCols()) { sortedBy = storageDescriptor.getSortCols().stream() .map(order -> SortingColumn.fromMetastoreApiOrder(order, tablePartitionName)) .collect(toImmutableList()); } return Optional.of(new HiveBucketProperty(storageDescriptor.getBucketCols(), storageDescriptor.getNumBuckets(), sortedBy)); }
static Table assemble(TableWrapper wrapper, SharedCache sharedCache) { Table t = wrapper.getTable().deepCopy(); if (wrapper.getSdHash() != null) { StorageDescriptor sdCopy = sharedCache.getSdFromCache(wrapper.getSdHash()).deepCopy(); if (sdCopy.getBucketCols() == null) { sdCopy.setBucketCols(Collections.emptyList()); } if (sdCopy.getSortCols() == null) { sdCopy.setSortCols(Collections.emptyList()); } if (sdCopy.getSkewedInfo() == null) { sdCopy.setSkewedInfo(new SkewedInfo(Collections.emptyList(), Collections.emptyList(), Collections.emptyMap())); } sdCopy.setLocation(wrapper.getLocation()); sdCopy.setParameters(wrapper.getParameters()); t.setSd(sdCopy); } return t; }
static Partition assemble(PartitionWrapper wrapper, SharedCache sharedCache) { Partition p = wrapper.getPartition().deepCopy(); if (wrapper.getSdHash() != null) { StorageDescriptor sdCopy = sharedCache.getSdFromCache(wrapper.getSdHash()).deepCopy(); if (sdCopy.getBucketCols() == null) { sdCopy.setBucketCols(Collections.emptyList()); } if (sdCopy.getSortCols() == null) { sdCopy.setSortCols(Collections.emptyList()); } if (sdCopy.getSkewedInfo() == null) { sdCopy.setSkewedInfo(new SkewedInfo(Collections.emptyList(), Collections.emptyList(), Collections.emptyMap())); } sdCopy.setLocation(wrapper.getLocation()); sdCopy.setParameters(wrapper.getParameters()); p.setSd(sdCopy); } return p; }
public StorageDescriptorWrapper(StorageDescriptor storageDescriptor) { sd = storageDescriptor; location = storageDescriptor.getLocation(); inputFormat = storageDescriptor.getInputFormat(); outputFormat = storageDescriptor.getOutputFormat(); compressed = storageDescriptor.isCompressed(); numBuckets = storageDescriptor.getNumBuckets(); serDeInfo = new SerDeInfoWrapper(storageDescriptor.getSerdeInfo()); if (sd.getSortCols() != null) { sortCols = Lists.newArrayList(); for (Order order : sd.getSortCols()) { sortCols.add(new OrderWrapper(order)); } } parameters = storageDescriptor.getParameters(); if (sd.getCols() != null) { this.columns = Lists.newArrayList(); for (FieldSchema fieldSchema : sd.getCols()) { this.columns.add(new FieldSchemaWrapper(fieldSchema)); } } }
formatOutput("Num Buckets:", String.valueOf(storageDesc.getNumBuckets()), tableInfo); formatOutput("Bucket Columns:", storageDesc.getBucketCols().toString(), tableInfo); formatOutput("Sort Columns:", storageDesc.getSortCols().toString(), tableInfo); if (storageDesc.isStoredAsSubDirectories()) {// optional parameter formatOutput("Stored As SubDirectories:", "Yes", tableInfo);
formatOutput("Num Buckets:", String.valueOf(storageDesc.getNumBuckets()), tableInfo); formatOutput("Bucket Columns:", storageDesc.getBucketCols().toString(), tableInfo); formatOutput("Sort Columns:", storageDesc.getSortCols().toString(), tableInfo); if (storageDesc.isStoredAsSubDirectories()) {// optional parameter formatOutput("Stored As SubDirectories:", "Yes", tableInfo);
public Object getFieldValue(_Fields field) { switch (field) { case COLS: return getCols(); case LOCATION: return getLocation(); case INPUT_FORMAT: return getInputFormat(); case OUTPUT_FORMAT: return getOutputFormat(); case COMPRESSED: return isCompressed(); case NUM_BUCKETS: return getNumBuckets(); case SERDE_INFO: return getSerdeInfo(); case BUCKET_COLS: return getBucketCols(); case SORT_COLS: return getSortCols(); case PARAMETERS: return getParameters(); case SKEWED_INFO: return getSkewedInfo(); case STORED_AS_SUB_DIRECTORIES: return isStoredAsSubDirectories(); } throw new IllegalStateException(); }
private static AddPartitionDesc getBaseAddPartitionDescFromPartition( Path fromPath, String dbName, ImportTableDesc tblDesc, Partition partition, ReplicationSpec replicationSpec, HiveConf conf) throws MetaException, SemanticException { AddPartitionDesc partsDesc = new AddPartitionDesc(dbName, tblDesc.getTableName(), EximUtil.makePartSpec(tblDesc.getPartCols(), partition.getValues()), partition.getSd().getLocation(), partition.getParameters()); AddPartitionDesc.OnePartitionDesc partDesc = partsDesc.getPartition(0); partDesc.setInputFormat(partition.getSd().getInputFormat()); partDesc.setOutputFormat(partition.getSd().getOutputFormat()); partDesc.setNumBuckets(partition.getSd().getNumBuckets()); partDesc.setCols(partition.getSd().getCols()); partDesc.setSerializationLib(partition.getSd().getSerdeInfo().getSerializationLib()); partDesc.setSerdeParams(partition.getSd().getSerdeInfo().getParameters()); partDesc.setBucketCols(partition.getSd().getBucketCols()); partDesc.setSortCols(partition.getSd().getSortCols()); if (replicationSpec.isInReplicationScope() && tblDesc.isExternal() && !replicationSpec.isMigratingToExternalTable()) { String newLocation = ReplExternalTables .externalTableLocation(conf, partition.getSd().getLocation()); LOG.debug("partition {} has data location: {}", partition, newLocation); partDesc.setLocation(newLocation); } else { partDesc.setLocation(new Path(fromPath, Warehouse.makePartName(tblDesc.getPartCols(), partition.getValues())).toString()); } return partsDesc; }
private static AddPartitionDesc getBaseAddPartitionDescFromPartition( Path fromPath, String dbname, ImportTableDesc tblDesc, Partition partition) throws MetaException, SemanticException { AddPartitionDesc partsDesc = new AddPartitionDesc(dbname, tblDesc.getTableName(), EximUtil.makePartSpec(tblDesc.getPartCols(), partition.getValues()), partition.getSd().getLocation(), partition.getParameters()); AddPartitionDesc.OnePartitionDesc partDesc = partsDesc.getPartition(0); partDesc.setInputFormat(partition.getSd().getInputFormat()); partDesc.setOutputFormat(partition.getSd().getOutputFormat()); partDesc.setNumBuckets(partition.getSd().getNumBuckets()); partDesc.setCols(partition.getSd().getCols()); partDesc.setSerializationLib(partition.getSd().getSerdeInfo().getSerializationLib()); partDesc.setSerdeParams(partition.getSd().getSerdeInfo().getParameters()); partDesc.setBucketCols(partition.getSd().getBucketCols()); partDesc.setSortCols(partition.getSd().getSortCols()); partDesc.setLocation(new Path(fromPath, Warehouse.makePartName(tblDesc.getPartCols(), partition.getValues())).toString()); return partsDesc; }
sd.getBucketCols().isEmpty()); Assert.assertTrue("The default value of the attribute 'sortCols' should be an empty list.", sd.getSortCols().isEmpty()); Assert.assertTrue("Per default the storage descriptor parameters should be empty.", sd.getParameters().isEmpty());
/** * Converts a storage descriptor to a db-backed storage descriptor. It points the * storage descriptor's column descriptor to the one passed as an argument, * so it does not create a new mcolumn descriptor object. * @param sd the storage descriptor to wrap in a db-backed object * @param mcd the db-backed column descriptor * @return the db-backed storage descriptor object * @throws MetaException */ private MStorageDescriptor convertToMStorageDescriptor(StorageDescriptor sd, MColumnDescriptor mcd) throws MetaException { if (sd == null) { return null; } return new MStorageDescriptor(mcd, sd .getLocation(), sd.getInputFormat(), sd.getOutputFormat(), sd .isCompressed(), sd.getNumBuckets(), convertToMSerDeInfo(sd .getSerdeInfo()), sd.getBucketCols(), convertToMOrders(sd.getSortCols()), sd.getParameters(), (null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo().getSkewedColNames(), convertToMStringLists((null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo() .getSkewedColValues()), covertToMapMStringList((null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo() .getSkewedColValueLocationMaps()), sd.isStoredAsSubDirectories()); }
partDesc.setSerdeParams(partition.getSd().getSerdeInfo().getParameters()); partDesc.setBucketCols(partition.getSd().getBucketCols()); partDesc.setSortCols(partition.getSd().getSortCols()); if (tblDesc.isExternal() && !replicationSpec().isMigratingToExternalTable()) {
private Partition createPartition(List<String> vals, Table table) throws MetaException { return new PartitionBuilder() .inTable(table) .setValues(vals) .addPartParam("key1", "S1") .addPartParam("key2", "S2") .addPartParam(EXCLUDE_KEY_PREFIX + "key1", "e1") .addPartParam(EXCLUDE_KEY_PREFIX + "key2", "e2") .setBucketCols(table.getSd().getBucketCols()) .setSortCols(table.getSd().getSortCols()) .setSerdeName(table.getSd().getSerdeInfo().getName()) .setSerdeLib(table.getSd().getSerdeInfo().getSerializationLib()) .setSerdeParams(table.getSd().getSerdeInfo().getParameters()) .build(conf); }