private static List<Partition> convertFromMetastore(Table tbl, List<org.apache.hadoop.hive.metastore.api.Partition> partitions) throws HiveException { if (partitions == null) { return new ArrayList<Partition>(); } List<Partition> results = new ArrayList<Partition>(partitions.size()); for (org.apache.hadoop.hive.metastore.api.Partition tPart : partitions) { results.add(new Partition(tbl, tPart)); } return results; }
public Path[] getPath() { Path[] ret = new Path[]{getDataLocation()}; return ret; }
@Override public Object getOutput() throws HiveException { return new Partition(table, partition.getTPartition()); }
private void validatePartition(Partition newPart) throws HiveException { // Remove the DDL time so that it gets refreshed if (newPart.getParameters() != null) { newPart.getParameters().remove(hive_metastoreConstants.DDL_TIME); } newPart.checkValidity(); }
AcidUtils.TableSnapshot tableSnapshot = null; if (transactional) { tableSnapshot = AcidUtils.getTableSnapshot(conf, newParts.get(0).getTable(), true); if (tmpPart.getParameters() != null) { tmpPart.getParameters().remove(hive_metastoreConstants.DDL_TIME); String location = tmpPart.getLocation(); if (location != null) { location = Utilities.getQualifiedPath(conf, new Path(location)); tmpPart.setLocation(location); newTParts.add(tmpPart.getTPartition());
private Partition getTargetPartition(Partition originPartition, Path targetLocation) throws IOException { try { Partition targetPartition = new Partition(this.hiveCopyEntityHelper.getTargetTable(), originPartition.getTPartition().deepCopy()); targetPartition.getTable().setDbName(this.hiveCopyEntityHelper.getTargetDatabase()); targetPartition.getTPartition().setDbName(this.hiveCopyEntityHelper.getTargetDatabase()); targetPartition.getTPartition().putToParameters(HiveDataset.REGISTERER, HiveCopyEntityHelper.GOBBLIN_DISTCP); targetPartition.getTPartition().putToParameters(HiveDataset.REGISTRATION_GENERATION_TIME_MILLIS, Long.toString(this.hiveCopyEntityHelper.getStartTime())); targetPartition.setLocation(targetLocation.toString()); targetPartition.getTPartition().unsetCreateTime(); return targetPartition; } catch (HiveException he) { throw new IOException(he); } }
private static void getPartitionMetaDataInformation(StringBuilder tableInfo, Partition part) { formatOutput("Partition Value:", part.getValues().toString(), tableInfo); formatOutput("Database:", part.getTPartition().getDbName(), tableInfo); formatOutput("Table:", part.getTable().getTableName(), tableInfo); formatOutput("CreateTime:", formatDate(part.getTPartition().getCreateTime()), tableInfo); formatOutput("LastAccessTime:", formatDate(part.getTPartition().getLastAccessTime()), tableInfo); formatOutput("Location:", part.getLocation(), tableInfo); if (part.getTPartition().getParameters().size() > 0) { tableInfo.append("Partition Parameters:").append(LINE_DELIM); displayAllParameters(part.getTPartition().getParameters(), tableInfo); } }
public PartitionDesc(final Partition part) throws HiveException { PartitionDescConstructorHelper(part, getTableDesc(part.getTable()), true); if (Utilities.isInputFileFormatSelfDescribing(this)) { // if IF is self describing no need to send column info per partition, since its not used anyway. Table tbl = part.getTable(); setProperties(MetaStoreUtils.getSchemaWithoutCols(part.getTPartition().getSd(), part.getTPartition().getSd(), part.getParameters(), tbl.getDbName(), tbl.getTableName(), tbl.getPartitionKeys())); } else { setProperties(part.getMetadataFromPartitionSchema()); } }
public HiveWorkUnit(HiveDataset hiveDataset, Partition partition) { this(hiveDataset); setPartitionName(partition.getName()); setPartitionLocation(partition.getLocation()); setPartitionKeys(partition.getTable().getPartitionKeys()); }
@Override public void clean() throws IOException { // Possible empty directories to clean for this partition (version) Set<Path> possiblyEmptyDirectories = new HashSet<>(); try (AutoReturnableObject<IMetaStoreClient> client = cleanableHiveDataset.getClientPool().getClient()) { Partition partition = hiveDatasetVersion.getPartition(); try { if (!cleanableHiveDataset.isSimulate()) { client.get().dropPartition(partition.getTable().getDbName(), partition.getTable().getTableName(), partition.getValues(), false); log.info("Successfully dropped partition " + partition.getCompleteName()); } else { log.info("Simulating drop partition " + partition.getCompleteName()); } if (cleanableHiveDataset.isShouldDeleteData()) { cleanableHiveDataset.getFsCleanableHelper().clean(hiveDatasetVersion, possiblyEmptyDirectories); } } catch (TException | IOException e) { log.warn(String.format("Failed to completely delete partition %s.", partition.getCompleteName()), e); throw new IOException(e); } } cleanableHiveDataset.getFsCleanableHelper().cleanEmptyDirectories(possiblyEmptyDirectories, cleanableHiveDataset); }
public HivePartitionFileSet(HiveCopyEntityHelper hiveCopyEntityHelper, Partition partition, Properties properties) { super(partition.getCompleteName(), hiveCopyEntityHelper.getDataset()); this.hiveCopyEntityHelper = hiveCopyEntityHelper; this.partition = partition; this.properties = properties; this.existingTargetPartition = Optional.fromNullable(this.hiveCopyEntityHelper.getTargetPartitions().get(this.partition.getValues())); this.eventSubmitter = new EventSubmitter.Builder(this.hiveCopyEntityHelper.getDataset().getMetricContext(), "hive.dataset.copy") .addMetadata("Partition", this.partition.getName()).build(); }
@Test public void testDefaults() throws Exception { DatePartitionHiveVersionFinder versionFinder = new DatePartitionHiveVersionFinder(this.fs, ConfigFactory.empty()); String tableName = "VfTb1"; Table tbl = this.hiveMetastoreTestUtils.createTestAvroTable(dbName, tableName, ImmutableList.of("datepartition")); org.apache.hadoop.hive.metastore.api.Partition tp = this.hiveMetastoreTestUtils.addTestPartition(tbl, ImmutableList.of("2016-01-01-20"), (int) System.currentTimeMillis()); Partition partition = new Partition(new org.apache.hadoop.hive.ql.metadata.Table(tbl), tp); assertThat(partition.getName(), is("datepartition=2016-01-01-20")); TimestampedHiveDatasetVersion dv = versionFinder.getDatasetVersion(partition); Assert.assertEquals(dv.getDateTime(), formatter.parseDateTime("2016/01/01/20")); }
public static String getPartitionInformation(Partition part) { StringBuilder tableInfo = new StringBuilder(DEFAULT_STRINGBUILDER_SIZE); // Table Metadata tableInfo.append(LINE_DELIM).append("# Detailed Partition Information").append(LINE_DELIM); getPartitionMetaDataInformation(tableInfo, part); // Storage information. if (part.getTable().getTableType() != TableType.VIRTUAL_VIEW) { tableInfo.append(LINE_DELIM).append("# Storage Information").append(LINE_DELIM); getStorageDescriptorInfo(tableInfo, part.getTPartition().getSd()); } return tableInfo.toString(); }
void updatePartitionLocation(String dbName, Table table, String partName, Partition part, Path newLocation) throws HiveException, TException { String msg = String.format("ALTER TABLE %s PARTITION (%s) SET LOCATION '%s'", getQualifiedName(table), partName, newLocation.toString()); LOG.info(msg); org.apache.hadoop.hive.ql.metadata.Partition modifiedPart = new org.apache.hadoop.hive.ql.metadata.Partition( new org.apache.hadoop.hive.ql.metadata.Table(table), part); modifiedPart.setLocation(newLocation.toString()); alterPartitionInternal(table, modifiedPart); }
/** * Get a prefix of the given parition's string representation. The second * argument, level, is used for the prefix length. For example, partition * (ds='2010-01-01', hr='00', min='00'), level 1 will return 'ds=2010-01-01', * and level 2 will return 'ds=2010-01-01/hr=00'. * * @param p * partition object * @param level * level for prefix depth * @return prefix of partition's string representation * @throws HiveException */ public static String getPartialName(Partition p, int level) throws HiveException { List<FieldSchema> fields = p.getTable().getPartCols().subList(0, level); List<String> values = p.getValues().subList(0, level); try { return Warehouse.makePartName(fields, values); } catch (MetaException e) { throw new HiveException("Wasn't able to generate name" + " for partial specification"); } }
public static HiveLocationDescriptor forPartition(Partition partition, FileSystem fs, Properties properties) throws IOException { return new HiveLocationDescriptor(partition.getDataLocation(), HiveUtils.getInputFormat(partition.getTPartition().getSd()), fs, properties); }
/** * Determines whether a partition has been archived * * @param p * @return is it archived? */ public static boolean isArchived(Partition p) { return MetaStoreUtils.isArchived(p.getTPartition()); }
/** * Get the update time of a {@link Partition} * * @return the update time if available, 0 otherwise * * {@inheritDoc} * @see HiveUnitUpdateProvider#getUpdateTime(org.apache.hadoop.hive.ql.metadata.Partition) */ @Override public long getUpdateTime(Partition partition) throws UpdateNotFoundException { try { return getUpdateTime(partition.getDataLocation()); } catch (IOException e) { throw new UpdateNotFoundException(String.format("Failed to get update time for %s", partition.getCompleteName()), e); } }
/** * Currently updated the {@link #HIVE_TABLE_AVRO_SCHEMA_URL} location for new hive partitions * @param targetTable, new Table to be registered in hive * @param sourcePartitions, source partitions * @throws IOException */ public static void updatePartitionAttributesIfAvro(Table targetTable, Map<List<String>, Partition> sourcePartitions, HiveCopyEntityHelper hiveHelper) throws IOException { if (isHiveTableAvroType(targetTable)) { for (Map.Entry<List<String>, Partition> partition : sourcePartitions.entrySet()) { updateAvroSchemaURL(partition.getValue().getCompleteName(), partition.getValue().getTPartition().getSd(), hiveHelper); } } }