/** * Table has no partitions, just copy data * * @param conf Configuration * @param outputInfo OutputInfo * @throws IOException I/O errors */ private void noPartitionsCopyData(Configuration conf, OutputInfo outputInfo) throws IOException { Preconditions.checkArgument(!outputInfo.hasPartitionInfo()); Path tablePath = new Path(outputInfo.getTableRoot()); Path writePath = new Path(outputInfo.getPartitionPath()); FileSystem tableFs = tablePath.getFileSystem(conf); FileSystem writePathFs = writePath.getFileSystem(conf); if (!tableFs.getUri().equals(writePathFs.getUri())) { LOG.error("Table's root path fs {} is not on same as its partition path fs {}", tableFs.getUri(), writePathFs.getUri()); throw new IllegalStateException("Table's root path fs " + tableFs.getUri() + " is not on same as its partition path fs " + writePathFs.getUri()); } LOG.info("No partitions, copying data from {} to {}", writePath, tablePath); FileSystems.move(tableFs, writePath, writePath, tablePath); tableFs.delete(writePath, true); }
/** * Check that the table's partition info and the user's match. * * @param conf Configuration * @param description HiveInputDescription * @param oti OutputInfo * @param outputConf OutputConf * @throws IOException */ private void checkPartitionInfo(Configuration conf, HiveOutputDescription description, OutputInfo oti, OutputConf outputConf) throws IOException { if (oti.hasPartitionInfo()) { if (!description.hasPartitionValues()) { throw new IOException("table is partitioned but user input isn't"); } if (outputConf.shouldDropPartitionIfExists()) { dropPartitionIfExists(conf, description, oti); } else { checkPartitionDoesntExist(conf, description, oti); } } else { if (description.hasPartitionValues()) { throw new IOException("table is not partitioned but user input is"); } else { checkTableIsEmpty(conf, description, oti); } } }
@Override public void commitJob(JobContext jobContext) throws IOException { baseCommitter.commitJob(jobContext); Configuration conf = jobContext.getConfiguration(); OutputConf outputConf = new OutputConf(conf, profileId); HiveOutputDescription outputDesc = outputConf.readOutputDescription(); OutputInfo outputInfo = outputConf.readOutputTableInfo(); if (outputInfo.hasPartitionInfo()) { registerPartitions(conf, outputDesc, outputInfo); } else { noPartitionsCopyData(conf, outputInfo); } writeSuccessFile(conf); }
@Override public Boolean idempotentTask() throws TException { ThriftHiveMetastore.Iface client = description.metastoreClient(conf); String db = description.getTableDesc().getDatabaseName(); String table = description.getTableDesc().getTableName(); if (oti.hasPartitionInfo()) { Map<String, String> partitionSpec = description.getPartitionValues(); List<String> partitionValues = listOfPartitionValues( partitionSpec, oti.getPartitionInfo()); if (partitionExists(client, db, table, partitionValues)) { LOG.error("Table " + db + ":" + table + " partition " + partitionSpec + " already exists"); return true; } } return false; } };
/** * Drop partition which we will be writing to * @param conf Configuration to use * @param description HiveOutputDescription * @param oti OutputInfo * @return True iff partition was dropped */ private boolean dropPartitionIfExists(Configuration conf, HiveOutputDescription description, OutputInfo oti) throws IOException { ThriftHiveMetastore.Iface client; try { client = description.metastoreClient(conf); } catch (TException e) { throw new IOException(e); } String db = description.getTableDesc().getDatabaseName(); String table = description.getTableDesc().getTableName(); if (oti.hasPartitionInfo()) { Map<String, String> partitionSpec = description.getPartitionValues(); List<String> partitionValues = listOfPartitionValues( partitionSpec, oti.getPartitionInfo()); if (partitionExists(client, db, table, partitionValues)) { LOG.info("Dropping partition {} from table {}:{}", partitionSpec, db, table); return dropPartition(client, db, table, partitionValues); } } return false; }
if (outputInfo.hasPartitionInfo()) { try { partitionPiece = HiveUtils.computePartitionPath( HadoopUtils.setOutputDir(conf, partitionPath); if (outputInfo.hasPartitionInfo()) { outputInfo.setFinalOutputPath(outputInfo.getPartitionPath()); } else {