/** * so to make sure we crate WriteEntity with the right WriteType. This is (at this point) only * for consistency since LockManager (which is the only thing that pays attention to WriteType) * has done it's job before the query ran. */ WriteEntity.WriteType getWriteType(LoadTableDesc tbd, AcidUtils.Operation operation) { if(tbd.getReplace()) { return WriteEntity.WriteType.INSERT_OVERWRITE; } switch (operation) { case DELETE: return WriteEntity.WriteType.DELETE; case UPDATE: return WriteEntity.WriteType.UPDATE; default: return WriteEntity.WriteType.INSERT; } } private boolean isSkewedStoredAsDirs(LoadTableDesc tbd) {
private WriteEntity.WriteType determineWriteType(LoadTableDesc ltd, boolean isNonNativeTable, String dest) { // Don't know the characteristics of non-native tables, // and don't have a rational way to guess, so assume the most // conservative case. if (isNonNativeTable) return WriteEntity.WriteType.INSERT_OVERWRITE; else return (ltd.getReplace() ? WriteEntity.WriteType.INSERT_OVERWRITE : getWriteType(dest)); }
private void updateStats(StatsAggregator statsAggregator, Map<String, String> parameters, String prefix, boolean atomic) throws HiveException { String aggKey = prefix.endsWith(Path.SEPARATOR) ? prefix : prefix + Path.SEPARATOR; for (String statType : StatsSetupConst.statsRequireCompute) { String value = statsAggregator.aggregateStats(aggKey, statType); if (value != null && !value.isEmpty()) { long longValue = Long.parseLong(value); if (work.getLoadTableDesc() != null && !work.getLoadTableDesc().getReplace()) { String originalValue = parameters.get(statType); if (originalValue != null) { longValue += Long.parseLong(originalValue); // todo: invalid + valid = invalid } } parameters.put(statType, String.valueOf(longValue)); } else { if (atomic) { throw new HiveException(ErrorMsg.STATSAGGREGATOR_MISSED_SOMESTATS, statType); } } } }
if (tbd.getPartitionSpec().size() == 0) { dc = new DataContainer(table.getTTable()); db.loadTable(tbd.getSourcePath(), tbd.getTable().getTableName(), tbd.getReplace(), work.isSrcLocal(), isSkewedStoredAsDirs(tbd), work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID, tbd.getTable().getTableName(), tbd.getPartitionSpec(), tbd.getReplace(), dpCtx.getNumDPCols(), isSkewedStoredAsDirs(tbd), db.validatePartitionNameCharacters(partVals); db.loadPartition(tbd.getSourcePath(), tbd.getTable().getTableName(), tbd.getPartitionSpec(), tbd.getReplace(), tbd.getInheritTableSpecs(), isSkewedStoredAsDirs(tbd), work.isSrcLocal(), work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID, hasFollowingStatsTask());
StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.FALSE); } else if (work.getTableSpecs() != null || (work.getLoadTableDesc() != null && work.getLoadTableDesc().getReplace()) || (work.getLoadFileDesc() != null && !work.getLoadFileDesc() .getDestinationCreateTable().isEmpty())) { StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.FALSE); } else if (work.getTableSpecs() != null || (work.getLoadTableDesc() != null && work.getLoadTableDesc().getReplace()) || (work.getLoadFileDesc() != null && !work.getLoadFileDesc() .getDestinationCreateTable().isEmpty())) {
if (mvWork.getLoadTableWork() != null) { statsWork = new BasicStatsWork(mvWork.getLoadTableWork()); truncate = mvWork.getLoadTableWork().getReplace(); String tableName = mvWork.getLoadTableWork().getTable().getTableName(); try {
private WriteEntity.WriteType determineWriteType(LoadTableDesc ltd, boolean isNonNativeTable) { // Don't know the characteristics of non-native tables, // and don't have a rational way to guess, so assume the most // conservative case. if (isNonNativeTable) return WriteEntity.WriteType.INSERT_OVERWRITE; else return (ltd.getReplace() ? WriteEntity.WriteType.INSERT_OVERWRITE : WriteEntity.WriteType.INSERT); }
private void updateStats(StatsAggregator statsAggregator, Map<String, String> parameters, String prefix, int maxPrefixLength, boolean atomic) throws HiveException { String aggKey = Utilities.getHashedStatsPrefix(prefix, maxPrefixLength); for (String statType : StatsSetupConst.statsRequireCompute) { String value = statsAggregator.aggregateStats(aggKey, statType); if (value != null && !value.isEmpty()) { long longValue = Long.parseLong(value); if (work.getLoadTableDesc() != null && !work.getLoadTableDesc().getReplace()) { String originalValue = parameters.get(statType); if (originalValue != null && !originalValue.equals("-1")) { longValue += Long.parseLong(originalValue); // todo: invalid + valid = invalid } } parameters.put(statType, String.valueOf(longValue)); } else { if (atomic) { throw new HiveException(ErrorMsg.STATSAGGREGATOR_MISSED_SOMESTATS, statType); } } } statsAggregator.cleanUp(aggKey); }
dc = new DataContainer(table.getTTable()); db.loadTable(new Path(tbd.getSourceDir()), tbd.getTable() .getTableName(), tbd.getReplace(), tbd.getHoldDDLTime()); if (work.getOutputs() != null) { work.getOutputs().add(new WriteEntity(table, true)); tbd.getTable().getTableName(), tbd.getPartitionSpec(), tbd.getReplace(), dpCtx.getNumDPCols(), tbd.getHoldDDLTime()); tbd.getPartitionSpec(), tbd.getReplace(), tbd.getHoldDDLTime()); Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false); dc = new DataContainer(table.getTTable(), partn.getTPartition());
dc = new DataContainer(table.getTTable()); db.loadTable(tbd.getSourcePath(), tbd.getTable() .getTableName(), tbd.getReplace(), tbd.getHoldDDLTime(), work.isSrcLocal(), isSkewedStoredAsDirs(tbd), work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID); if (work.getOutputs() != null) { work.getOutputs().add(new WriteEntity(table, (tbd.getReplace() ? WriteEntity.WriteType.INSERT_OVERWRITE : WriteEntity.WriteType.INSERT))); tbd.getTable().getTableName(), tbd.getPartitionSpec(), tbd.getReplace(), dpCtx.getNumDPCols(), tbd.getHoldDDLTime(), (tbd.getReplace() ? WriteEntity.WriteType.INSERT_OVERWRITE : WriteEntity.WriteType.INSERT)); if (work.getOutputs() != null) { db.validatePartitionNameCharacters(partVals); db.loadPartition(tbd.getSourcePath(), tbd.getTable().getTableName(), tbd.getPartitionSpec(), tbd.getReplace(), tbd.getHoldDDLTime(), tbd.getInheritTableSpecs(), isSkewedStoredAsDirs(tbd), work.isSrcLocal(), work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID); (tbd.getReplace() ? WriteEntity.WriteType.INSERT_OVERWRITE : WriteEntity.WriteType.INSERT)));
if (!outputs.add(new WriteEntity(dest_part, (ltd.getReplace() ? WriteEntity.WriteType.INSERT_OVERWRITE : WriteEntity.WriteType.INSERT)))) {