@Override public void process(Object row, int tag) throws HiveException { // Ignore the tag passed in, which should be 0, not what we want htsOperator.process(row, conf.getTag()); }
@Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); ObjectInspector[] inputOIs = new ObjectInspector[conf.getTagLength()]; byte tag = conf.getTag(); inputOIs[tag] = inputObjInspectors[0]; conf.setTagOrder(new Byte[]{ tag }); numReplication = (short) hconf.getInt(MAPRED_FILE_REPLICATION, DEFAULT_REPLICATION); htsOperator.setConf(conf); htsOperator.initialize(hconf, inputOIs); }
@Override protected void initializeOp(Configuration hconf) throws HiveException { super.initializeOp(hconf); ObjectInspector[] inputOIs = new ObjectInspector[conf.getTagLength()]; byte tag = conf.getTag(); inputOIs[tag] = inputObjInspectors[0]; conf.setTagOrder(new Byte[]{ tag }); int dfsMaxReplication = hconf.getInt(DFS_REPLICATION_MAX, minReplication); // minReplication value should not cross the value of dfs.replication.max minReplication = Math.min(minReplication, dfsMaxReplication); htsOperator.setConf(conf); htsOperator.initialize(hconf, inputOIs); }
@Override public void process(Object row, int tag) throws HiveException { // Ignore the tag passed in, which should be 0, not what we want htsOperator.process(row, conf.getTag()); }
private boolean validateSparkHashTableSinkOperator(SparkHashTableSinkOperator op) { SparkHashTableSinkDesc desc = op.getConf(); byte tag = desc.getTag(); // it's essentially a MapJoinDesc List<ExprNodeDesc> filterExprs = desc.getFilters().get(tag); List<ExprNodeDesc> keyExprs = desc.getKeys().get(tag); List<ExprNodeDesc> valueExprs = desc.getExprs().get(tag); return validateExprNodeDesc( filterExprs, "Filter", VectorExpressionDescriptor.Mode.FILTER, /* allowComplex */ true) && validateExprNodeDesc(keyExprs, "Key") && validateExprNodeDesc(valueExprs, "Value"); }
private boolean validateSparkHashTableSinkOperator(SparkHashTableSinkOperator op) { SparkHashTableSinkDesc desc = op.getConf(); byte tag = desc.getTag(); // it's essentially a MapJoinDesc List<ExprNodeDesc> filterExprs = desc.getFilters().get(tag); List<ExprNodeDesc> keyExprs = desc.getKeys().get(tag); List<ExprNodeDesc> valueExprs = desc.getExprs().get(tag); return validateExprNodeDesc(filterExprs, "Filter", VectorExpressionDescriptor.Mode.FILTER) && validateExprNodeDesc(keyExprs, "Key") && validateExprNodeDesc(valueExprs, "Value"); }
@Override public void closeOp(boolean abort) throws HiveException { try { MapJoinPersistableTableContainer[] mapJoinTables = htsOperator.mapJoinTables; byte tag = conf.getTag(); if (mapJoinTables == null || mapJoinTables.length < tag || mapJoinTables[tag] == null) { LOG.debug("mapJoinTable is null"); } else if (abort) { if (LOG.isDebugEnabled()) { LOG.debug("Aborting, skip dumping side-table for tag: " + tag); } } else { String method = PerfLogger.SPARK_FLUSH_HASHTABLE + getName(); perfLogger.PerfLogBegin(CLASS_NAME, method); try { flushToFile(mapJoinTables[tag], tag); } finally { perfLogger.PerfLogEnd(CLASS_NAME, method); } } super.closeOp(abort); } catch (HiveException e) { throw e; } catch (Exception e) { throw new HiveException(e); } }
@Override public void closeOp(boolean abort) throws HiveException { try { MapJoinPersistableTableContainer[] mapJoinTables = htsOperator.mapJoinTables; byte tag = conf.getTag(); if (mapJoinTables == null || mapJoinTables.length < tag || mapJoinTables[tag] == null) { LOG.debug("mapJoinTable is null"); } else if (abort) { if (LOG.isDebugEnabled()) { LOG.debug("Aborting, skip dumping side-table for tag: " + tag); } } else { String method = PerfLogger.SPARK_FLUSH_HASHTABLE + getName(); perfLogger.PerfLogBegin(CLASS_NAME, method); try { flushToFile(mapJoinTables[tag], tag); } finally { perfLogger.PerfLogEnd(CLASS_NAME, method); } } super.closeOp(abort); } catch (HiveException e) { throw e; } catch (Exception e) { throw new HiveException(e); } }