OutputDescriptor.create(MROutput.class.getName()) .setUserPayload(TezUtils.createUserPayloadFromConf(conf)), null, null));
private void verifyOutputSpecAndProto(OutputSpec outputSpec, IOSpecProto outputSpecProto) { assertEquals(outputSpec.getPhysicalEdgeCount(), outputSpecProto.getPhysicalEdgeCount()); assertEquals(outputSpec.getDestinationVertexName(), outputSpecProto.getConnectedVertexName()); assertEquals(outputSpec.getOutputDescriptor().getClassName(), outputSpecProto.getIoDescriptor().getClassName()); assertEquals(outputSpec.getOutputDescriptor().getUserPayload().getPayload(), outputSpecProto.getIoDescriptor().getUserPayload().getUserPayload().asReadOnlyByteBuffer()); } }
OutputDescriptor.create(outputClassName).setUserPayload(createUserPayload()), (doCommit ? OutputCommitterDescriptor.create( MROutputCommitter.class.getName()) : null), null); if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT, TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) { ds.getOutputDescriptor().setHistoryText(TezUtils.convertToHistoryText(conf));
private static OutputDescriptor convertOutputDescriptorFromProto( EntityDescriptorProto proto) { String className = proto.getClassName(); UserPayload payload = convertPayloadFromProto(proto); OutputDescriptor od = OutputDescriptor.create(className); setUserPayload(od, payload); return od; }
public String toString() { return "{ destinationVertexName=" + destinationVertexName + ", physicalEdgeCount=" + physicalEdgeCount + ", outputClassName=" + outputDescriptor.getClassName() + " }"; } }
public static Map<String,Object> convertEdgeProperty( EdgeProperty edge) { Map<String, Object> jsonDescriptor = new HashMap<String, Object>(); jsonDescriptor.put(DATA_MOVEMENT_TYPE_KEY, edge.getDataMovementType().name()); jsonDescriptor.put(DATA_SOURCE_TYPE_KEY, edge.getDataSourceType().name()); jsonDescriptor.put(SCHEDULING_TYPE_KEY, edge.getSchedulingType().name()); jsonDescriptor.put(EDGE_SOURCE_CLASS_KEY, edge.getEdgeSource().getClassName()); jsonDescriptor.put(EDGE_DESTINATION_CLASS_KEY, edge.getEdgeDestination().getClassName()); String history = edge.getEdgeSource().getHistoryText(); if (history != null) { jsonDescriptor.put(OUTPUT_USER_PAYLOAD_AS_TEXT, history); } history = edge.getEdgeDestination().getHistoryText(); if (history != null) { jsonDescriptor.put(INPUT_USER_PAYLOAD_AS_TEXT, history); } EdgeManagerPluginDescriptor descriptor = edge.getEdgeManagerDescriptor(); if (descriptor != null) { jsonDescriptor.put(EDGE_MANAGER_CLASS_KEY, descriptor.getClassName()); if (descriptor.getHistoryText() != null && !descriptor.getHistoryText().isEmpty()) { jsonDescriptor.put(USER_PAYLOAD_AS_TEXT, descriptor.getHistoryText()); } } return jsonDescriptor; }
@Override public UserPayload getOutputUserPayload() { return output.getIODescriptor().getUserPayload(); }
/** * Modify the EdgeProperty to set the history text if available * @param edgeConfig Edge config * @param edgeProperty Edge property to be be modified */ static void setEdgePropertyHistoryText(HadoopKeyValuesBasedBaseEdgeConfig edgeConfig, EdgeProperty edgeProperty) { String inputHistoryText = edgeConfig.getInputHistoryText(); if (inputHistoryText != null) { edgeProperty.getEdgeDestination().setHistoryText(inputHistoryText); } String outputHistoryText = edgeConfig.getOutputHistoryText(); if (outputHistoryText != null) { edgeProperty.getEdgeSource().setHistoryText(outputHistoryText); } }
@Override public void readFields(DataInput in) throws IOException { destinationVertexName = StringInterner.weakIntern(in.readUTF()); physicalEdgeCount = in.readInt(); outputDescriptor = new OutputDescriptor(); outputDescriptor.readFields(in); }
public static OutputDescriptor create(String outputClassName) { return new OutputDescriptor(outputClassName); } }
OutputDescriptor od = OutputDescriptor.create(MROutputLegacy.class.getName()) .setUserPayload(vertexUserPayload); if (stageConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT, TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) { od.setHistoryText(TezUtils.convertToHistoryText(stageConf));
public static OutputDescriptor convertOutputDescriptorFromDAGPlan( TezEntityDescriptorProto proto) { String className = proto.getClassName(); UserPayload payload = convertTezUserPayloadFromDAGPlan(proto); OutputDescriptor od = OutputDescriptor.create(className); setUserPayload(od, payload); return od; }
@Override public String toString() { return "{ " + dataMovementType + " : " + inputDescriptor.getClassName() + " >> " + dataSourceType + " >> " + outputDescriptor.getClassName() + " >> " + (edgeManagerDescriptor == null ? "NullEdgeManager" : edgeManagerDescriptor.getClassName()) + " }"; }
private OutputContext createOutputContext(OutputSpec outputSpec, int outputIndex) { OutputContext outputContext = new TezOutputContextImpl(tezConf, localDirs, appAttemptNumber, tezUmbilical, taskSpec.getDAGName(), taskSpec.getVertexName(), outputSpec.getDestinationVertexName(), taskSpec.getVertexParallelism(), taskSpec.getTaskAttemptID(), outputIndex, outputSpec.getOutputDescriptor().getUserPayload(), this, serviceConsumerMetadata, envMap, initialMemoryDistributor, outputSpec.getOutputDescriptor(), objectRegistry, ExecutionContext, memAvailable, sharedExecutor); return outputContext; }
MROutputLegacy.createConfigBuilder(finalReduceConf, TextOutputFormat.class, outputPath) .build()); finalReduceVertex.getDataSinks().get(0).getOutputDescriptor().setHistoryText( TezUtils.convertToHistoryText("HDFS Output " + outputPath, finalReduceConf)); vertices.add(finalReduceVertex);
OutputDescriptor.create(MROutput.class.getName()) .setUserPayload(TezUtils.createUserPayloadFromConf(conf)), null, null));
v2.addDataSource(INPUT, dataSourceDescriptor); OutputDescriptor outputDescriptor = OutputDescriptor.create(FakeOutput.class.getName()); OutputCommitterDescriptor outputCommitterDescriptor = OutputCommitterDescriptor.create(FakeOutputCommitter.class.getName());
private LogicalOutput createOutput(OutputSpec outputSpec, OutputContext outputContext) throws TezException { OutputDescriptor outputDesc = outputSpec.getOutputDescriptor(); Output output = ReflectionUtils.createClazzInstance(outputDesc.getClassName(), new Class[]{OutputContext.class, Integer.TYPE}, new Object[]{outputContext, outputSpec.getPhysicalEdgeCount()}); if (!(output instanceof LogicalOutput)) { throw new TezUncheckedException(output.getClass().getName() + " is not a sub-type of LogicalOutput." + " Only LogicalOutput sub-types supported by LogicalIOProcessor."); } return (LogicalOutput) output; }
UserPayload output1Payload = UserPayload.create(output1Bb); OutputDescriptor od1 = OutputDescriptor.create("output1ClassName").setUserPayload(output1Payload); OutputSpec outputSpec1 = new OutputSpec("destVertexName1", od1, 55); OutputSpec outputSpec2 = new OutputSpec("destVertexName2", od1, 66);
public List<TezEvent> generateEmptyEventsForAttempt(TezTaskAttemptID attempt) throws Exception { if (!edgeProperty.getEdgeSource().getClassName().startsWith("org.apache.tez")) { throw new TezException("Only org.apache.tez outputs are allowed for max percent failure feature. Disallowed Output: " + edgeProperty.getEdgeSource().getClassName()); } List<Event> events = new ArrayList<>(); Deflater deflater = TezCommonUtils.newBestCompressionDeflater(); try { ShuffleUtils.generateEventsForNonStartedOutput(events, edgeManager.getNumDestinationConsumerTasks(attempt.getTaskID().getId()), null, false, true, deflater); } catch (Exception e) { throw new TezException(e); } EventMetaData sourceInfo = new EventMetaData(EventMetaData.EventProducerConsumerType.INPUT, sourceVertex.getName(), getDestinationVertexName(), attempt); List<TezEvent> tezEvents = new ArrayList<>(events.size()); for (Event e : events) { TezEvent tezEvent = new TezEvent(e, sourceInfo); tezEvents.add(tezEvent); } return tezEvents; }