@Override public RecordWriter getRecordWriter(FragmentContext context, EasyWriter writer) throws IOException { Map<String, String> options = new HashMap<>(); options.put("location", writer.getLocation()); FragmentHandle handle = context.getHandle(); String fragmentId = String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId()); options.put("prefix", fragmentId); options.put("separator", " "); options.put(FileSystem.FS_DEFAULT_NAME_KEY, ((FileSystemConfig) writer.getStorageConfig()).getConnection()); options.put("extension", "json"); options.put("extended", Boolean.toString(context.getOptions().getOption(ExecConstants.JSON_EXTENDED_TYPES))); options.put("uglify", Boolean.toString(context.getOptions().getOption(ExecConstants.JSON_WRITER_UGLIFY))); options.put("skipnulls", Boolean.toString(context.getOptions().getOption(ExecConstants.JSON_WRITER_SKIPNULLFIELDS))); options.put("enableNanInf", Boolean.toString(context.getOptions().getOption(ExecConstants.JSON_WRITER_NAN_INF_NUMBERS_VALIDATOR))); RecordWriter recordWriter = new JsonRecordWriter(writer.getStorageStrategy()); recordWriter.init(options); return recordWriter; }
@Override public RecordWriter getRecordWriter(final FragmentContext context, final EasyWriter writer) throws IOException { final Map<String, String> options = new HashMap<>(); options.put("location", writer.getLocation()); FragmentHandle handle = context.getHandle(); String fragmentId = String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId()); options.put("prefix", fragmentId); options.put("separator", getConfig().getFieldDelimiterAsString()); options.put(FileSystem.FS_DEFAULT_NAME_KEY, ((FileSystemConfig) writer.getStorageConfig()).getConnection()); options.put("extension", getConfig().getExtensions().get(0)); RecordWriter recordWriter = new DrillTextRecordWriter(context.getAllocator(), writer.getStorageStrategy()); recordWriter.init(options); return recordWriter; }
public WriterRecordBatch(Writer writer, RecordBatch incoming, FragmentContext context, RecordWriter recordWriter) throws OutOfMemoryException { super(writer, context, false); this.incoming = incoming; final FragmentHandle handle = context.getHandle(); fragmentUniqueId = String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId()); this.recordWriter = recordWriter; }
public ExternalSortBatch(ExternalSort popConfig, FragmentContext context, RecordBatch incoming) throws OutOfMemoryException { super(popConfig, context, true); this.incoming = incoming; DrillConfig config = context.getConfig(); Configuration conf = new Configuration(); conf.set(FileSystem.FS_DEFAULT_NAME_KEY, config.getString(ExecConstants.EXTERNAL_SORT_SPILL_FILESYSTEM)); try { this.fs = FileSystem.get(conf); } catch (IOException e) { throw new RuntimeException(e); } SPILL_BATCH_GROUP_SIZE = config.getInt(ExecConstants.EXTERNAL_SORT_SPILL_GROUP_SIZE); SPILL_THRESHOLD = config.getInt(ExecConstants.EXTERNAL_SORT_SPILL_THRESHOLD); dirs = Iterators.cycle(config.getStringList(ExecConstants.EXTERNAL_SORT_SPILL_DIRS)); oAllocator = oContext.getAllocator(); copierAllocator = oAllocator.newChildAllocator(oAllocator.getName() + ":copier", PriorityQueueCopier.INITIAL_ALLOCATION, PriorityQueueCopier.MAX_ALLOCATION); FragmentHandle handle = context.getHandle(); fileName = String.format("%s_majorfragment%s_minorfragment%s_operator%s", QueryIdHelper.getQueryId(handle.getQueryId()), handle.getMajorFragmentId(), handle.getMinorFragmentId(), popConfig.getOperatorId()); }
private boolean updateFragmentStatus(final FragmentStatus fragmentStatus) { final FragmentHandle fragmentHandle = fragmentStatus.getHandle(); final int majorFragmentId = fragmentHandle.getMajorFragmentId(); final int minorFragmentId = fragmentHandle.getMinorFragmentId(); final FragmentData data = fragmentDataMap.get(majorFragmentId).get(minorFragmentId); final FragmentState oldState = data.getState(); final boolean inTerminalState = isTerminal(oldState); final FragmentState currentState = fragmentStatus.getProfile().getState(); if (inTerminalState || (oldState == FragmentState.CANCELLATION_REQUESTED && !isTerminal(currentState))) { // Already in a terminal state, or invalid state transition from CANCELLATION_REQUESTED. This shouldn't happen. logger.warn(String.format("Received status message for fragment %s after fragment was in state %s. New state was %s", QueryIdHelper.getQueryIdentifier(fragmentHandle), oldState, currentState)); return false; } data.setStatus(fragmentStatus); return oldState != currentState; }
private FragmentStatus getStatus(final FragmentState state, final UserException ex) { final FragmentStatus.Builder status = FragmentStatus.newBuilder(); final MinorFragmentProfile.Builder b = MinorFragmentProfile.newBuilder(); context.getStats().addMetricsToStatus(b); b.setState(state); if (ex != null) { final boolean verbose = context.getOptions().getOption(ExecConstants.ENABLE_VERBOSE_ERRORS_KEY).bool_val; b.setError(ex.getOrCreatePBError(verbose)); } status.setHandle(context.getHandle()); b.setMemoryUsed(context.getAllocator().getAllocatedMemory()); b.setMinorFragmentId(context.getHandle().getMinorFragmentId()); status.setProfile(b); return status.build(); }
public Builder mergeFrom(org.apache.drill.exec.proto.ExecProtos.FragmentHandle other) { if (other == org.apache.drill.exec.proto.ExecProtos.FragmentHandle.getDefaultInstance()) return this; if (other.hasQueryId()) { mergeQueryId(other.getQueryId()); } if (other.hasMajorFragmentId()) { setMajorFragmentId(other.getMajorFragmentId()); } if (other.hasMinorFragmentId()) { setMinorFragmentId(other.getMinorFragmentId()); } if (other.hasParentQueryId()) { mergeParentQueryId(other.getParentQueryId()); } this.mergeUnknownFields(other.getUnknownFields()); return this; }
@Override public void receivingFragmentFinished(FragmentHandle handle) { final int id = handle.getMinorFragmentId(); if (remainingReceivers.compareAndSet(id, 0, 1)) { if (partitioner == null) { terminations.add(id); } else { partitioner.getOutgoingBatches(id).terminate(); } int remaining = remaingReceiverCount.decrementAndGet(); if (remaining == 0) { done = true; } } }
private void sendFinalState() { final FragmentState outcome = fragmentState.get(); if (outcome == FragmentState.FAILED) { final FragmentHandle handle = getContext().getHandle(); final UserException uex = UserException.systemError(deferredException.getAndClear()) .addIdentity(getContext().getEndpoint()) .addContext("Fragment", handle.getMajorFragmentId() + ":" + handle.getMinorFragmentId()) .build(logger); statusReporter.fail(uex); } else { statusReporter.stateChanged(outcome); } statusReporter.close(); }
public static String getFragmentId(final FragmentHandle fragmentHandle) { return fragmentHandle.getMajorFragmentId() + ":" + fragmentHandle.getMinorFragmentId(); }
public static String getQueryIdentifier(final FragmentHandle h) { return getQueryId(h.getQueryId()) + ":" + h.getMajorFragmentId() + ":" + h.getMinorFragmentId(); }
public FragmentData(final FragmentHandle handle, final DrillbitEndpoint endpoint, final boolean isLocal) { this.endpoint = endpoint; this.isLocal = isLocal; final MinorFragmentProfile f = MinorFragmentProfile.newBuilder() .setState(FragmentState.SENDING) .setMinorFragmentId(handle.getMinorFragmentId()) .setEndpoint(endpoint) .build(); status = FragmentStatus.newBuilder() .setHandle(handle) .setProfile(f) .build(); }
private Path getPath() { ExecProtos.FragmentHandle handle = context.getHandle(); String qid = QueryIdHelper.getQueryId(handle.getQueryId()); int majorFragmentId = handle.getMajorFragmentId(); int minorFragmentId = handle.getMinorFragmentId(); String fileName = Joiner.on(Path.SEPARATOR).join(getDir(), qid, majorFragmentId, minorFragmentId, oppositeId, bufferIndex); return new Path(fileName); } }
public static String getFileNameForQueryFragment(FragmentContext context, String location, String tag) { /* * From the context, get the query id, major fragment id, minor fragment id. This will be used as the file name to * which we will dump the incoming buffer data */ ExecProtos.FragmentHandle handle = context.getHandle(); String qid = QueryIdHelper.getQueryId(handle.getQueryId()); int majorFragmentId = handle.getMajorFragmentId(); int minorFragmentId = handle.getMinorFragmentId(); String fileName = String.format("%s//%s_%s_%s_%s", location, qid, majorFragmentId, minorFragmentId, tag); return fileName; }
@Override public String getFragIdString() { final FragmentHandle handle = getHandle(); final String frag = handle != null ? handle.getMajorFragmentId() + ":" + handle.getMinorFragmentId() : "0:0"; return frag; }
private void addFragment(final FragmentData fragmentData) { final FragmentHandle fragmentHandle = fragmentData.getHandle(); final int majorFragmentId = fragmentHandle.getMajorFragmentId(); final int minorFragmentId = fragmentHandle.getMinorFragmentId(); IntObjectHashMap<FragmentData> minorMap = fragmentDataMap.get(majorFragmentId); if (minorMap == null) { minorMap = new IntObjectHashMap<>(); fragmentDataMap.put(majorFragmentId, minorMap); } minorMap.put(minorFragmentId, fragmentData); fragmentDataSet.add(fragmentData); }
public static String getExecutorThreadName(final FragmentHandle fragmentHandle) { return String.format("%s:frag:%s:%s", getQueryId(fragmentHandle.getQueryId()), fragmentHandle.getMajorFragmentId(), fragmentHandle.getMinorFragmentId()); }
public void writeTo(com.dyuproject.protostuff.Output output, org.apache.drill.exec.proto.ExecProtos.FragmentHandle message) throws java.io.IOException { if(message.hasQueryId()) output.writeObject(1, message.getQueryId(), org.apache.drill.exec.proto.SchemaUserBitShared.QueryId.WRITE, false); if(message.hasMajorFragmentId()) output.writeInt32(2, message.getMajorFragmentId(), false); if(message.hasMinorFragmentId()) output.writeInt32(3, message.getMinorFragmentId(), false); if(message.hasParentQueryId()) output.writeObject(4, message.getParentQueryId(), org.apache.drill.exec.proto.SchemaUserBitShared.QueryId.WRITE, false); } public boolean isInitialized(org.apache.drill.exec.proto.ExecProtos.FragmentHandle message)
public Builder mergeFrom(org.apache.drill.exec.proto.ExecProtos.FragmentHandle other) { if (other == org.apache.drill.exec.proto.ExecProtos.FragmentHandle.getDefaultInstance()) return this; if (other.hasQueryId()) { mergeQueryId(other.getQueryId()); } if (other.hasMajorFragmentId()) { setMajorFragmentId(other.getMajorFragmentId()); } if (other.hasMinorFragmentId()) { setMinorFragmentId(other.getMinorFragmentId()); } if (other.hasParentQueryId()) { mergeParentQueryId(other.getParentQueryId()); } this.mergeUnknownFields(other.getUnknownFields()); return this; }
public void writeTo(com.dyuproject.protostuff.Output output, org.apache.drill.exec.proto.ExecProtos.FragmentHandle message) throws java.io.IOException { if(message.hasQueryId()) output.writeObject(1, message.getQueryId(), org.apache.drill.exec.proto.SchemaUserBitShared.QueryId.WRITE, false); if(message.hasMajorFragmentId()) output.writeInt32(2, message.getMajorFragmentId(), false); if(message.hasMinorFragmentId()) output.writeInt32(3, message.getMinorFragmentId(), false); if(message.hasParentQueryId()) output.writeObject(4, message.getParentQueryId(), org.apache.drill.exec.proto.SchemaUserBitShared.QueryId.WRITE, false); } public boolean isInitialized(org.apache.drill.exec.proto.ExecProtos.FragmentHandle message)