@Override public RecordWriter getRecordWriter(FragmentContext context, EasyWriter writer) throws IOException { Map<String, String> options = new HashMap<>(); options.put("location", writer.getLocation()); FragmentHandle handle = context.getHandle(); String fragmentId = String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId()); options.put("prefix", fragmentId); options.put("separator", " "); options.put(FileSystem.FS_DEFAULT_NAME_KEY, ((FileSystemConfig) writer.getStorageConfig()).getConnection()); options.put("extension", "json"); options.put("extended", Boolean.toString(context.getOptions().getOption(ExecConstants.JSON_EXTENDED_TYPES))); options.put("uglify", Boolean.toString(context.getOptions().getOption(ExecConstants.JSON_WRITER_UGLIFY))); options.put("skipnulls", Boolean.toString(context.getOptions().getOption(ExecConstants.JSON_WRITER_SKIPNULLFIELDS))); options.put("enableNanInf", Boolean.toString(context.getOptions().getOption(ExecConstants.JSON_WRITER_NAN_INF_NUMBERS_VALIDATOR))); RecordWriter recordWriter = new JsonRecordWriter(writer.getStorageStrategy()); recordWriter.init(options); return recordWriter; }
@Override public RecordWriter getRecordWriter(final FragmentContext context, final EasyWriter writer) throws IOException { final Map<String, String> options = new HashMap<>(); options.put("location", writer.getLocation()); FragmentHandle handle = context.getHandle(); String fragmentId = String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId()); options.put("prefix", fragmentId); options.put("separator", getConfig().getFieldDelimiterAsString()); options.put(FileSystem.FS_DEFAULT_NAME_KEY, ((FileSystemConfig) writer.getStorageConfig()).getConnection()); options.put("extension", getConfig().getExtensions().get(0)); RecordWriter recordWriter = new DrillTextRecordWriter(context.getAllocator(), writer.getStorageStrategy()); recordWriter.init(options); return recordWriter; }
public RecordWriter getRecordWriter(FragmentContext context, ParquetWriter writer) throws IOException, OutOfMemoryException { Map<String, String> options = new HashMap<>(); options.put("location", writer.getLocation()); FragmentHandle handle = context.getHandle(); String fragmentId = String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId()); options.put("prefix", fragmentId); options.put(FileSystem.FS_DEFAULT_NAME_KEY, ((FileSystemConfig) writer.getStorageConfig()).getConnection()); options.put(ExecConstants.PARQUET_BLOCK_SIZE, context.getOptions().getOption(ExecConstants.PARQUET_BLOCK_SIZE).num_val.toString()); options.put(ExecConstants.PARQUET_WRITER_USE_SINGLE_FS_BLOCK, context.getOptions().getOption(ExecConstants.PARQUET_WRITER_USE_SINGLE_FS_BLOCK).bool_val.toString()); options.put(ExecConstants.PARQUET_PAGE_SIZE, context.getOptions().getOption(ExecConstants.PARQUET_PAGE_SIZE).num_val.toString()); options.put(ExecConstants.PARQUET_DICT_PAGE_SIZE, context.getOptions().getOption(ExecConstants.PARQUET_DICT_PAGE_SIZE).num_val.toString()); options.put(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE, context.getOptions().getOption(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE).string_val); options.put(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING, context.getOptions().getOption(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING).bool_val.toString()); options.put(ExecConstants.PARQUET_WRITER_LOGICAL_TYPE_FOR_DECIMALS, context.getOptions().getOption(ExecConstants.PARQUET_WRITER_LOGICAL_TYPE_FOR_DECIMALS).string_val); options.put(ExecConstants.PARQUET_WRITER_USE_PRIMITIVE_TYPES_FOR_DECIMALS, context.getOptions().getOption(ExecConstants.PARQUET_WRITER_USE_PRIMITIVE_TYPES_FOR_DECIMALS).bool_val.toString()); RecordWriter recordWriter = new ParquetRecordWriter(context, writer); recordWriter.init(options); return recordWriter; }
private void sendEmptyBatch(boolean isLast) { BatchSchema schema = incoming.getSchema(); if (schema == null) { // If the incoming batch has no schema (possible when there are no input records), // create an empty schema to avoid NPE. schema = BatchSchema.newBuilder().build(); } FragmentHandle handle = context.getHandle(); for (MinorFragmentEndpoint destination : popConfig.getDestinations()) { AccountingDataTunnel tunnel = context.getDataTunnel(destination.getEndpoint()); FragmentWritableBatch writableBatch = FragmentWritableBatch.getEmptyBatchWithSchema( isLast, handle.getQueryId(), handle.getMajorFragmentId(), handle.getMinorFragmentId(), operator.getOppositeMajorFragmentId(), destination.getId(), schema); stats.startWait(); try { tunnel.sendRecordBatch(writableBatch); } finally { stats.stopWait(); } } stats.addLongStat(Metric.BATCHES_SENT, 1); }
public ExternalSortBatch(ExternalSort popConfig, FragmentContext context, RecordBatch incoming) throws OutOfMemoryException { super(popConfig, context, true); this.incoming = incoming; DrillConfig config = context.getConfig(); Configuration conf = new Configuration(); conf.set(FileSystem.FS_DEFAULT_NAME_KEY, config.getString(ExecConstants.EXTERNAL_SORT_SPILL_FILESYSTEM)); try { this.fs = FileSystem.get(conf); } catch (IOException e) { throw new RuntimeException(e); } SPILL_BATCH_GROUP_SIZE = config.getInt(ExecConstants.EXTERNAL_SORT_SPILL_GROUP_SIZE); SPILL_THRESHOLD = config.getInt(ExecConstants.EXTERNAL_SORT_SPILL_THRESHOLD); dirs = Iterators.cycle(config.getStringList(ExecConstants.EXTERNAL_SORT_SPILL_DIRS)); oAllocator = oContext.getAllocator(); copierAllocator = oAllocator.newChildAllocator(oAllocator.getName() + ":copier", PriorityQueueCopier.INITIAL_ALLOCATION, PriorityQueueCopier.MAX_ALLOCATION); FragmentHandle handle = context.getHandle(); fileName = String.format("%s_majorfragment%s_minorfragment%s_operator%s", QueryIdHelper.getQueryId(handle.getQueryId()), handle.getMajorFragmentId(), handle.getMinorFragmentId(), popConfig.getOperatorId()); }
public OrderedPartitionRecordBatch(OrderedPartitionSender pop, RecordBatch incoming, FragmentContext context) throws OutOfMemoryException { super(pop, context); this.incoming = incoming; this.partitions = pop.getDestinations().size(); this.sendingMajorFragmentWidth = pop.getSendingWidth(); this.recordsToSample = pop.getRecordsToSample(); this.samplingFactor = pop.getSamplingFactor(); this.completionFactor = pop.getCompletionFactor(); DistributedCache cache = null; // Clearly, this code is not used! this.mmap = cache.getMultiMap(MULTI_CACHE_CONFIG); this.tableMap = cache.getMap(SINGLE_CACHE_CONFIG); Preconditions.checkNotNull(tableMap); this.mapKey = String.format("%s_%d", context.getHandle().getQueryId(), context.getHandle().getMajorFragmentId()); this.minorFragmentSampleCount = cache.getCounter(mapKey); SchemaPath outputPath = popConfig.getRef(); MaterializedField outputField = MaterializedField.create(outputPath.getAsNamePart().getName(), Types.required(TypeProtos.MinorType.INT)); this.partitionKeyVector = (IntVector) TypeHelper.getNewVector(outputField, oContext.getAllocator()); }
private boolean updateFragmentStatus(final FragmentStatus fragmentStatus) { final FragmentHandle fragmentHandle = fragmentStatus.getHandle(); final int majorFragmentId = fragmentHandle.getMajorFragmentId(); final int minorFragmentId = fragmentHandle.getMinorFragmentId(); final FragmentData data = fragmentDataMap.get(majorFragmentId).get(minorFragmentId); final FragmentState oldState = data.getState(); final boolean inTerminalState = isTerminal(oldState); final FragmentState currentState = fragmentStatus.getProfile().getState(); if (inTerminalState || (oldState == FragmentState.CANCELLATION_REQUESTED && !isTerminal(currentState))) { // Already in a terminal state, or invalid state transition from CANCELLATION_REQUESTED. This shouldn't happen. logger.warn(String.format("Received status message for fragment %s after fragment was in state %s. New state was %s", QueryIdHelper.getQueryIdentifier(fragmentHandle), oldState, currentState)); return false; } data.setStatus(fragmentStatus); return oldState != currentState; }
public Builder mergeFrom(org.apache.drill.exec.proto.ExecProtos.FragmentHandle other) { if (other == org.apache.drill.exec.proto.ExecProtos.FragmentHandle.getDefaultInstance()) return this; if (other.hasQueryId()) { mergeQueryId(other.getQueryId()); } if (other.hasMajorFragmentId()) { setMajorFragmentId(other.getMajorFragmentId()); } if (other.hasMinorFragmentId()) { setMinorFragmentId(other.getMinorFragmentId()); } if (other.hasParentQueryId()) { mergeParentQueryId(other.getParentQueryId()); } this.mergeUnknownFields(other.getUnknownFields()); return this; }
public static String getFragmentId(final FragmentHandle fragmentHandle) { return fragmentHandle.getMajorFragmentId() + ":" + fragmentHandle.getMinorFragmentId(); }
public static String getQueryIdentifier(final FragmentHandle h) { return getQueryId(h.getQueryId()) + ":" + h.getMajorFragmentId() + ":" + h.getMinorFragmentId(); }
private void sendFinalState() { final FragmentState outcome = fragmentState.get(); if (outcome == FragmentState.FAILED) { final FragmentHandle handle = getContext().getHandle(); final UserException uex = UserException.systemError(deferredException.getAndClear()) .addIdentity(getContext().getEndpoint()) .addContext("Fragment", handle.getMajorFragmentId() + ":" + handle.getMinorFragmentId()) .build(logger); statusReporter.fail(uex); } else { statusReporter.stateChanged(outcome); } statusReporter.close(); }
public static String getFileNameForQueryFragment(FragmentContext context, String location, String tag) { /* * From the context, get the query id, major fragment id, minor fragment id. This will be used as the file name to * which we will dump the incoming buffer data */ ExecProtos.FragmentHandle handle = context.getHandle(); String qid = QueryIdHelper.getQueryId(handle.getQueryId()); int majorFragmentId = handle.getMajorFragmentId(); int minorFragmentId = handle.getMinorFragmentId(); String fileName = String.format("%s//%s_%s_%s_%s", location, qid, majorFragmentId, minorFragmentId, tag); return fileName; }
private Path getPath() { ExecProtos.FragmentHandle handle = context.getHandle(); String qid = QueryIdHelper.getQueryId(handle.getQueryId()); int majorFragmentId = handle.getMajorFragmentId(); int minorFragmentId = handle.getMinorFragmentId(); String fileName = Joiner.on(Path.SEPARATOR).join(getDir(), qid, majorFragmentId, minorFragmentId, oppositeId, bufferIndex); return new Path(fileName); } }
public WriterRecordBatch(Writer writer, RecordBatch incoming, FragmentContext context, RecordWriter recordWriter) throws OutOfMemoryException { super(writer, context, false); this.incoming = incoming; final FragmentHandle handle = context.getHandle(); fragmentUniqueId = String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId()); this.recordWriter = recordWriter; }
public static String getExecutorThreadName(final FragmentHandle fragmentHandle) { return String.format("%s:frag:%s:%s", getQueryId(fragmentHandle.getQueryId()), fragmentHandle.getMajorFragmentId(), fragmentHandle.getMinorFragmentId()); }
@Override public String getFragIdString() { final FragmentHandle handle = getHandle(); final String frag = handle != null ? handle.getMajorFragmentId() + ":" + handle.getMinorFragmentId() : "0:0"; return frag; }
private void addFragment(final FragmentData fragmentData) { final FragmentHandle fragmentHandle = fragmentData.getHandle(); final int majorFragmentId = fragmentHandle.getMajorFragmentId(); final int minorFragmentId = fragmentHandle.getMinorFragmentId(); IntObjectHashMap<FragmentData> minorMap = fragmentDataMap.get(majorFragmentId); if (minorMap == null) { minorMap = new IntObjectHashMap<>(); fragmentDataMap.put(majorFragmentId, minorMap); } minorMap.put(minorFragmentId, fragmentData); fragmentDataSet.add(fragmentData); }
public void writeTo(com.dyuproject.protostuff.Output output, org.apache.drill.exec.proto.ExecProtos.FragmentHandle message) throws java.io.IOException { if(message.hasQueryId()) output.writeObject(1, message.getQueryId(), org.apache.drill.exec.proto.SchemaUserBitShared.QueryId.WRITE, false); if(message.hasMajorFragmentId()) output.writeInt32(2, message.getMajorFragmentId(), false); if(message.hasMinorFragmentId()) output.writeInt32(3, message.getMinorFragmentId(), false); if(message.hasParentQueryId()) output.writeObject(4, message.getParentQueryId(), org.apache.drill.exec.proto.SchemaUserBitShared.QueryId.WRITE, false); } public boolean isInitialized(org.apache.drill.exec.proto.ExecProtos.FragmentHandle message)
public Builder mergeFrom(org.apache.drill.exec.proto.ExecProtos.FragmentHandle other) { if (other == org.apache.drill.exec.proto.ExecProtos.FragmentHandle.getDefaultInstance()) return this; if (other.hasQueryId()) { mergeQueryId(other.getQueryId()); } if (other.hasMajorFragmentId()) { setMajorFragmentId(other.getMajorFragmentId()); } if (other.hasMinorFragmentId()) { setMinorFragmentId(other.getMinorFragmentId()); } if (other.hasParentQueryId()) { mergeParentQueryId(other.getParentQueryId()); } this.mergeUnknownFields(other.getUnknownFields()); return this; }
public void writeTo(com.dyuproject.protostuff.Output output, org.apache.drill.exec.proto.ExecProtos.FragmentHandle message) throws java.io.IOException { if(message.hasQueryId()) output.writeObject(1, message.getQueryId(), org.apache.drill.exec.proto.SchemaUserBitShared.QueryId.WRITE, false); if(message.hasMajorFragmentId()) output.writeInt32(2, message.getMajorFragmentId(), false); if(message.hasMinorFragmentId()) output.writeInt32(3, message.getMinorFragmentId(), false); if(message.hasParentQueryId()) output.writeObject(4, message.getParentQueryId(), org.apache.drill.exec.proto.SchemaUserBitShared.QueryId.WRITE, false); } public boolean isInitialized(org.apache.drill.exec.proto.ExecProtos.FragmentHandle message)