blockedMonitor != null, builder.build(), rawInputDataSize.convertToMostSuccinctDataSize(), rawInputPositions, rawInputReadTime, processedInputDataSize.convertToMostSuccinctDataSize(), processedInputPositions, outputDataSize.convertToMostSuccinctDataSize(), outputPositions, succinctBytes(physicalWrittenDataSize),
private static DataSize writeRcFileColumnOld(File outputFile, Format format, Compression compression, Type type, Iterator<?> values) throws Exception { ObjectInspector columnObjectInspector = getJavaObjectInspector(type); RecordWriter recordWriter = createRcFileWriterOld(outputFile, compression, columnObjectInspector); SettableStructObjectInspector objectInspector = createSettableStructObjectInspector("test", columnObjectInspector); Object row = objectInspector.create(); List<StructField> fields = ImmutableList.copyOf(objectInspector.getAllStructFieldRefs()); @SuppressWarnings("deprecation") Serializer serializer = format.createSerializer(); Properties tableProperties = new Properties(); tableProperties.setProperty("columns", "test"); tableProperties.setProperty("columns.types", objectInspector.getTypeName()); serializer.initialize(new JobConf(false), tableProperties); while (values.hasNext()) { Object value = values.next(); value = preprocessWriteValueOld(type, value); objectInspector.setStructFieldData(row, fields.get(0), value); Writable record = serializer.serialize(row, objectInspector); recordWriter.write(record); } recordWriter.close(false); return new DataSize(outputFile.length(), BYTE).convertToMostSuccinctDataSize(); }
public static DataSize succinctDataSize(double size, Unit unit) { return new DataSize(size, unit).convertToMostSuccinctDataSize(); }
public synchronized ListenableFuture<?> reserveMemory(long bytes) { checkArgument(bytes >= 0, "bytes is negative"); if (reserved + bytes > maxMemory) { throw exceededLocalLimit(new DataSize(maxMemory, DataSize.Unit.BYTE).convertToMostSuccinctDataSize()); } ListenableFuture<?> future = memoryPool.reserve(queryId, bytes); reserved += bytes; return future; }
public void addCopyShardDataRate(DataSize size, Duration duration) { DataSize rate = dataRate(size, duration).convertToMostSuccinctDataSize(); copyToBackupBytesPerSecond.add(Math.round(rate.toBytes())); copyToBackupShardSizeBytes.add(size.toBytes()); copyToBackupTimeInMilliSeconds.add(duration.toMillis()); }
@Test(dataProvider = "conversions") public void testConvertToMostSuccinctDataSize(DataSize.Unit unit, DataSize.Unit toUnit, double factor) { DataSize size = new DataSize(factor, toUnit); DataSize actual = size.convertToMostSuccinctDataSize(); assertThat(actual).isEqualTo(new DataSize(1, unit)); assertThat(actual.getValue(unit)).isCloseTo(1.0, offset(0.001)); assertThat(actual.getUnit()).isEqualTo(unit); }
public void addCopyShardDataRate(DataSize size, Duration duration) { DataSize rate = dataRate(size, duration).convertToMostSuccinctDataSize(); copyToBackupBytesPerSecond.add(Math.round(rate.toBytes())); copyToBackupShardSizeBytes.add(size.toBytes()); copyToBackupTimeInMilliSeconds.add(duration.toMillis()); }
queuedTime.convertToMostSuccinctTimeUnit(), elapsedTime.convertToMostSuccinctTimeUnit(), new DataSize(memoryReservation.get(), BYTE).convertToMostSuccinctDataSize(), new DataSize(systemMemoryReservation.get(), BYTE).convertToMostSuccinctDataSize(), new Duration(totalScheduledTime, NANOSECONDS).convertToMostSuccinctTimeUnit(), new Duration(totalCpuTime, NANOSECONDS).convertToMostSuccinctTimeUnit(), blockedMonitor != null, builder.build(), rawInputDataSize.convertToMostSuccinctDataSize(), rawInputPositions, rawInputReadTime, processedInputDataSize.convertToMostSuccinctDataSize(), processedInputPositions, outputDataSize.convertToMostSuccinctDataSize(), outputPositions, ImmutableList.copyOf(transform(operatorContexts, OperatorContext::getOperatorStats)));
new Duration(addInputCpuNanos.get(), NANOSECONDS).convertToMostSuccinctTimeUnit(), new Duration(addInputUserNanos.get(), NANOSECONDS).convertToMostSuccinctTimeUnit(), new DataSize(inputDataSize.getTotalCount(), BYTE).convertToMostSuccinctDataSize(), inputPositions.getTotalCount(), new Duration(getOutputCpuNanos.get(), NANOSECONDS).convertToMostSuccinctTimeUnit(), new Duration(getOutputUserNanos.get(), NANOSECONDS).convertToMostSuccinctTimeUnit(), new DataSize(outputDataSize.getTotalCount(), BYTE).convertToMostSuccinctDataSize(), outputPositions.getTotalCount(), new Duration(finishUserNanos.get(), NANOSECONDS).convertToMostSuccinctTimeUnit(), new DataSize(memoryReservation.get(), BYTE).convertToMostSuccinctDataSize(), new DataSize(systemMemoryContext.getReservedBytes(), BYTE).convertToMostSuccinctDataSize(), memoryFuture.get().isDone() ? Optional.empty() : Optional.of(WAITING_FOR_MEMORY), info);
blockedMonitor != null, builder.build(), rawInputDataSize.convertToMostSuccinctDataSize(), rawInputPositions, rawInputReadTime, processedInputDataSize.convertToMostSuccinctDataSize(), processedInputPositions, outputDataSize.convertToMostSuccinctDataSize(), outputPositions, succinctBytes(physicalWrittenDataSize),
blockedMonitor != null, builder.build(), physicalInputDataSize.convertToMostSuccinctDataSize(), physicalInputPositions, physicalInputReadTime, internalNetworkInputDataSize.convertToMostSuccinctDataSize(), internalNetworkInputPositions, internalNetworkInputReadTime, rawInputDataSize.convertToMostSuccinctDataSize(), rawInputPositions, rawInputReadTime, processedInputDataSize.convertToMostSuccinctDataSize(), processedInputPositions, outputDataSize.convertToMostSuccinctDataSize(), outputPositions, succinctBytes(physicalWrittenDataSize),
completedDrivers, new DataSize(memoryReservation.get(), BYTE).convertToMostSuccinctDataSize(), new DataSize(systemMemoryReservation.get(), BYTE).convertToMostSuccinctDataSize(), blockedReasons, new DataSize(rawInputDataSize, BYTE).convertToMostSuccinctDataSize(), rawInputPositions, new DataSize(processedInputDataSize, BYTE).convertToMostSuccinctDataSize(), processedInputPositions, new DataSize(outputDataSize, BYTE).convertToMostSuccinctDataSize(), outputPositions,
DataSize rate = dataRate(size, duration).convertToMostSuccinctDataSize(); stats.addShardRecoveryDataRate(rate, size, duration);
DataSize rate = dataRate(size, duration).convertToMostSuccinctDataSize(); stats.addShardRecoveryDataRate(rate, size, duration);
private static DataSize writeRcFileColumnOld(File outputFile, Format format, Compression compression, Type type, Iterator<?> values) throws Exception { ObjectInspector columnObjectInspector = getJavaObjectInspector(type); RecordWriter recordWriter = createRcFileWriterOld(outputFile, compression, columnObjectInspector); SettableStructObjectInspector objectInspector = createSettableStructObjectInspector("test", columnObjectInspector); Object row = objectInspector.create(); List<StructField> fields = ImmutableList.copyOf(objectInspector.getAllStructFieldRefs()); Serializer serializer = format.createSerializer(); Properties tableProperties = new Properties(); tableProperties.setProperty("columns", "test"); tableProperties.setProperty("columns.types", objectInspector.getTypeName()); serializer.initialize(new JobConf(false), tableProperties); while (values.hasNext()) { Object value = values.next(); value = preprocessWriteValueOld(type, value); objectInspector.setStructFieldData(row, fields.get(0), value); Writable record = serializer.serialize(row, objectInspector); recordWriter.write(record); } recordWriter.close(false); return new DataSize(outputFile.length(), BYTE).convertToMostSuccinctDataSize(); }
private static DataSize writeRcFileColumnOld(File outputFile, Format format, Compression compression, Type type, Iterator<?> values) throws Exception { ObjectInspector columnObjectInspector = getJavaObjectInspector(type); RecordWriter recordWriter = createRcFileWriterOld(outputFile, compression, columnObjectInspector); SettableStructObjectInspector objectInspector = createSettableStructObjectInspector("test", columnObjectInspector); Object row = objectInspector.create(); List<StructField> fields = ImmutableList.copyOf(objectInspector.getAllStructFieldRefs()); Serializer serializer = format.createSerializer(); Properties tableProperties = new Properties(); tableProperties.setProperty("columns", "test"); tableProperties.setProperty("columns.types", objectInspector.getTypeName()); serializer.initialize(new JobConf(false), tableProperties); while (values.hasNext()) { Object value = values.next(); value = preprocessWriteValueOld(type, value); objectInspector.setStructFieldData(row, fields.get(0), value); Writable record = serializer.serialize(row, objectInspector); recordWriter.write(record); } recordWriter.close(false); return new DataSize(outputFile.length(), BYTE).convertToMostSuccinctDataSize(); }
private static DataSize writeRcFileColumnOld(File outputFile, Format format, Compression compression, Type type, Iterator<?> values) throws Exception { ObjectInspector columnObjectInspector = getJavaObjectInspector(type); RecordWriter recordWriter = createRcFileWriterOld(outputFile, compression, columnObjectInspector); SettableStructObjectInspector objectInspector = createSettableStructObjectInspector("test", columnObjectInspector); Object row = objectInspector.create(); List<StructField> fields = ImmutableList.copyOf(objectInspector.getAllStructFieldRefs()); @SuppressWarnings("deprecation") Serializer serializer = format.createSerializer(); Properties tableProperties = new Properties(); tableProperties.setProperty("columns", "test"); tableProperties.setProperty("columns.types", objectInspector.getTypeName()); serializer.initialize(new JobConf(false), tableProperties); while (values.hasNext()) { Object value = values.next(); value = preprocessWriteValueOld(type, value); objectInspector.setStructFieldData(row, fields.get(0), value); Writable record = serializer.serialize(row, objectInspector); recordWriter.write(record); } recordWriter.close(false); return new DataSize(outputFile.length(), BYTE).convertToMostSuccinctDataSize(); }
new DataSize(totalMemoryReservation, BYTE).convertToMostSuccinctDataSize(), new DataSize(peakMemoryReservation, BYTE).convertToMostSuccinctDataSize(), new Duration(totalScheduledTime, NANOSECONDS).convertToMostSuccinctTimeUnit(), new Duration(totalCpuTime, NANOSECONDS).convertToMostSuccinctTimeUnit(), blockedReasons, new DataSize(rawInputDataSize, BYTE).convertToMostSuccinctDataSize(), rawInputPositions, new DataSize(processedInputDataSize, BYTE).convertToMostSuccinctDataSize(), processedInputPositions, new DataSize(outputDataSize, BYTE).convertToMostSuccinctDataSize(), outputPositions);
private static DataSize writeParquetColumn(JobConf jobConf, File outputFile, CompressionCodecName compressionCodecName, ObjectInspector columnObjectInspector, Iterator<?> values) throws Exception { RecordWriter recordWriter = new MapredParquetOutputFormat().getHiveRecordWriter(jobConf, new Path(outputFile.toURI()), Text.class, compressionCodecName != UNCOMPRESSED, createTableProperties("test", columnObjectInspector.getTypeName()), () -> { } ); SettableStructObjectInspector objectInspector = createSettableStructObjectInspector("test", columnObjectInspector); Object row = objectInspector.create(); List<StructField> fields = ImmutableList.copyOf(objectInspector.getAllStructFieldRefs()); int i = 0; while (values.hasNext()) { Object value = values.next(); objectInspector.setStructFieldData(row, fields.get(0), value); ParquetHiveSerDe serde = new ParquetHiveSerDe(); serde.initialize(jobConf, createTableProperties("test", columnObjectInspector.getTypeName()), null); Writable record = serde.serialize(row, objectInspector); recordWriter.write(record); i++; } recordWriter.close(false); return new DataSize(outputFile.length(), Unit.BYTE).convertToMostSuccinctDataSize(); }
new Duration(addInputCpu, NANOSECONDS).convertToMostSuccinctTimeUnit(), new Duration(addInputUser, NANOSECONDS).convertToMostSuccinctTimeUnit(), new DataSize(inputDataSize, BYTE).convertToMostSuccinctDataSize(), inputPositions, new Duration(getOutputCpu, NANOSECONDS).convertToMostSuccinctTimeUnit(), new Duration(getOutputUser, NANOSECONDS).convertToMostSuccinctTimeUnit(), new DataSize(outputDataSize, BYTE).convertToMostSuccinctDataSize(), outputPositions, new Duration(finishUser, NANOSECONDS).convertToMostSuccinctTimeUnit(), new DataSize(memoryReservation, BYTE).convertToMostSuccinctDataSize(), new DataSize(systemMemoryReservation, BYTE).convertToMostSuccinctDataSize(), blockedReason,