/** * Creates a new user exception builder . * * @see com.dremio.exec.proto.UserBitShared.DremioPBError.ErrorType#DATA_WRITE * @return user exception builder */ public static Builder dataWriteError() { return dataWriteError(null); }
private UserException.Builder getExceptionBuilder(final Exception ex, final HashAggErrorType errorType) { switch (errorType) { case SPILL_READ: return (ex == null) ? (UserException.dataReadError()) : (UserException.dataReadError(ex)); case SPILL_WRITE: return (ex == null) ? (UserException.dataWriteError()) : (UserException.dataWriteError(ex)); case OOM: return (ex == null) ? (UserException.memoryError()) : (UserException.memoryError(ex)); default: /* should never be hit since VectorizedHashAggOperator controls the operator type */ return null; } }
@Override public void makeSpillSubdirs(String id) throws UserException { //TODO: use only the healthy spill directories, once health checks implemented (shortly!). Reviewer: if you see this code, ask Vanco to fix it! ArrayList<String> healthySpillDirs = this.healthySpillDirs; // Create spill directories for each disk. for (String directory : healthySpillDirs) { try { final Path spillDirPath = new Path(new Path(directory), id); FileSystem fileSystem = spillDirPath.getFileSystem(SPILLING_CONFIG); if (!fileSystem.mkdirs(spillDirPath, PERMISSIONS)) { //TODO: withContextParameters() throw UserException.dataWriteError() .message("Failed to create directory for spilling. Please check that the spill location is accessible and confirm read, write & execute permissions.") .addContext("Spill directory path", directory) .build(logger); } } catch (Exception e) { //TODO: withContextParameters() throw UserException.dataWriteError(e) .message("Failed to create spill directory for id " + id) .addContext("Spill directory path", directory) .build(logger); } } }
@Override public void startPartition(WritePartition partition) { if(!partition.isSinglePartition()){ throw UserException.dataWriteError().message("Arrow writer doesn't support data partitioning.").build(logger); } }
@Override public SpillDirectory getSpillSubdir(String id) throws UserException { ArrayList<String> currentSpillDirs = Lists.newArrayList(spillDirs); while (!currentSpillDirs.isEmpty()) { // pick a random spill directory final int index = ThreadLocalRandom.current().nextInt(currentSpillDirs.size()); final String spillDir = currentSpillDirs.get(index); final Path spillDirPath = new Path(spillDir); if (isHealthy(spillDirPath)) { try { //TODO: track number of spills created in 'spillDir' FileSystem fileSystem = spillDirPath.getFileSystem(SPILLING_CONFIG); final Path spillSubdir = new Path(spillDirPath, id); return new SpillDirectory(spillSubdir, fileSystem); } catch (IOException e) { // Ignore this 'spillDir'. Still consider the others } } // Hm... 'spillDir' didn't work out. Let's consider the others currentSpillDirs.remove(index); } // TODO: withContextParameters() throw UserException.dataWriteError() .message("Failed to spill to disk. Please check space availability") .addContext("spill id", id) .addContext("all spill locations", spillDirs.toString()) .build(logger); }
@Override public void writeTo(DatasetConfig datasetConfig, Class<?> type, Type genericType, Annotation[] annotations, MediaType mediaType, MultivaluedMap<String, Object> httpHeaders, OutputStream entityStream) throws IOException, WebApplicationException { final String hostname; if (httpHeaders.containsKey(WebServer.X_DREMIO_HOSTNAME)) { hostname = (String) httpHeaders.getFirst(WebServer.X_DREMIO_HOSTNAME); } else { hostname = masterNode; } // Change headers to force download and suggest a filename. String fullPath = Joiner.on(".").join(datasetConfig.getFullPathList()); httpHeaders.putSingle(HttpHeaders.CONTENT_DISPOSITION, format("attachment; filename=\"%s.tds\"", fullPath)); try { final XMLStreamWriter xmlStreamWriter = xmlOutputFactory.createXMLStreamWriter(entityStream, "UTF-8"); xmlStreamWriter.writeStartDocument("utf-8", "1.0"); writeDatasource(xmlStreamWriter, datasetConfig, hostname, mediaType); xmlStreamWriter.writeEndDocument(); xmlStreamWriter.close(); } catch (XMLStreamException e) { throw UserExceptionMapper.withStatus( UserException.dataWriteError(e) .message("Cannot generate TDS file") .addContext("Details", e.getMessage()), Status.INTERNAL_SERVER_ERROR ).build(logger); } }
@Override public void startPartition(WritePartition partition) throws Exception { if(this.partition != null){ close(); } this.partition = partition; // open a new file for writing data with new schema try { this.path = fs.canonicalizePath(partition.qualified(location, prefix + "_" + index + "." + extension)); fos = fs.create(path); stream = new PrintStream(fos); stream.write(ByteOrderMark.UTF_8.getBytes(), 0, ByteOrderMark.UTF_8.length()); logger.debug("Created file: {}", path); } catch (IOException e) { throw UserException.dataWriteError(e) .message("Failure while attempting to write file %s.", path) .build(logger); } index++; String columns = Joiner.on(fieldDelimiter).join(columnNames); stream.print(columns); stream.print(lineDelimiter); }
public SpillFile getSpillFile(String fileName) throws RuntimeException { try { final SpillDirectory spillDirectory = spillService.getSpillSubdir(id); return new SpillFile(spillDirectory.getFileSystem(), new Path(spillDirectory.getSpillDirPath(), fileName)); } catch (UserException e) { throw UserException.dataWriteError(e) .addContext("for %s spill id %s", caller, id) .addContext("Caller", caller) .build(logger); } }
@Override public void startPartition(WritePartition partition) throws Exception { if (index >= maxPartitions) { throw UserException.dataWriteError() .message("Materialization cancelled due to excessive partition creation. A single thread can only generate %d partitions. " + "Typically, this is a problem if you configure a partition or distribution column that has high cardinality. " + "If you want to increase this limit, you can change the \"store.max_partitions\" system option.", maxPartitions) .build(logger); } flushAndClose(); this.partition = partition; newSchema(); }
public SpillManager(SabotConfig sabotConfig, OptionManager optionManager, String id, Configuration hadoopConf, SpillService spillService, String caller) { final List<String> directories = new ArrayList<>(sabotConfig.getStringList(ExecConstants.SPILL_DIRS)); if (directories.isEmpty()) { throw UserException.dataWriteError().message("No spill locations specified.").build(logger); } this.id = id; this.caller = caller; this.hadoopConf = hadoopConf; this.spillService = spillService; // load options if (optionManager != null) { this.minDiskSpacePercentage = optionManager.getOption(ExecConstants.SPILL_DISK_SPACE_LIMIT_PERCENTAGE); this.minDiskSpace = optionManager.getOption(ExecConstants.SPILL_DISK_SPACE_LIMIT_BYTES); this.healthCheckInterval = optionManager.getOption(ExecConstants.SPILL_DISK_SPACE_CHECK_INTERVAL); } else { this.minDiskSpacePercentage = ExecConstants.SPILL_DISK_SPACE_LIMIT_PERCENTAGE.getDefault().getFloatVal(); this.minDiskSpace = ExecConstants.SPILL_DISK_SPACE_LIMIT_BYTES.getDefault().getNumVal(); this.healthCheckInterval = ExecConstants.SPILL_DISK_SPACE_CHECK_INTERVAL.getDefault().getNumVal(); } try { spillService.makeSpillSubdirs(id); } catch (UserException e) { throw UserException.dataWriteError(e) .addContext("Caller", caller) .build(logger); } }
@Override public void startPartition(WritePartition partition) throws Exception { // close previous partition if open. if(this.partition != null){ close(); } this.partition = partition; try { this.fileName = fs.canonicalizePath(partition.qualified(location, prefix + "_0." + extension)); stream = fs.create(fileName); JsonGenerator generator = factory.createGenerator(stream.getWrappedStream()).useDefaultPrettyPrinter(); if (uglify) { generator = generator.setPrettyPrinter(new MinimalPrettyPrinter(LINE_FEED)); } if(useExtendedOutput){ gen = new ExtendedJsonOutput(generator); }else{ gen = new BasicJsonOutput(generator); } logger.debug("Created file: {}", fileName); } catch (IOException ex) { throw UserException.dataWriteError(ex) .message("Failure writing JSON file %s.", fileName) .build(logger); } }
private void rotateRuns() { if(memoryRun.isEmpty()){ final String message = "Memory failed due to not enough memory to sort even one batch of records."; tracer.setExternalSortAllocatorState(allocator); throw tracer.prepareAndThrowException(new OutOfMemoryException(message), null); } try { memoryRun.closeToDisk(diskRuns); memoryRun = new MemoryRun(config, producer, allocator, incoming.getSchema(), tracer, batchsizeMultiplier); } catch (Exception e) { throw UserException.dataWriteError(e) .message("Failure while attempting to spill sort data to disk.") .build(logger); } }
@Override public void noMoreToConsume() { state = State.CAN_PRODUCE; if(diskRuns.isEmpty()){ // no spills // we can return the existing (already sorted) data copier = memoryRun.closeToCopier(output, targetBatchSize); sortState = SortState.COPY_FROM_MEMORY; } else { // some spills sortState = SortState.CONSOLIDATE; // spill remainders to only deal with disk runs if (!memoryRun.isEmpty()) { try { memoryRun.closeToDisk(diskRuns); } catch (Exception ex) { throw UserException.dataWriteError(ex).message("Failure while attempting to spill sort data to disk.") .build(logger); } } // only need to deal with disk runs. consolidateIfNecessary(); } updateStats(); }
@Test public void testBuildUserExceptionWithCause() { String message = "Test message"; UserException uex = UserException.dataWriteError(new RuntimeException(message)).build(logger); DremioPBError error = uex.getOrCreatePBError(false); // cause message should be used Assert.assertEquals(ErrorType.DATA_WRITE, error.getErrorType()); Assert.assertEquals(message, uex.getOriginalMessage()); }
@Test public void testBuildUserExceptionWithCauseAndMessage() { String messageA = "Test message A"; String messageB = "Test message B"; UserException uex = UserException.dataWriteError(new RuntimeException(messageA)).message(messageB).build(logger); DremioPBError error = uex.getOrCreatePBError(false); // passed message should override the cause message Assert.assertEquals(ErrorType.DATA_WRITE, error.getErrorType()); Assert.assertEquals(messageB, uex.getOriginalMessage()); }
@Test public void testBuildUserExceptionWithMessage() { String message = "Test message"; UserException uex = UserException.dataWriteError().message(message).build(logger); DremioPBError error = uex.getOrCreatePBError(false); Assert.assertEquals(ErrorType.DATA_WRITE, error.getErrorType()); Assert.assertEquals(message, uex.getOriginalMessage()); }
throw UserException.dataWriteError(t) .message("Failed to read data from parquet file") .addContext("File path", path)
@Test public void testBuildUserExceptionWithUserExceptionCauseAndMessage() { String messageA = "Test message A"; String messageB = "Test message B"; UserException original = UserException.connectionError().message(messageA).build(logger); UserException uex = UserException.dataWriteError(wrap(original, 5)).message(messageB).build(logger); //builder should return the unwrapped original user exception and not build a new one Assert.assertEquals(original, uex); DremioPBError error = uex.getOrCreatePBError(false); Assert.assertEquals(messageA, uex.getOriginalMessage()); Assert.assertFalse(error.getMessage().contains(messageB)); // messageB should not be part of the context }
@Override public int writeBatch(int offset, int length) throws IOException { if(offset != 0 || length != incoming.getRecordCount()){ throw UserException.dataWriteError().message("You cannot partition data written in Arrow format.").build(logger); } final int recordCount = incoming.getRecordCount(); final long startOffset = currentFileOutputStream.getPos(); final WritableBatch writableBatch = WritableBatch.getBatchNoHVWrap(recordCount, incoming, false /* isSv2 */); final VectorAccessibleSerializable serializer = new VectorAccessibleSerializable(writableBatch, null/*allocator*/); serializer.writeToStream(currentFileOutputStream); final long endOffset = currentFileOutputStream.getPos(); final ArrowRecordBatchSummary summary = ArrowRecordBatchSummary .newBuilder() .setOffset(startOffset) .setRecordCount(recordCount) .build(); footerBuilder.addBatch(summary); this.recordCount += recordCount; writeStatsListener.bytesWritten(endOffset - startOffset); return recordCount; }
} catch (IOException e) { throw UserException .dataWriteError(e) .message("Failed to drop table: " + e.getMessage()) .build(logger);