@Override public void setup(VectorAccessible incoming, OutputEntryListener outputEntryListener, WriteStatsListener writeStatsListener) throws IOException { Preconditions.checkArgument(incoming.getSchema().getSelectionVectorMode() == SelectionVectorMode.NONE, "SelectionVector remover is not supported."); this.incoming = incoming; this.outputEntryListener = outputEntryListener; this.writeStatsListener = writeStatsListener; this.fs = FileSystemWrapper.get(location, writerConfig.getFsConf(), stats); this.currentFile = fs.canonicalizePath(new Path(location, String.format("%s_%d.%s", prefix, nextFileIndex, extension))); this.relativePath = currentFile.getName(); this.currentFileOutputStream = fs.create(currentFile); listOfFilesCreated.add(currentFile); // write magic word bytes currentFileOutputStream.write(MAGIC_STRING.getBytes()); for(final VectorWrapper<? extends ValueVector> vw : incoming) { Preconditions.checkArgument(!vw.isHyper(), "Writing hyper vectors to arrow format is not supported."); footerBuilder.addField(TypeHelper.getMetadata(vw.getValueVector())); } nextFileIndex++; recordCount = 0; }
@Override public void startPartition(WritePartition partition) throws Exception { if(this.partition != null){ close(); } this.partition = partition; // open a new file for writing data with new schema try { this.path = fs.canonicalizePath(partition.qualified(location, prefix + "_" + index + "." + extension)); fos = fs.create(path); stream = new PrintStream(fos); stream.write(ByteOrderMark.UTF_8.getBytes(), 0, ByteOrderMark.UTF_8.length()); logger.debug("Created file: {}", path); } catch (IOException e) { throw UserException.dataWriteError(e) .message("Failure while attempting to write file %s.", path) .build(logger); } index++; String columns = Joiner.on(fieldDelimiter).join(columnNames); stream.print(columns); stream.print(lineDelimiter); }
@Override public void startPartition(WritePartition partition) throws Exception { // close previous partition if open. if(this.partition != null){ close(); } this.partition = partition; try { this.fileName = fs.canonicalizePath(partition.qualified(location, prefix + "_0." + extension)); stream = fs.create(fileName); JsonGenerator generator = factory.createGenerator(stream.getWrappedStream()).useDefaultPrettyPrinter(); if (uglify) { generator = generator.setPrettyPrinter(new MinimalPrettyPrinter(LINE_FEED)); } if(useExtendedOutput){ gen = new ExtendedJsonOutput(generator); }else{ gen = new BasicJsonOutput(generator); } logger.debug("Created file: {}", fileName); } catch (IOException ex) { throw UserException.dataWriteError(ex) .message("Failure writing JSON file %s.", fileName) .build(logger); } }
@Override public boolean createOrUpdateView(NamespaceKey key, View view, SchemaConfig schemaConfig) throws IOException { if(!getMutability().hasMutationCapability(MutationType.VIEW, schemaConfig.isSystemUser())) { throw UserException.parseError() .message("Unable to create view. Schema [%s] is immutable for this user.", key.getParent()) .build(logger); } Path viewPath = getViewPath(key.getPathComponents()); FileSystemWrapper fs = getFS(schemaConfig.getUserName()); boolean replaced = fs.exists(viewPath); final FsPermission viewPerms = new FsPermission(schemaConfig.getOption(ExecConstants.NEW_VIEW_DEFAULT_PERMS_KEY).getStringVal()); try (OutputStream stream = FileSystemWrapper.create(fs, viewPath, viewPerms)) { lpPersistance.getMapper().writeValue(stream, view); } return replaced; }
os = dfs.create(new Path(tempFolder.getRoot().getPath(), "dremioFSWriteTest.txt"));