public static RecordWriter getRecordWriter(JobConf jc, OutputFormat<?, ?> outputFormat, Class<? extends Writable> valueClass, boolean isCompressed, Properties tableProp, Path outPath, Reporter reporter ) throws IOException, HiveException { if (!(outputFormat instanceof HiveOutputFormat)) { outputFormat = new HivePassThroughOutputFormat(outputFormat); } return ((HiveOutputFormat)outputFormat).getHiveRecordWriter( jc, outPath, valueClass, isCompressed, tableProp, reporter); }
public static RecordWriter getRecordWriter(JobConf jc, OutputFormat<?, ?> outputFormat, Class<? extends Writable> valueClass, boolean isCompressed, Properties tableProp, Path outPath, Reporter reporter ) throws IOException, HiveException { if (!(outputFormat instanceof HiveOutputFormat)) { outputFormat = new HivePassThroughOutputFormat(outputFormat); } return ((HiveOutputFormat)outputFormat).getHiveRecordWriter( jc, outPath, valueClass, isCompressed, tableProp, reporter); }
public static RecordWriter createRecordWriter(Path target, JobConf conf, Properties properties, String outputFormatName, ConnectorSession session) { try { boolean compress = HiveConf.getBoolVar(conf, COMPRESSRESULT); if (outputFormatName.equals(RCFileOutputFormat.class.getName())) { return createRcFileWriter(target, conf, properties, compress); } if (outputFormatName.equals(MapredParquetOutputFormat.class.getName())) { return createParquetWriter(target, conf, properties, compress, session); } Object writer = Class.forName(outputFormatName).getConstructor().newInstance(); return ((HiveOutputFormat<?, ?>) writer).getHiveRecordWriter(conf, target, Text.class, compress, properties, Reporter.NULL); } catch (IOException | ReflectiveOperationException e) { throw new PrestoException(HIVE_WRITER_DATA_ERROR, e); } }
@Override public org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter getHiveRecordWriter( JobConf jc, Path finalOutPath, Class<? extends Writable> valueClass, boolean isCompressed, Properties tableProperties, Progressable progress) throws IOException { if (actualOutputFormat instanceof HiveOutputFormat) { return ((HiveOutputFormat<K, V>) actualOutputFormat).getHiveRecordWriter(jc, finalOutPath, valueClass, isCompressed, tableProperties, progress); } FileSystem fs = finalOutPath.getFileSystem(jc); RecordWriter<?, ?> recordWriter = actualOutputFormat.getRecordWriter(fs, jc, null, progress); return new HivePassThroughRecordWriter(recordWriter); } }
private RecordWriter getWriter() throws IOException { try { HiveOutputFormat<?, ?> outputFormat = HiveOutputFormat.class .cast(Class.forName(this.properties.getProp(HiveWritableHdfsDataWriterBuilder.WRITER_OUTPUT_FORMAT_CLASS)) .newInstance()); @SuppressWarnings("unchecked") Class<? extends Writable> writableClass = (Class<? extends Writable>) Class .forName(this.properties.getProp(HiveWritableHdfsDataWriterBuilder.WRITER_WRITABLE_CLASS)); return outputFormat.getHiveRecordWriter(new JobConf(), this.stagingFile, writableClass, true, this.properties.getProperties(), null); } catch (Throwable t) { throw new IOException(String.format("Failed to create writer"), t); } }
@Override public org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter getHiveRecordWriter( JobConf jc, Path finalOutPath, Class<? extends Writable> valueClass, boolean isCompressed, Properties tableProperties, Progressable progress) throws IOException { if (actualOutputFormat instanceof HiveOutputFormat) { return ((HiveOutputFormat<K, V>) actualOutputFormat).getHiveRecordWriter(jc, finalOutPath, valueClass, isCompressed, tableProperties, progress); } FileSystem fs = finalOutPath.getFileSystem(jc); RecordWriter<?, ?> recordWriter = actualOutputFormat.getRecordWriter(fs, jc, null, progress); return new HivePassThroughRecordWriter(recordWriter); } }
@SuppressWarnings({"rawtypes", "unchecked"}) private static Path createEmptyFile(Path hiveScratchDir, HiveOutputFormat outFileFormat, JobConf job, Properties props, boolean dummyRow) throws IOException, InstantiationException, IllegalAccessException { // create a dummy empty file in a new directory String newDir = hiveScratchDir + Path.SEPARATOR + UUID.randomUUID().toString(); Path newPath = new Path(newDir); FileSystem fs = newPath.getFileSystem(job); fs.mkdirs(newPath); //Qualify the path against the file system. The user configured path might contain default port which is skipped //in the file status. This makes sure that all paths which goes into PathToPartitionInfo are always listed status //file path. newPath = fs.makeQualified(newPath); String newFile = newDir + Path.SEPARATOR + "emptyFile"; Path newFilePath = new Path(newFile); RecordWriter recWriter = outFileFormat.getHiveRecordWriter(job, newFilePath, Text.class, false, props, null); if (dummyRow) { // empty files are omitted at CombineHiveInputFormat. // for meta-data only query, it effectively makes partition columns disappear.. // this could be fixed by other methods, but this seemed to be the most easy (HIVEV-2955) recWriter.write(new Text("empty")); // written via HiveIgnoreKeyTextOutputFormat } recWriter.close(false); return StringInternUtils.internUriStringsInPath(newPath); }
@SuppressWarnings({"rawtypes", "unchecked"}) private static Path createEmptyFile(Path hiveScratchDir, HiveOutputFormat outFileFormat, JobConf job, Properties props, boolean dummyRow) throws IOException, InstantiationException, IllegalAccessException { // create a dummy empty file in a new directory String newDir = hiveScratchDir + Path.SEPARATOR + UUID.randomUUID().toString(); Path newPath = new Path(newDir); FileSystem fs = newPath.getFileSystem(job); fs.mkdirs(newPath); //Qualify the path against the file system. The user configured path might contain default port which is skipped //in the file status. This makes sure that all paths which goes into PathToPartitionInfo are always listed status //file path. newPath = fs.makeQualified(newPath); String newFile = newDir + Path.SEPARATOR + "emptyFile"; Path newFilePath = new Path(newFile); RecordWriter recWriter = outFileFormat.getHiveRecordWriter(job, newFilePath, Text.class, false, props, null); if (dummyRow) { // empty files are omitted at CombineHiveInputFormat. // for meta-data only query, it effectively makes partition columns disappear.. // this could be fixed by other methods, but this seemed to be the most easy (HIVEV-2955) recWriter.write(new Text("empty")); // written via HiveIgnoreKeyTextOutputFormat } recWriter.close(false); return StringInternUtils.internUriStringsInPath(newPath); }
configureCompression(jobConf, compressionCodec); RecordWriter recordWriter = outputFormat.getHiveRecordWriter( jobConf, new Path(filePath),
@Test public void testEmptyFile() throws Exception { Properties properties = new Properties(); properties.setProperty("columns", "x,y"); properties.setProperty("columns.types", "int:int"); HiveOutputFormat<?, ?> outFormat = new OrcOutputFormat(); org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter writer = outFormat.getHiveRecordWriter(conf, testFilePath, MyRow.class, true, properties, Reporter.NULL); writer.close(true); AbstractSerDe serde = new OrcSerde(); SerDeUtils.initializeSerDe(serde, conf, properties, null); InputFormat<?,?> in = new OrcInputFormat(); FileInputFormat.setInputPaths(conf, testFilePath.toString()); InputSplit[] splits = in.getSplits(conf, 1); assertTrue(0 == splits.length); assertEquals(null, serde.getSerDeStats()); }
@Test(expected = RuntimeException.class) public void testSplitGenFailure() throws IOException { Properties properties = new Properties(); HiveOutputFormat<?, ?> outFormat = new OrcOutputFormat(); org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter writer = outFormat.getHiveRecordWriter(conf, testFilePath, MyRow.class, true, properties, Reporter.NULL); writer.write(new OrcSerde().serialize(null,null)); writer.close(true); InputFormat<?,?> in = new OrcInputFormat(); fs.setPermission(testFilePath, FsPermission.createImmutable((short) 0333)); FileInputFormat.setInputPaths(conf, testFilePath.toString()); try { in.getSplits(conf, 1); } catch (RuntimeException e) { assertEquals(true, e.getMessage().contains("Permission denied")); throw e; } }
HiveOutputFormat<?, ?> outFormat = new OrcOutputFormat(); org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter writer = outFormat.getHiveRecordWriter(conf, testFilePath, MyRow.class, true, properties, Reporter.NULL); writer.write(serde.serialize(new MyRow(1,2), inspector));
HiveOutputFormat<?, ?> outFormat = new OrcOutputFormat(); org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter writer = outFormat.getHiveRecordWriter(conf, testFilePath, StringRow.class, true, properties, Reporter.NULL); writer.write(serde.serialize(new StringRow("owen"), inspector));
HiveOutputFormat<?, ?> outFormat = new OrcOutputFormat(); org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter writer = outFormat.getHiveRecordWriter(conf, testFilePath, MyRow.class, true, properties, Reporter.NULL);
public static RecordWriter getRecordWriter(JobConf jc, HiveOutputFormat<?, ?> hiveOutputFormat, final Class<? extends Writable> valueClass, boolean isCompressed, Properties tableProp, Path outPath) throws IOException, HiveException { if (hiveOutputFormat != null) { return hiveOutputFormat.getHiveRecordWriter(jc, outPath, valueClass, isCompressed, tableProp, null); } return null; }
public static RecordWriter getRecordWriter(JobConf jc, OutputFormat<?, ?> outputFormat, Class<? extends Writable> valueClass, boolean isCompressed, Properties tableProp, Path outPath, Reporter reporter ) throws IOException, HiveException { if (!(outputFormat instanceof HiveOutputFormat)) { outputFormat = new HivePassThroughOutputFormat(outputFormat); } return ((HiveOutputFormat)outputFormat).getHiveRecordWriter( jc, outPath, valueClass, isCompressed, tableProp, reporter); }
public static RecordWriter createRecordWriter(Path target, JobConf conf, boolean compress, Properties properties, String outputFormatName) { try { Object writer = Class.forName(outputFormatName).getConstructor().newInstance(); return ((HiveOutputFormat<?, ?>) writer).getHiveRecordWriter(conf, target, Text.class, compress, properties, Reporter.NULL); } catch (IOException | ReflectiveOperationException e) { throw new PrestoException(HIVE_WRITER_DATA_ERROR, e); } }
private RecordWriter getWriter() throws IOException { try { HiveOutputFormat<?, ?> outputFormat = HiveOutputFormat.class .cast(Class.forName(this.properties.getProp(HiveWritableHdfsDataWriterBuilder.WRITER_OUTPUT_FORMAT_CLASS)) .newInstance()); @SuppressWarnings("unchecked") Class<? extends Writable> writableClass = (Class<? extends Writable>) Class .forName(this.properties.getProp(HiveWritableHdfsDataWriterBuilder.WRITER_WRITABLE_CLASS)); return outputFormat.getHiveRecordWriter(new JobConf(), this.stagingFile, writableClass, true, this.properties.getProperties(), null); } catch (Throwable t) { throw new IOException(String.format("Failed to create writer"), t); } }
@Override public org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter getHiveRecordWriter( JobConf jc, Path finalOutPath, Class<? extends Writable> valueClass, boolean isCompressed, Properties tableProperties, Progressable progress) throws IOException { if (actualOutputFormat instanceof HiveOutputFormat) { return ((HiveOutputFormat<K, V>) actualOutputFormat).getHiveRecordWriter(jc, finalOutPath, valueClass, isCompressed, tableProperties, progress); } FileSystem fs = finalOutPath.getFileSystem(jc); RecordWriter<?, ?> recordWriter = actualOutputFormat.getRecordWriter(fs, jc, null, progress); return new HivePassThroughRecordWriter(recordWriter); } }
public static RecordWriter createRecordWriter(Path target, JobConf conf, Properties properties, String outputFormatName, ConnectorSession session) { try { boolean compress = HiveConf.getBoolVar(conf, COMPRESSRESULT); if (outputFormatName.equals(RCFileOutputFormat.class.getName())) { return createRcFileWriter(target, conf, properties, compress); } if (outputFormatName.equals(MapredParquetOutputFormat.class.getName())) { return createParquetWriter(target, conf, properties, compress, session); } Object writer = Class.forName(outputFormatName).getConstructor().newInstance(); return ((HiveOutputFormat<?, ?>) writer).getHiveRecordWriter(conf, target, Text.class, compress, properties, Reporter.NULL); } catch (IOException | ReflectiveOperationException e) { throw new PrestoException(HIVE_WRITER_DATA_ERROR, e); } }