CompressionOption(CompressionType value, CompressionCodec codec) { this.value = value; this.codec = (CompressionType.NONE != value && null == codec) ? new DefaultCodec() : codec; } CompressionType getValue() {
@Override public void open(String filePath) throws IOException { DefaultCodec defCodec = new DefaultCodec(); CompressionType cType = CompressionType.BLOCK; open(filePath, defCodec, cType); }
@Override CompressionCodec getCodec() { if (codec == null) { codec = new DefaultCodec(); codec.setConf(conf); } return codec; }
/** * Find the codecs specified in the config value io.compression.codecs * and register them. Defaults to gzip and deflate. */ public CompressionCodecFactory(Configuration conf) { codecs = new TreeMap<String, CompressionCodec>(); codecsByClassName = new HashMap<String, CompressionCodec>(); codecsByName = new HashMap<String, CompressionCodec>(); List<Class<? extends CompressionCodec>> codecClasses = getCodecClasses(conf); if (codecClasses == null || codecClasses.isEmpty()) { addCodec(new GzipCodec()); addCodec(new DefaultCodec()); } else { for (Class<? extends CompressionCodec> codecClass : codecClasses) { addCodec(ReflectionUtils.newInstance(codecClass, conf)); } } }
@SuppressWarnings("deprecation") SequenceFile.Writer writer = closer.register(SequenceFile.createWriter(this.fs, this.conf, tmpTablePath, Text.class, this.stateClass, SequenceFile.CompressionType.BLOCK, new DefaultCodec())); writer.append(new Text(Strings.nullToEmpty(state.getId())), state); } catch (Throwable t) {
@SuppressWarnings("deprecation") SequenceFile.Writer writer = closer.register(SequenceFile.createWriter(this.fs, this.conf, tmpTablePath, Text.class, this.stateClass, SequenceFile.CompressionType.BLOCK, new DefaultCodec())); for (T state : states) { writer.append(new Text(Strings.nullToEmpty(state.getId())), state);
public static String getFileExtension(JobConf conf, StorageFormat storageFormat) { // text format files must have the correct extension when compressed if (!HiveConf.getBoolVar(conf, COMPRESSRESULT) || !HiveIgnoreKeyTextOutputFormat.class.getName().equals(storageFormat.getOutputFormat())) { return ""; } String compressionCodecClass = conf.get("mapred.output.compression.codec"); if (compressionCodecClass == null) { return new DefaultCodec().getDefaultExtension(); } try { Class<? extends CompressionCodec> codecClass = conf.getClassByName(compressionCodecClass).asSubclass(CompressionCodec.class); return ReflectionUtil.newInstance(codecClass, conf).getDefaultExtension(); } catch (ClassNotFoundException e) { throw new PrestoException(HIVE_UNSUPPORTED_FORMAT, "Compression codec not found: " + compressionCodecClass, e); } catch (RuntimeException e) { throw new PrestoException(HIVE_UNSUPPORTED_FORMAT, "Failed to load compression codec: " + compressionCodecClass, e); } }
CompressionCodec codec = new DefaultCodec(); testcase.columnMaxSize = 30;
/** * For running a few tests of methods herein. * @param args * @throws IOException */ public static void main(String[] args) throws IOException { int count = 1024; int size = 10240; for (String arg : args) { if (arg.startsWith(COUNT)) { count = Integer.parseInt(arg.replace(COUNT, "")); } else if (arg.startsWith(SIZE)) { size = Integer.parseInt(arg.replace(SIZE, "")); } else { usage(1); } } CellBlockBuilder builder = new CellBlockBuilder(HBaseConfiguration.create()); timerTests(builder, count, size, new KeyValueCodec(), null); timerTests(builder, count, size, new KeyValueCodec(), new DefaultCodec()); timerTests(builder, count, size, new KeyValueCodec(), new GzipCodec()); } }
private void writeTest(FileSystem fs, int count, Path file, byte[][] fieldsData, Configuration conf) throws IOException, SerDeException { cleanup(); RCFileOutputFormat.setColumnNumber(conf, fieldsData.length); RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null, new DefaultCodec()); BytesRefArrayWritable bytes = new BytesRefArrayWritable(fieldsData.length); for (int i = 0; i < fieldsData.length; i++) { BytesRefWritable cu = null; cu = new BytesRefWritable(fieldsData[i], 0, fieldsData[i].length); bytes.set(i, cu); } for (int i = 0; i < count; i++) { writer.append(bytes); } writer.close(); long fileLen = fs.getFileStatus(file).getLen(); System.out.println("The file size of RCFile with " + bytes.size() + " number columns and " + count + " number rows is " + fileLen); }
new DefaultCodec()); BytesRefArrayWritable bytes = new BytesRefArrayWritable(record.length); final int recCount = 100;
new Text("cat"), new Text("dog")), new DefaultCodec()); BytesRefArrayWritable bytes = new BytesRefArrayWritable(record_1.length); for (int i = 0; i < record_1.length; i++) {
new Text("cat"), new Text("dog")), new DefaultCodec());
codec = new DefaultCodec(); ((Configurable)codec).setConf(conf);
@Test public void testBuildCellBlock() throws IOException { doBuildCellBlockUndoCellBlock(this.builder, new KeyValueCodec(), null); doBuildCellBlockUndoCellBlock(this.builder, new KeyValueCodec(), new DefaultCodec()); doBuildCellBlockUndoCellBlock(this.builder, new KeyValueCodec(), new GzipCodec()); }
/** * Create a new GZ codec * * @param bufferSize * buffer size to for GZ * @return created codec */ @Override protected CompressionCodec createNewCodec(final int bufferSize) { DefaultCodec myCodec = new DefaultCodec(); Configuration myConf = new Configuration(conf); // only use the buffersize if > 0, otherwise we'll use // the default defined within the codec if (bufferSize > 0) myConf.setInt(BUFFER_SIZE_OPT, bufferSize); myCodec.setConf(myConf); return myCodec; }
/** * For running a few tests of methods herein. * @param args * @throws IOException */ public static void main(String[] args) throws IOException { int count = 1024; int size = 10240; for (String arg : args) { if (arg.startsWith(COUNT)) { count = Integer.parseInt(arg.replace(COUNT, "")); } else if (arg.startsWith(SIZE)) { size = Integer.parseInt(arg.replace(SIZE, "")); } else { usage(1); } } CellBlockBuilder builder = new CellBlockBuilder(HBaseConfiguration.create()); timerTests(builder, count, size, new KeyValueCodec(), null); timerTests(builder, count, size, new KeyValueCodec(), new DefaultCodec()); timerTests(builder, count, size, new KeyValueCodec(), new GzipCodec()); } }
CompressionOption(CompressionType value, CompressionCodec codec) { this.value = value; this.codec = (CompressionType.NONE != value && null == codec) ? new DefaultCodec() : codec; } CompressionType getValue() {
@Test public void testBuildCellBlock() throws IOException { doBuildCellBlockUndoCellBlock(this.builder, new KeyValueCodec(), null); doBuildCellBlockUndoCellBlock(this.builder, new KeyValueCodec(), new DefaultCodec()); doBuildCellBlockUndoCellBlock(this.builder, new KeyValueCodec(), new GzipCodec()); }
@Before public void setup() { this.codec = new DefaultCodec(); this.codec.setConf(new Configuration()); }