@Override protected void setValue(long length) { if (value == null || value.getPath() == null) { value = new StreamLocator(file, codec); } } }
Path path = ((StreamLocator)value).getPath(); if (fs == null) { URI fileUri = path.toUri(); fs = FileSystem.get(fileUri, conf); switch (((StreamLocator)value).getCodec()) { case GZIP: InputStream fileIn = fs.open(path); default: LOG.error("Unsupported compression codec: " + ((StreamLocator)value).getCodec() + " for document " + key); return content;
@Override public boolean nextKeyValue() throws IOException, InterruptedException { if (iterator.hasNext()) { FileSplit split = iterator.next(); setFile(split.getPath()); String uri = makeURIFromPath(file); if (setKey(uri, 0, 0, true)) { return true; } value = new StreamLocator(file, CompressionCodec.NONE); bytesRead += split.getLength(); return true; } return false; } }