/** {@inheritDoc} */ @Override public long length() { return is.length(); }
/** * Read some data from the given file with the given offset. * * @param path File path. * @param off Offset. * @param len Length. * @throws Exception If failed. */ private void read(IgfsPath path, int off, int len) throws Exception { IgfsInputStream is = igfsPrimary.open(path); is.readFully(off, new byte[len]); is.close(); }
/** * Constructor. * * @param is Base input stream. * @param start Start position. * @param maxLen Maximum stream length. * @throws IOException In case of exception. */ public IgfsRangeInputStream(IgfsInputStream is, long start, long maxLen) throws IOException { if (is == null) throw new IllegalArgumentException("Input stream cannot be null."); if (start < 0) throw new IllegalArgumentException("Start position cannot be negative."); if (start >= is.length()) throw new IllegalArgumentException("Start position cannot be greater that file length."); if (maxLen < 0) throw new IllegalArgumentException("Length cannot be negative."); if (start + maxLen > is.length()) throw new IllegalArgumentException("Sum of start position and length cannot be greater than file length."); this.is = is; this.start = start; this.maxLen = maxLen; is.seek(start); }
/** * Runs query check result. * * @param expRes Expected result. * @param qry Query. * @throws Exception If failed. */ private void checkQuery(String expRes, String qry) throws Exception { assertEquals(0, executeHiveQuery("drop table if exists result")); assertEquals(0, executeHiveQuery( "create table result " + "row format delimited fields terminated by ' ' " + "stored as textfile " + "location '/result' as " + qry )); IgfsInputStream in = igfs.open(new IgfsPath("/result/000000_0")); byte[] buf = new byte[(int) in.length()]; in.read(buf); assertEquals(expRes, new String(buf)); }
/** {@inheritDoc} */ @Override public void seek(long pos) throws IOException { if (pos < 0) throw new IOException("Seek position cannot be negative: " + pos); is.seek(start + pos); this.pos = pos; }
/** {@inheritDoc} */ @Override public int read() throws IOException { if (pos < maxLen) { int res = is.read(); if (res != -1) pos++; return res; } else return -1; }
/** {@inheritDoc} */ @Override public void handleFile(String strPath) throws Exception { IgfsPath path = new IgfsPath(strPath); IgfsInputStream in; try { in = fs.open(path); } catch (IgfsPathNotFoundException ex) { System.out.println("file " + path.toString() + " not exist: " + ex); throw ex; } catch (IgniteException ex) { System.out.println("open file " + path.toString() + " failed: " + ex); throw ex; } try { for (int i = 0; i < size / dataBufer.capacity(); i++) in.read(dataBufer.array()); } catch (IOException ex) { System.out.println("read file " + path.toString() + " failed: " + ex); throw ex; } finally { in.close(); } }
/** {@inheritDoc} */ @Override public void close() throws IOException { is.close(); }
/** {@inheritDoc} */ @Override public IgniteInternalFuture<byte[]> readData(HadoopIgfsStreamDelegate delegate, long pos, int len, @Nullable byte[] outBuf, int outOff, int outLen) { IgfsInputStream stream = delegate.target(); try { byte[] res = null; if (outBuf != null) { int outTailLen = outBuf.length - outOff; if (len <= outTailLen) stream.readFully(pos, outBuf, outOff, len); else { stream.readFully(pos, outBuf, outOff, outTailLen); int remainderLen = len - outTailLen; res = new byte[remainderLen]; stream.readFully(pos, res, 0, remainderLen); } } else { res = new byte[len]; stream.readFully(pos, res, 0, len); } return new GridFinishedFuture<>(res); } catch (IllegalStateException | IOException e) { HadoopIgfsStreamEventListener lsnr = lsnrs.get(delegate); if (lsnr != null) lsnr.onError(e.getMessage()); return new GridFinishedFuture<>(e); } }
/** {@inheritDoc} */ @Override public IgfsFileRange resolveRecords(IgniteFileSystem fs, IgfsInputStream stream, IgfsFileRange suggestedRecord) throws IgniteException, IOException { long suggestedStart = suggestedRecord.start(); long suggestedEnd = suggestedStart + suggestedRecord.length(); IgniteBiTuple<State, Delimiter> firstDelim = findFirstDelimiter(stream, suggestedStart); State state = firstDelim != null ? firstDelim.getKey() : new State(); Delimiter curDelim = firstDelim.getValue(); while (curDelim != null && curDelim.end < suggestedStart) curDelim = nextDelimiter(stream, state); if (curDelim != null && (curDelim.end >= suggestedStart && curDelim.end < suggestedEnd) || suggestedStart == 0 ) { // We found start delimiter. long start = suggestedStart == 0 ? 0 : curDelim.end; if (curDelim == null || curDelim.end < suggestedEnd) { IgniteBiTuple<State, Delimiter> lastDelim = findFirstDelimiter(stream, suggestedEnd); state = lastDelim != null ? firstDelim.getKey() : new State(); curDelim = lastDelim.getValue(); while (curDelim != null && curDelim.end < suggestedEnd) curDelim = nextDelimiter(stream, state); } long end = curDelim != null ? curDelim.end : stream.position(); return new IgfsFileRange(suggestedRecord.path(), start, end - start); } else // We failed to find any delimiters up to the EOS. return null; }
/** * Constructor. * * @param is Base input stream. * @param start Start position. * @param maxLen Maximum stream length. * @throws IOException In case of exception. */ public IgfsRangeInputStream(IgfsInputStream is, long start, long maxLen) throws IOException { if (is == null) throw new IllegalArgumentException("Input stream cannot be null."); if (start < 0) throw new IllegalArgumentException("Start position cannot be negative."); if (start >= is.length()) throw new IllegalArgumentException("Start position cannot be greater that file length."); if (maxLen < 0) throw new IllegalArgumentException("Length cannot be negative."); if (start + maxLen > is.length()) throw new IllegalArgumentException("Sum of start position and length cannot be greater than file length."); this.is = is; this.start = start; this.maxLen = maxLen; is.seek(start); }
/** * Find first delimiter. In order to achieve this we have to rewind the stream until we find the delimiter * which stands at least [maxDelimLen] from the start search position or until we faced stream start. * Otherwise we cannot be sure that delimiter position is determined correctly. * * @param stream IGFS input stream. * @param startPos Start search position. * @return The first found delimiter. * @throws IOException In case of IO exception. */ @Nullable private IgniteBiTuple<State, Delimiter> findFirstDelimiter(IgfsInputStream stream, long startPos) throws IOException { State state; Delimiter delim; long curPos = Math.max(0, startPos - maxDelimLen); while (true) { stream.seek(curPos); state = new State(); delim = nextDelimiter(stream, state); if (curPos == 0 || delim == null || delim.start - curPos > maxDelimLen - 1) break; else curPos = Math.max(0, curPos - maxDelimLen); } return F.t(state, delim); }
/** {@inheritDoc} */ @Override public int read(@NotNull byte[] b, int off, int len) throws IOException { if (pos < maxLen) { len = (int)Math.min(len, maxLen - pos); int res = is.read(b, off, len); if (res != -1) pos += res; return res; } else return -1; }