final long partitionLengthInSpill = spills[i].partitionLengths[partition]; if (partitionLengthInSpill > 0) { InputStream partitionInputStream = new LimitedInputStream(spillInputStreams[i], partitionLengthInSpill, false); try {
final long partitionLengthInSpill = spills[i].partitionLengths[partition]; if (partitionLengthInSpill > 0) { InputStream partitionInputStream = new LimitedInputStream(spillInputStreams[i], partitionLengthInSpill, false); try {
final long partitionLengthInSpill = spills[i].partitionLengths[partition]; if (partitionLengthInSpill > 0) { InputStream partitionInputStream = new LimitedInputStream(spillInputStreams[i], partitionLengthInSpill, false); try {
private List<Tuple2<Object, Object>> readRecordsFromFile() throws IOException { final ArrayList<Tuple2<Object, Object>> recordsList = new ArrayList<>(); long startOffset = 0; for (int i = 0; i < NUM_PARTITITONS; i++) { final long partitionSize = partitionSizesInMergedFile[i]; if (partitionSize > 0) { FileInputStream fin = new FileInputStream(mergedOutputFile); fin.getChannel().position(startOffset); InputStream in = new LimitedInputStream(fin, partitionSize); in = blockManager.serializerManager().wrapForEncryption(in); if (conf.getBoolean("spark.shuffle.compress", true)) { in = CompressionCodec$.MODULE$.createCodec(conf).compressedInputStream(in); } DeserializationStream recordsStream = serializer.newInstance().deserializeStream(in); Iterator<Tuple2<Object, Object>> records = recordsStream.asKeyValueIterator(); while (records.hasNext()) { Tuple2<Object, Object> record = records.next(); assertEquals(i, hashPartitioner.getPartition(record._1())); recordsList.add(record); } recordsStream.close(); startOffset += partitionSize; } } return recordsList; }
private List<Tuple2<Object, Object>> readRecordsFromFile() throws IOException { final ArrayList<Tuple2<Object, Object>> recordsList = new ArrayList<>(); long startOffset = 0; for (int i = 0; i < NUM_PARTITITONS; i++) { final long partitionSize = partitionSizesInMergedFile[i]; if (partitionSize > 0) { FileInputStream fin = new FileInputStream(mergedOutputFile); fin.getChannel().position(startOffset); InputStream in = new LimitedInputStream(fin, partitionSize); in = blockManager.serializerManager().wrapForEncryption(in); if (conf.getBoolean("spark.shuffle.compress", true)) { in = CompressionCodec$.MODULE$.createCodec(conf).compressedInputStream(in); } DeserializationStream recordsStream = serializer.newInstance().deserializeStream(in); Iterator<Tuple2<Object, Object>> records = recordsStream.asKeyValueIterator(); while (records.hasNext()) { Tuple2<Object, Object> record = records.next(); assertEquals(i, hashPartitioner.getPartition(record._1())); recordsList.add(record); } recordsStream.close(); startOffset += partitionSize; } } return recordsList; }
private List<Tuple2<Object, Object>> readRecordsFromFile() throws IOException { final ArrayList<Tuple2<Object, Object>> recordsList = new ArrayList<>(); long startOffset = 0; for (int i = 0; i < NUM_PARTITITONS; i++) { final long partitionSize = partitionSizesInMergedFile[i]; if (partitionSize > 0) { FileInputStream fin = new FileInputStream(mergedOutputFile); fin.getChannel().position(startOffset); InputStream in = new LimitedInputStream(fin, partitionSize); in = blockManager.serializerManager().wrapForEncryption(in); if (conf.getBoolean("spark.shuffle.compress", true)) { in = CompressionCodec$.MODULE$.createCodec(conf).compressedInputStream(in); } DeserializationStream recordsStream = serializer.newInstance().deserializeStream(in); Iterator<Tuple2<Object, Object>> records = recordsStream.asKeyValueIterator(); while (records.hasNext()) { Tuple2<Object, Object> record = records.next(); assertEquals(i, hashPartitioner.getPartition(record._1())); recordsList.add(record); } recordsStream.close(); startOffset += partitionSize; } } return recordsList; }
throw new FileUploadIOException(e); istream = new LimitedInputStream(istream, fileSizeMax) { @Override protected void raiseError(long pSizeMax, long pCount)
@Override public InputStream createInputStream() throws IOException { FileInputStream is = null; boolean shouldClose = true; try { is = new FileInputStream(file); ByteStreams.skipFully(is, offset); InputStream r = new LimitedInputStream(is, length); shouldClose = false; return r; } catch (IOException e) { String errorMessage = "Error in reading " + this; if (is != null) { long size = file.length(); errorMessage = "Error in reading " + this + " (actual file length " + size + ")"; } throw new IOException(errorMessage, e); } finally { if (shouldClose) { JavaUtils.closeQuietly(is); } } }
@Override public InputStream createInputStream() throws IOException { FileInputStream is = null; boolean shouldClose = true; try { is = new FileInputStream(file); ByteStreams.skipFully(is, offset); InputStream r = new LimitedInputStream(is, length); shouldClose = false; return r; } catch (IOException e) { String errorMessage = "Error in reading " + this; if (is != null) { long size = file.length(); errorMessage = "Error in reading " + this + " (actual file length " + size + ")"; } throw new IOException(errorMessage, e); } finally { if (shouldClose) { JavaUtils.closeQuietly(is); } } }
@Override public InputStream createInputStream() throws IOException { FileInputStream is = null; try { is = new FileInputStream(file); ByteStreams.skipFully(is, offset); return new LimitedInputStream(is, length); } catch (IOException e) { try { if (is != null) { long size = file.length(); throw new IOException("Error in reading " + this + " (actual file length " + size + ")", e); } } catch (IOException ignored) { // ignore } finally { JavaUtils.closeQuietly(is); } throw new IOException("Error in opening " + this, e); } catch (RuntimeException e) { JavaUtils.closeQuietly(is); throw e; } }
@Override public InputStream createInputStream() throws IOException { FileInputStream is = null; try { is = new FileInputStream(file); ByteStreams.skipFully(is, offset); return new LimitedInputStream(is, length); } catch (IOException e) { try { if (is != null) { long size = file.length(); throw new IOException("Error in reading " + this + " (actual file length " + size + ")", e); } } catch (IOException ignored) { // ignore } finally { JavaUtils.closeQuietly(is); } throw new IOException("Error in opening " + this, e); } catch (RuntimeException e) { JavaUtils.closeQuietly(is); throw e; } }