public HDFSTransactionLogReaderV2(SequenceFile.Reader reader) { this.reader = reader; this.transactionEdits = new ArrayDeque<>(); this.key = new LongWritable(); this.commitMarkerCodec = new CommitMarkerCodec(); }
@Override public void commitMarker(int count) throws IOException { CommitMarkerCodec.writeMarker(internalWriter, count); }
@Override public void close() throws IOException { if (closed) { return; } try { commitMarkerCodec.close(); } finally { reader.close(); closed = true; } }
@Test public void testRandomCommitMarkers() throws Exception { List<Integer> randomInts = new ArrayList<>(); Path newLog = new Path(TMP_FOLDER.newFolder().getAbsolutePath(), LOG_FILE); // Write a bunch of random commit markers try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, newLog, LongWritable.class, LongWritable.class, SequenceFile.CompressionType.NONE)) { for (int i = 0; i < 1000; i++) { int randomNum = RANDOM.nextInt(Integer.MAX_VALUE); CommitMarkerCodec.writeMarker(writer, randomNum); randomInts.add(randomNum); } writer.hflush(); writer.hsync(); } // Read the commit markers back to verify the marker try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, newLog, conf); CommitMarkerCodec markerCodec = new CommitMarkerCodec()) { for (int num : randomInts) { Assert.assertEquals(num, markerCodec.readMarker(reader)); } } }
@Test public void testIncorrectCommitMarker() throws Exception { Path newLog = new Path(TMP_FOLDER.newFolder().getAbsolutePath(), LOG_FILE); // Write an incorrect marker try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, newLog, LongWritable.class, LongWritable.class, SequenceFile.CompressionType.NONE)) { String invalidKey = "IncorrectKey"; SequenceFile.ValueBytes valueBytes = new CommitMarkerCodec.CommitEntriesCount(100); writer.appendRaw(invalidKey.getBytes(), 0, invalidKey.length(), valueBytes); writer.hflush(); writer.hsync(); } // Read the commit markers back to verify the marker try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, newLog, conf); CommitMarkerCodec markerCodec = new CommitMarkerCodec()) { try { markerCodec.readMarker(reader); Assert.fail("Expected an IOException to be thrown"); } catch (IOException e) { // expected } } } }
private void populateTransactionEdits() throws IOException { // read the marker to determine numEntries to read. int numEntries = 0; try { // can throw EOFException if reading of incomplete commit marker, no other action required since we can safely // ignore this numEntries = commitMarkerCodec.readMarker(reader); } catch (EOFException e) { LOG.warn("Reached EOF in log while trying to read commit marker", e); } for (int i = 0; i < numEntries; i++) { TransactionEdit edit = new TransactionEdit(); try { if (reader.next(key, edit)) { transactionEdits.add(edit); } else { throw new EOFException("Attempt to read TransactionEdit failed."); } } catch (EOFException e) { // we have reached EOF before reading back numEntries, we clear the partial list and return. LOG.warn("Reached EOF in log before reading {} entries. Ignoring all {} edits since the last marker", numEntries, transactionEdits.size(), e); transactionEdits.clear(); } } } }
public int readMarker(SequenceFile.Reader reader) throws IOException { if (valueBytes == null) { valueBytes = reader.createValueBytes(); } rawKey.reset(); rawValue.reset(); // valueBytes need not be reset since nextRaw call does it (and it is a private method) int status = reader.nextRaw(rawKey, valueBytes); // if we reach EOF, return -1 if (status == -1) { return -1; } // Check if the marker key is valid and return the count if (isMarkerValid()) { valueBytes.writeUncompressedBytes(rawValue); rawValue.flush(); // rawValue.getData() may return a larger byte array but Ints.fromByteArray will only read the first four bytes return Ints.fromByteArray(rawValue.getData()); } // EOF not reached and marker is not valid, then thrown an IOException since we can't make progress throw new IOException(String.format("Invalid key for num entries appended found %s, expected : %s", new String(rawKey.getData()), TxConstants.TransactionLog.NUM_ENTRIES_APPENDED)); }
@Test public void testIncompleteCommitMarker() throws Exception { Path newLog = new Path(TMP_FOLDER.newFolder().getAbsolutePath(), LOG_FILE); try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, newLog, LongWritable.class, LongWritable.class, SequenceFile.CompressionType.NONE)) { String key = TxConstants.TransactionLog.NUM_ENTRIES_APPENDED; SequenceFile.ValueBytes valueBytes = new IncompleteValueBytes(); writer.appendRaw(key.getBytes(), 0, key.length(), valueBytes); writer.hflush(); writer.hsync(); } // Read the incomplete commit marker try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, newLog, conf); CommitMarkerCodec markerCodec = new CommitMarkerCodec()) { try { markerCodec.readMarker(reader); Assert.fail("Expected EOF Exception to be thrown"); } catch (EOFException e) { // expected since we didn't write the value bytes } } }
private void populateTransactionEdits() throws IOException { // read the marker to determine numEntries to read. int numEntries = 0; try { // can throw EOFException if reading of incomplete commit marker, no other action required since we can safely // ignore this numEntries = commitMarkerCodec.readMarker(reader); } catch (EOFException e) { LOG.warn("Reached EOF in log while trying to read commit marker", e); } for (int i = 0; i < numEntries; i++) { TransactionEdit edit = new TransactionEdit(); try { if (reader.next(key, edit)) { transactionEdits.add(edit); } else { throw new EOFException("Attempt to read TransactionEdit failed."); } } catch (EOFException e) { // we have reached EOF before reading back numEntries, we clear the partial list and return. LOG.warn("Reached EOF in log before reading {} entries. Ignoring all {} edits since the last marker", numEntries, transactionEdits.size(), e); transactionEdits.clear(); } } } }
public int readMarker(SequenceFile.Reader reader) throws IOException { if (valueBytes == null) { valueBytes = reader.createValueBytes(); } rawKey.reset(); rawValue.reset(); // valueBytes need not be reset since nextRaw call does it (and it is a private method) int status = reader.nextRaw(rawKey, valueBytes); // if we reach EOF, return -1 if (status == -1) { return -1; } // Check if the marker key is valid and return the count if (isMarkerValid()) { valueBytes.writeUncompressedBytes(rawValue); rawValue.flush(); // rawValue.getData() may return a larger byte array but Ints.fromByteArray will only read the first four bytes return Ints.fromByteArray(rawValue.getData()); } // EOF not reached and marker is not valid, then thrown an IOException since we can't make progress throw new IOException(String.format("Invalid key for num entries appended found %s, expected : %s", new String(rawKey.getData()), TxConstants.TransactionLog.NUM_ENTRIES_APPENDED)); }
@Override public void commitMarker(int count) throws IOException { CommitMarkerCodec.writeMarker(internalWriter, count); }
public HDFSTransactionLogReaderV2(SequenceFile.Reader reader) { this.reader = reader; this.transactionEdits = new ArrayDeque<>(); this.key = new LongWritable(); this.commitMarkerCodec = new CommitMarkerCodec(); }
@Override public void close() throws IOException { if (closed) { return; } try { commitMarkerCodec.close(); } finally { reader.close(); closed = true; } }
private void writeNumWrites(SequenceFile.Writer writer, final int size) throws Exception { String key = TxConstants.TransactionLog.NUM_ENTRIES_APPENDED; CommitMarkerCodec.writeMarker(writer, size); }