private void populateTransactionEdits() throws IOException { // read the marker to determine numEntries to read. int numEntries = 0; try { // can throw EOFException if reading of incomplete commit marker, no other action required since we can safely // ignore this numEntries = commitMarkerCodec.readMarker(reader); } catch (EOFException e) { LOG.warn("Reached EOF in log while trying to read commit marker", e); } for (int i = 0; i < numEntries; i++) { TransactionEdit edit = new TransactionEdit(); try { if (reader.next(key, edit)) { transactionEdits.add(edit); } else { throw new EOFException("Attempt to read TransactionEdit failed."); } } catch (EOFException e) { // we have reached EOF before reading back numEntries, we clear the partial list and return. LOG.warn("Reached EOF in log before reading {} entries. Ignoring all {} edits since the last marker", numEntries, transactionEdits.size(), e); transactionEdits.clear(); } } } }
private void populateTransactionEdits() throws IOException { // read the marker to determine numEntries to read. int numEntries = 0; try { // can throw EOFException if reading of incomplete commit marker, no other action required since we can safely // ignore this numEntries = commitMarkerCodec.readMarker(reader); } catch (EOFException e) { LOG.warn("Reached EOF in log while trying to read commit marker", e); } for (int i = 0; i < numEntries; i++) { TransactionEdit edit = new TransactionEdit(); try { if (reader.next(key, edit)) { transactionEdits.add(edit); } else { throw new EOFException("Attempt to read TransactionEdit failed."); } } catch (EOFException e) { // we have reached EOF before reading back numEntries, we clear the partial list and return. LOG.warn("Reached EOF in log before reading {} entries. Ignoring all {} edits since the last marker", numEntries, transactionEdits.size(), e); transactionEdits.clear(); } } } }
@Test public void testRandomCommitMarkers() throws Exception { List<Integer> randomInts = new ArrayList<>(); Path newLog = new Path(TMP_FOLDER.newFolder().getAbsolutePath(), LOG_FILE); // Write a bunch of random commit markers try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, newLog, LongWritable.class, LongWritable.class, SequenceFile.CompressionType.NONE)) { for (int i = 0; i < 1000; i++) { int randomNum = RANDOM.nextInt(Integer.MAX_VALUE); CommitMarkerCodec.writeMarker(writer, randomNum); randomInts.add(randomNum); } writer.hflush(); writer.hsync(); } // Read the commit markers back to verify the marker try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, newLog, conf); CommitMarkerCodec markerCodec = new CommitMarkerCodec()) { for (int num : randomInts) { Assert.assertEquals(num, markerCodec.readMarker(reader)); } } }
@Test public void testIncorrectCommitMarker() throws Exception { Path newLog = new Path(TMP_FOLDER.newFolder().getAbsolutePath(), LOG_FILE); // Write an incorrect marker try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, newLog, LongWritable.class, LongWritable.class, SequenceFile.CompressionType.NONE)) { String invalidKey = "IncorrectKey"; SequenceFile.ValueBytes valueBytes = new CommitMarkerCodec.CommitEntriesCount(100); writer.appendRaw(invalidKey.getBytes(), 0, invalidKey.length(), valueBytes); writer.hflush(); writer.hsync(); } // Read the commit markers back to verify the marker try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, newLog, conf); CommitMarkerCodec markerCodec = new CommitMarkerCodec()) { try { markerCodec.readMarker(reader); Assert.fail("Expected an IOException to be thrown"); } catch (IOException e) { // expected } } } }
@Test public void testIncompleteCommitMarker() throws Exception { Path newLog = new Path(TMP_FOLDER.newFolder().getAbsolutePath(), LOG_FILE); try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, newLog, LongWritable.class, LongWritable.class, SequenceFile.CompressionType.NONE)) { String key = TxConstants.TransactionLog.NUM_ENTRIES_APPENDED; SequenceFile.ValueBytes valueBytes = new IncompleteValueBytes(); writer.appendRaw(key.getBytes(), 0, key.length(), valueBytes); writer.hflush(); writer.hsync(); } // Read the incomplete commit marker try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, newLog, conf); CommitMarkerCodec markerCodec = new CommitMarkerCodec()) { try { markerCodec.readMarker(reader); Assert.fail("Expected EOF Exception to be thrown"); } catch (EOFException e) { // expected since we didn't write the value bytes } } }