public WriteKey(Location item) { file = item.getDataFileId(); offset = item.getOffset(); // TODO: see if we can build a better hash hash = (int)(file ^ offset); }
public void writePayload(Location object, DataOutput dataOut) throws IOException { dataOut.writeInt(object.getDataFileId()); dataOut.writeInt(object.getOffset()); }
@Override public void writePayload(Location object, DataOutput dataOut) throws IOException { dataOut.writeInt(object.getDataFileId()); dataOut.writeInt(object.getOffset()); dataOut.writeInt(object.getSize()); }
KahaLocation convert(Location location) { KahaLocation rc = new KahaLocation(); rc.setLogId(location.getDataFileId()); rc.setOffset(location.getOffset()); return rc; }
KahaLocation convert(Location location) { KahaLocation rc = new KahaLocation(); rc.setLogId(location.getDataFileId()); rc.setOffset(location.getOffset()); return rc; }
public ByteSequence readRecord(Location location) throws IOException { if (!location.isValid()) { throw new IOException("Invalid location: " + location); } Journal.WriteCommand asyncWrite = inflightWrites.get(new Journal.WriteKey(location)); if (asyncWrite != null) { return asyncWrite.data; } try { if (location.getSize() == Location.NOT_SET) { file.seek(location.getOffset()); location.setSize(file.readInt()); location.setType(file.readByte()); } else { file.seek(location.getOffset() + Journal.RECORD_HEAD_SPACE); } if ((long)location.getOffset() + location.getSize() > dataFile.length) { throw new IOException("Invalid location size: " + location + ", size: " + location.getSize()); } byte[] data = new byte[location.getSize() - Journal.RECORD_HEAD_SPACE]; file.readFully(data); return new ByteSequence(data, 0, data.length); } catch (RuntimeException e) { throw new IOException("Invalid location: " + location + " : " + e, e); } }
public void updateRecord(Location location, ByteSequence data, boolean sync) throws IOException { file.seek(location.getOffset() + Journal.RECORD_HEAD_SPACE); int size = Math.min(data.getLength(), location.getSize()); file.write(data.getData(), data.getOffset(), size); if (sync) { file.sync(); } }
public void corruptRecoveryLocation(Location recoveryPosition) throws IOException { DataFile dataFile = getDataFile(recoveryPosition); // with corruption on recovery we have no faith in the content - slip to the next batch record or eof DataFileAccessor reader = accessorPool.openDataFileAccessor(dataFile); try { RandomAccessFile randomAccessFile = reader.getRaf().getRaf(); randomAccessFile.seek(recoveryPosition.getOffset() + 1); byte[] data = new byte[getWriteBatchSize()]; ByteSequence bs = new ByteSequence(data, 0, randomAccessFile.read(data)); int nextOffset = 0; if (findNextBatchRecord(bs, randomAccessFile) >= 0) { nextOffset = Math.toIntExact(randomAccessFile.getFilePointer() - bs.remaining()); } else { nextOffset = Math.toIntExact(randomAccessFile.length()); } Sequence sequence = new Sequence(recoveryPosition.getOffset(), nextOffset - 1); LOG.warn("Corrupt journal records found in '{}' between offsets: {}", dataFile.getFile(), sequence); // skip corruption on getNextLocation recoveryPosition.setOffset(nextOffset); recoveryPosition.setSize(-1); dataFile.corruptedBlocks.add(sequence); } catch (IOException e) { } finally { accessorPool.closeDataFileAccessor(reader); } }
public void readLocationDetails(Location location) throws IOException { Journal.WriteCommand asyncWrite = inflightWrites.get(new Journal.WriteKey(location)); if (asyncWrite != null) { location.setSize(asyncWrite.location.getSize()); location.setType(asyncWrite.location.getType()); } else { file.seek(location.getOffset()); location.setSize(file.readInt()); location.setType(file.readByte()); } }
if (size >= 0 && location.getOffset() + BATCH_CONTROL_RECORD_SIZE + size <= totalFileLength) { if (size == 0) { location.setOffset(location.getOffset() + BATCH_CONTROL_RECORD_SIZE + size); } else { Sequence sequence = new Sequence(location.getOffset(), nextOffset - 1); LOG.warn("Corrupt journal records found in '{}' between offsets: {}", dataFile.getFile(), sequence); dataFile.corruptedBlocks.add(sequence); dataFile.setLength(location.getOffset()); if (existingLen > dataFile.getLength()) { totalLength.addAndGet(dataFile.getLength() - existingLen); if (dataFile.corruptedBlocks.getTail().getLast() + 1 == location.getOffset()) { dataFile.setLength((int) dataFile.corruptedBlocks.removeLastSequence().getFirst());
} else { cur = new Location(location); cur.setOffset(location.getOffset() + location.getSize()); cur.setOffset(cur.getOffset() + cur.getSize()); if (dataFile.getLength() <= cur.getOffset()) { synchronized (currentDataFile) { dataFile = dataFile.getNext(); Sequence corruptedRange = dataFile.corruptedBlocks.get(cur.getOffset()); if (corruptedRange != null) {
public WriteKey(Location item) { file = item.getDataFileId(); offset = item.getOffset(); // TODO: see if we can build a better hash hash = (int)(file ^ offset); }
@Override public void writePayload(Location object, DataOutput dataOut) throws IOException { dataOut.writeInt(object.getDataFileId()); dataOut.writeInt(object.getOffset()); dataOut.writeInt(object.getSize()); }
@Override public void writePayload(Location object, DataOutput dataOut) throws IOException { dataOut.writeInt(object.getDataFileId()); dataOut.writeInt(object.getOffset()); dataOut.writeInt(object.getSize()); }
@Override public void writePayload(Location object, DataOutput dataOut) throws IOException { dataOut.writeInt(object.getDataFileId()); dataOut.writeInt(object.getOffset()); dataOut.writeInt(object.getSize()); }
KahaLocation convert(Location location) { KahaLocation rc = new KahaLocation(); rc.setLogId(location.getDataFileId()); rc.setOffset(location.getOffset()); return rc; }
KahaLocation convert(Location location) { KahaLocation rc = new KahaLocation(); rc.setLogId(location.getDataFileId()); rc.setOffset(location.getOffset()); return rc; }
KahaLocation convert(Location location) { KahaLocation rc = new KahaLocation(); rc.setLogId(location.getDataFileId()); rc.setOffset(location.getOffset()); return rc; }
public void updateRecord(Location location, ByteSequence data, boolean sync) throws IOException { file.seek(location.getOffset() + Journal.RECORD_HEAD_SPACE); int size = Math.min(data.getLength(), location.getSize()); file.write(data.getData(), data.getOffset(), size); if (sync) { file.sync(); } }