public static AzureSegmentArchiveEntry toIndexEntry(Map<String, String> metadata, int length) { UUID uuid = UUID.fromString(metadata.get(METADATA_SEGMENT_UUID)); long msb = uuid.getMostSignificantBits(); long lsb = uuid.getLeastSignificantBits(); int position = Integer.parseInt(metadata.get(METADATA_SEGMENT_POSITION)); int generation = Integer.parseInt(metadata.get(METADATA_SEGMENT_GENERATION)); int fullGeneration = Integer.parseInt(metadata.get(METADATA_SEGMENT_FULL_GENERATION)); boolean compacted = Boolean.parseBoolean(metadata.get(METADATA_SEGMENT_COMPACTED)); return new AzureSegmentArchiveEntry(msb, lsb, position, length, generation, fullGeneration, compacted); }
public static HashMap<String, String> toSegmentMetadata(AzureSegmentArchiveEntry indexEntry) { HashMap<String, String> map = new HashMap<>(); map.put(METADATA_TYPE, TYPE_SEGMENT); map.put(METADATA_SEGMENT_UUID, new UUID(indexEntry.getMsb(), indexEntry.getLsb()).toString()); map.put(METADATA_SEGMENT_POSITION, String.valueOf(indexEntry.getPosition())); map.put(METADATA_SEGMENT_GENERATION, String.valueOf(indexEntry.getGeneration())); map.put(METADATA_SEGMENT_FULL_GENERATION, String.valueOf(indexEntry.getFullGeneration())); map.put(METADATA_SEGMENT_COMPACTED, String.valueOf(indexEntry.isCompacted())); return map; }
public static String getSegmentFileName(AzureSegmentArchiveEntry indexEntry) { return getSegmentFileName(indexEntry.getPosition(), indexEntry.getMsb(), indexEntry.getLsb()); }
@Override public Buffer readSegment(long msb, long lsb) throws IOException { AzureSegmentArchiveEntry indexEntry = index.get(new UUID(msb, lsb)); if (indexEntry == null) { return null; } Buffer buffer; if (OFF_HEAP) { buffer = Buffer.allocateDirect(indexEntry.getLength()); } else { buffer = Buffer.allocate(indexEntry.getLength()); } ioMonitor.beforeSegmentRead(pathAsFile(), msb, lsb, indexEntry.getLength()); Stopwatch stopwatch = Stopwatch.createStarted(); readBufferFully(getBlob(getSegmentFileName(indexEntry)), buffer); long elapsed = stopwatch.elapsed(TimeUnit.NANOSECONDS); ioMonitor.afterSegmentRead(pathAsFile(), msb, lsb, indexEntry.getLength(), elapsed); return buffer; }
public static String getSegmentFileName(AzureSegmentArchiveEntry indexEntry) { return getSegmentFileName(indexEntry.getPosition(), indexEntry.getMsb(), indexEntry.getLsb()); }
@Override public Buffer readSegment(long msb, long lsb) throws IOException { AzureSegmentArchiveEntry indexEntry = index.get(new UUID(msb, lsb)); if (indexEntry == null) { return null; } Buffer buffer; if (OFF_HEAP) { buffer = Buffer.allocateDirect(indexEntry.getLength()); } else { buffer = Buffer.allocate(indexEntry.getLength()); } ioMonitor.beforeSegmentRead(pathAsFile(), msb, lsb, indexEntry.getLength()); Stopwatch stopwatch = Stopwatch.createStarted(); readBufferFully(getBlob(getSegmentFileName(indexEntry)), buffer); long elapsed = stopwatch.elapsed(TimeUnit.NANOSECONDS); ioMonitor.afterSegmentRead(pathAsFile(), msb, lsb, indexEntry.getLength(), elapsed); return buffer; }
public static HashMap<String, String> toSegmentMetadata(AzureSegmentArchiveEntry indexEntry) { HashMap<String, String> map = new HashMap<>(); map.put(METADATA_TYPE, TYPE_SEGMENT); map.put(METADATA_SEGMENT_UUID, new UUID(indexEntry.getMsb(), indexEntry.getLsb()).toString()); map.put(METADATA_SEGMENT_POSITION, String.valueOf(indexEntry.getPosition())); map.put(METADATA_SEGMENT_GENERATION, String.valueOf(indexEntry.getGeneration())); map.put(METADATA_SEGMENT_FULL_GENERATION, String.valueOf(indexEntry.getFullGeneration())); map.put(METADATA_SEGMENT_COMPACTED, String.valueOf(indexEntry.isCompacted())); return map; }
public static AzureSegmentArchiveEntry toIndexEntry(Map<String, String> metadata, int length) { UUID uuid = UUID.fromString(metadata.get(METADATA_SEGMENT_UUID)); long msb = uuid.getMostSignificantBits(); long lsb = uuid.getLeastSignificantBits(); int position = Integer.parseInt(metadata.get(METADATA_SEGMENT_POSITION)); int generation = Integer.parseInt(metadata.get(METADATA_SEGMENT_GENERATION)); int fullGeneration = Integer.parseInt(metadata.get(METADATA_SEGMENT_FULL_GENERATION)); boolean compacted = Boolean.parseBoolean(metadata.get(METADATA_SEGMENT_COMPACTED)); return new AzureSegmentArchiveEntry(msb, lsb, position, length, generation, fullGeneration, compacted); }
@Override public Buffer readSegment(long msb, long lsb) throws IOException { UUID uuid = new UUID(msb, lsb); Optional<SegmentWriteAction> segment = queue.map(q -> q.read(uuid)); if (segment.isPresent()) { return segment.get().toBuffer(); } AzureSegmentArchiveEntry indexEntry = index.get(new UUID(msb, lsb)); if (indexEntry == null) { return null; } Buffer buffer; if (OFF_HEAP) { buffer = Buffer.allocateDirect(indexEntry.getLength()); } else { buffer = Buffer.allocate(indexEntry.getLength()); } readBufferFully(getBlob(getSegmentFileName(indexEntry)), buffer); return buffer; }
AzureSegmentArchiveReader(CloudBlobDirectory archiveDirectory, IOMonitor ioMonitor) throws IOException { this.archiveDirectory = archiveDirectory; this.ioMonitor = ioMonitor; long length = 0; for (CloudBlob blob : AzureUtilities.getBlobs(archiveDirectory).collect(Collectors.toList())) { Map<String, String> metadata = blob.getMetadata(); if (AzureBlobMetadata.isSegment(metadata)) { AzureSegmentArchiveEntry indexEntry = AzureBlobMetadata.toIndexEntry(metadata, (int) blob.getProperties().getLength()); index.put(new UUID(indexEntry.getMsb(), indexEntry.getLsb()), indexEntry); } length += blob.getProperties().getLength(); } this.length = length; }
private static AzureSegmentArchiveEntry tarEntry(long i) { return new AzureSegmentArchiveEntry(0, i, 0, 0, 0, 0, false); }
@Override public Buffer readSegment(long msb, long lsb) throws IOException { UUID uuid = new UUID(msb, lsb); Optional<SegmentWriteAction> segment = queue.map(q -> q.read(uuid)); if (segment.isPresent()) { return segment.get().toBuffer(); } AzureSegmentArchiveEntry indexEntry = index.get(new UUID(msb, lsb)); if (indexEntry == null) { return null; } Buffer buffer; if (OFF_HEAP) { buffer = Buffer.allocateDirect(indexEntry.getLength()); } else { buffer = Buffer.allocate(indexEntry.getLength()); } readBufferFully(getBlob(getSegmentFileName(indexEntry)), buffer); return buffer; }
AzureSegmentArchiveReader(CloudBlobDirectory archiveDirectory, IOMonitor ioMonitor) throws IOException { this.archiveDirectory = archiveDirectory; this.ioMonitor = ioMonitor; long length = 0; for (CloudBlob blob : AzureUtilities.getBlobs(archiveDirectory).collect(Collectors.toList())) { Map<String, String> metadata = blob.getMetadata(); if (AzureBlobMetadata.isSegment(metadata)) { AzureSegmentArchiveEntry indexEntry = AzureBlobMetadata.toIndexEntry(metadata, (int) blob.getProperties().getLength()); index.put(new UUID(indexEntry.getMsb(), indexEntry.getLsb()), indexEntry); } length += blob.getProperties().getLength(); } this.length = length; }
@Override public void writeSegment(long msb, long lsb, byte[] data, int offset, int size, int generation, int fullGeneration, boolean compacted) throws IOException { created = true; AzureSegmentArchiveEntry entry = new AzureSegmentArchiveEntry(msb, lsb, entries++, size, generation, fullGeneration, compacted); if (queue.isPresent()) { queue.get().addToQueue(entry, data, offset, size); } else { doWriteEntry(entry, data, offset, size); } index.put(new UUID(msb, lsb), entry); totalLength += size; monitor.written(size); }
private void doWriteEntry(AzureSegmentArchiveEntry indexEntry, byte[] data, int offset, int size) throws IOException { long msb = indexEntry.getMsb(); long lsb = indexEntry.getLsb(); ioMonitor.beforeSegmentWrite(pathAsFile(), msb, lsb, size); Stopwatch stopwatch = Stopwatch.createStarted(); try { CloudBlockBlob blob = getBlob(getSegmentFileName(indexEntry)); blob.setMetadata(AzureBlobMetadata.toSegmentMetadata(indexEntry)); blob.uploadFromByteArray(data, offset, size); blob.uploadMetadata(); } catch (StorageException e) { throw new IOException(e); } ioMonitor.afterSegmentWrite(pathAsFile(), msb, lsb, size, stopwatch.elapsed(TimeUnit.NANOSECONDS)); }
@Override public void writeSegment(long msb, long lsb, byte[] data, int offset, int size, int generation, int fullGeneration, boolean compacted) throws IOException { created = true; AzureSegmentArchiveEntry entry = new AzureSegmentArchiveEntry(msb, lsb, entries++, size, generation, fullGeneration, compacted); if (queue.isPresent()) { queue.get().addToQueue(entry, data, offset, size); } else { doWriteEntry(entry, data, offset, size); } index.put(new UUID(msb, lsb), entry); totalLength += size; monitor.written(size); }
private void doWriteEntry(AzureSegmentArchiveEntry indexEntry, byte[] data, int offset, int size) throws IOException { long msb = indexEntry.getMsb(); long lsb = indexEntry.getLsb(); ioMonitor.beforeSegmentWrite(pathAsFile(), msb, lsb, size); Stopwatch stopwatch = Stopwatch.createStarted(); try { CloudBlockBlob blob = getBlob(getSegmentFileName(indexEntry)); blob.setMetadata(AzureBlobMetadata.toSegmentMetadata(indexEntry)); blob.uploadFromByteArray(data, offset, size); blob.uploadMetadata(); } catch (StorageException e) { throw new IOException(e); } ioMonitor.afterSegmentWrite(pathAsFile(), msb, lsb, size, stopwatch.elapsed(TimeUnit.NANOSECONDS)); }