public UUID getUuid() { return new UUID(indexEntry.getMsb(), indexEntry.getLsb()); }
public UUID getUuid() { return new UUID(indexEntry.getMsb(), indexEntry.getLsb()); }
public static HashMap<String, String> toSegmentMetadata(AzureSegmentArchiveEntry indexEntry) { HashMap<String, String> map = new HashMap<>(); map.put(METADATA_TYPE, TYPE_SEGMENT); map.put(METADATA_SEGMENT_UUID, new UUID(indexEntry.getMsb(), indexEntry.getLsb()).toString()); map.put(METADATA_SEGMENT_POSITION, String.valueOf(indexEntry.getPosition())); map.put(METADATA_SEGMENT_GENERATION, String.valueOf(indexEntry.getGeneration())); map.put(METADATA_SEGMENT_FULL_GENERATION, String.valueOf(indexEntry.getFullGeneration())); map.put(METADATA_SEGMENT_COMPACTED, String.valueOf(indexEntry.isCompacted())); return map; }
public static HashMap<String, String> toSegmentMetadata(AzureSegmentArchiveEntry indexEntry) { HashMap<String, String> map = new HashMap<>(); map.put(METADATA_TYPE, TYPE_SEGMENT); map.put(METADATA_SEGMENT_UUID, new UUID(indexEntry.getMsb(), indexEntry.getLsb()).toString()); map.put(METADATA_SEGMENT_POSITION, String.valueOf(indexEntry.getPosition())); map.put(METADATA_SEGMENT_GENERATION, String.valueOf(indexEntry.getGeneration())); map.put(METADATA_SEGMENT_FULL_GENERATION, String.valueOf(indexEntry.getFullGeneration())); map.put(METADATA_SEGMENT_COMPACTED, String.valueOf(indexEntry.isCompacted())); return map; }
public static String getSegmentFileName(AzureSegmentArchiveEntry indexEntry) { return getSegmentFileName(indexEntry.getPosition(), indexEntry.getMsb(), indexEntry.getLsb()); }
public static String getSegmentFileName(AzureSegmentArchiveEntry indexEntry) { return getSegmentFileName(indexEntry.getPosition(), indexEntry.getMsb(), indexEntry.getLsb()); }
AzureSegmentArchiveReader(CloudBlobDirectory archiveDirectory, IOMonitor ioMonitor) throws IOException { this.archiveDirectory = archiveDirectory; this.ioMonitor = ioMonitor; long length = 0; for (CloudBlob blob : AzureUtilities.getBlobs(archiveDirectory).collect(Collectors.toList())) { Map<String, String> metadata = blob.getMetadata(); if (AzureBlobMetadata.isSegment(metadata)) { AzureSegmentArchiveEntry indexEntry = AzureBlobMetadata.toIndexEntry(metadata, (int) blob.getProperties().getLength()); index.put(new UUID(indexEntry.getMsb(), indexEntry.getLsb()), indexEntry); } length += blob.getProperties().getLength(); } this.length = length; }
AzureSegmentArchiveReader(CloudBlobDirectory archiveDirectory, IOMonitor ioMonitor) throws IOException { this.archiveDirectory = archiveDirectory; this.ioMonitor = ioMonitor; long length = 0; for (CloudBlob blob : AzureUtilities.getBlobs(archiveDirectory).collect(Collectors.toList())) { Map<String, String> metadata = blob.getMetadata(); if (AzureBlobMetadata.isSegment(metadata)) { AzureSegmentArchiveEntry indexEntry = AzureBlobMetadata.toIndexEntry(metadata, (int) blob.getProperties().getLength()); index.put(new UUID(indexEntry.getMsb(), indexEntry.getLsb()), indexEntry); } length += blob.getProperties().getLength(); } this.length = length; }
private void doWriteEntry(AzureSegmentArchiveEntry indexEntry, byte[] data, int offset, int size) throws IOException { long msb = indexEntry.getMsb(); long lsb = indexEntry.getLsb(); ioMonitor.beforeSegmentWrite(pathAsFile(), msb, lsb, size); Stopwatch stopwatch = Stopwatch.createStarted(); try { CloudBlockBlob blob = getBlob(getSegmentFileName(indexEntry)); blob.setMetadata(AzureBlobMetadata.toSegmentMetadata(indexEntry)); blob.uploadFromByteArray(data, offset, size); blob.uploadMetadata(); } catch (StorageException e) { throw new IOException(e); } ioMonitor.afterSegmentWrite(pathAsFile(), msb, lsb, size, stopwatch.elapsed(TimeUnit.NANOSECONDS)); }
private void doWriteEntry(AzureSegmentArchiveEntry indexEntry, byte[] data, int offset, int size) throws IOException { long msb = indexEntry.getMsb(); long lsb = indexEntry.getLsb(); ioMonitor.beforeSegmentWrite(pathAsFile(), msb, lsb, size); Stopwatch stopwatch = Stopwatch.createStarted(); try { CloudBlockBlob blob = getBlob(getSegmentFileName(indexEntry)); blob.setMetadata(AzureBlobMetadata.toSegmentMetadata(indexEntry)); blob.uploadFromByteArray(data, offset, size); blob.uploadMetadata(); } catch (StorageException e) { throw new IOException(e); } ioMonitor.afterSegmentWrite(pathAsFile(), msb, lsb, size, stopwatch.elapsed(TimeUnit.NANOSECONDS)); }
} catch (InterruptedException e) { added.add(new UUID(tarEntry.getMsb(), tarEntry.getLsb())); });
throw new IOException(); added.add(new UUID(tarEntry.getMsb(), tarEntry.getLsb())); });
@Test public void testQueue() throws IOException, InterruptedException { Set<UUID> added = Collections.synchronizedSet(new HashSet<>()); Semaphore semaphore = new Semaphore(0); queue = new SegmentWriteQueue((tarEntry, data, offset, size) -> { try { semaphore.acquire(); } catch (InterruptedException e) { } added.add(new UUID(tarEntry.getMsb(), tarEntry.getLsb())); }); for (int i = 0; i < 10; i++) { queue.addToQueue(tarEntry(i), EMPTY_DATA, 0, 0); } for (int i = 0; i < 10; i++) { assertNotNull("Segments should be available for read", queue.read(uuid(i))); } assertFalse("Queue shouldn't be empty", queue.isEmpty()); semaphore.release(Integer.MAX_VALUE); while (!queue.isEmpty()) { Thread.sleep(10); } assertEquals("There should be 10 segments consumed",10, added.size()); for (int i = 0; i < 10; i++) { assertTrue("Missing consumed segment", added.contains(uuid(i))); } }