public static SegmentNodeStorePersistence newSegmentNodeStorePersistence(SegmentStoreType storeType, String pathOrUri) { SegmentNodeStorePersistence persistence = null; switch (storeType) { case AZURE: CloudBlobDirectory cloudBlobDirectory = createCloudBlobDirectory(pathOrUri.substring(3)); persistence = new AzurePersistence(cloudBlobDirectory); break; default: persistence = new TarPersistence(new File(pathOrUri)); } return persistence; }
public static SegmentNodeStorePersistence newSegmentNodeStorePersistence(SegmentStoreType storeType, String pathOrUri) { SegmentNodeStorePersistence persistence = null; switch (storeType) { case AZURE: CloudBlobDirectory cloudBlobDirectory = createCloudBlobDirectory(pathOrUri.substring(3)); persistence = new AzurePersistence(cloudBlobDirectory); break; default: persistence = new TarPersistence(new File(pathOrUri)); } return persistence; }
private AzurePersistence createAzurePersistence() throws StorageException, URISyntaxException, InvalidKeyException { CloudBlobDirectory cloudBlobDirectory = null; if (accountName != null && uri != null) { String key = System.getenv("AZURE_SECRET_KEY"); StorageCredentials credentials = new StorageCredentialsAccountAndKey(accountName, key); cloudBlobDirectory = AzureUtilities.cloudBlobDirectoryFrom(credentials, uri, dir); } else if (connectionString != null && containerName != null) { cloudBlobDirectory = AzureUtilities.cloudBlobDirectoryFrom(connectionString, containerName, dir); } if (cloudBlobDirectory == null) { throw new IllegalArgumentException("Could not connect to Azure storage. Too few connection parameters specified!"); } return new AzurePersistence(cloudBlobDirectory); }
private AzurePersistence createAzurePersistence() throws StorageException, URISyntaxException, InvalidKeyException { CloudBlobDirectory cloudBlobDirectory = null; if (accountName != null && uri != null) { String key = System.getenv("AZURE_SECRET_KEY"); StorageCredentials credentials = new StorageCredentialsAccountAndKey(accountName, key); cloudBlobDirectory = AzureUtilities.cloudBlobDirectoryFrom(credentials, uri, dir); } else if (connectionString != null && containerName != null) { cloudBlobDirectory = AzureUtilities.cloudBlobDirectoryFrom(connectionString, containerName, dir); } if (cloudBlobDirectory == null) { throw new IllegalArgumentException("Could not connect to Azure storage. Too few connection parameters specified!"); } return new AzurePersistence(cloudBlobDirectory); }
@Override protected SegmentNodeStorePersistence getPersistence() throws IOException { try { return new AzurePersistence(container.getDirectoryReference("oak")); } catch (URISyntaxException e) { throw new IOException(e); } } }
@Override protected SegmentNodeStorePersistence getPersistence() throws Exception { return new AzurePersistence(container.getDirectoryReference("oak")); }
private static SegmentNodeStorePersistence createAzurePersistence(Configuration configuration) throws IOException { try { StringBuilder connectionString = new StringBuilder(); if (configuration.connectionURL() == null || configuration.connectionURL().trim().isEmpty()) { connectionString.append("DefaultEndpointsProtocol=https;"); connectionString.append("AccountName=").append(configuration.accountName()).append(';'); connectionString.append("AccountKey=").append(configuration.accessKey()).append(';'); } else { connectionString.append(configuration.connectionURL()); } log.info("Connection string: '{}'", connectionString.toString()); CloudStorageAccount cloud = CloudStorageAccount.parse(connectionString.toString()); CloudBlobContainer container = cloud.createCloudBlobClient().getContainerReference(configuration.containerName()); container.createIfNotExists(); String path = configuration.rootPath(); if (path != null && path.length() > 0 && path.charAt(0) == '/') { path = path.substring(1); } AzurePersistence persistence = new AzurePersistence(container.getDirectoryReference(path)); return persistence; } catch (StorageException | URISyntaxException | InvalidKeyException e) { throw new IOException(e); } }
public Builder withSource(CloudBlobDirectory dir) throws URISyntaxException, StorageException { this.source = new AzurePersistence(dir); this.sourceName = storeDescription(SegmentStoreType.AZURE, dir.getContainer().getName() + "/" + dir.getPrefix()); return this; }
public Builder withTarget(CloudBlobDirectory dir) throws URISyntaxException, StorageException { this.target = new AzurePersistence(dir); this.targetName = storeDescription(SegmentStoreType.AZURE, dir.getContainer().getName() + "/" + dir.getPrefix()); return this; }
protected SegmentNodeStorePersistence getAzurePersistence() throws Exception { return new AzurePersistence(azurite.getContainer(AZURE_CONTAINER).getDirectoryReference(AZURE_DIRECTORY)); }
@Test public void testRecovery() throws StorageException, URISyntaxException, IOException { SegmentArchiveManager manager = new AzurePersistence(container.getDirectoryReference("oak")).createArchiveManager(false, false, new IOMonitorAdapter(), new FileStoreMonitorAdapter()); SegmentArchiveWriter writer = manager.create("data00000a.tar"); List<UUID> uuids = new ArrayList<>(); for (int i = 0; i < 10; i++) { UUID u = UUID.randomUUID(); writer.writeSegment(u.getMostSignificantBits(), u.getLeastSignificantBits(), new byte[10], 0, 10, 0, 0, false); uuids.add(u); } writer.flush(); writer.close(); container.getBlockBlobReference("oak/data00000a.tar/0005." + uuids.get(5).toString()).delete(); LinkedHashMap<UUID, byte[]> recovered = new LinkedHashMap<>(); manager.recoverEntries("data00000a.tar", recovered); assertEquals(uuids.subList(0, 5), newArrayList(recovered.keySet())); }
@Test public void testManifest() throws URISyntaxException, IOException { ManifestFile manifestFile = new AzurePersistence(container.getDirectoryReference("oak")).getManifestFile(); assertFalse(manifestFile.exists()); Properties props = new Properties(); props.setProperty("xyz", "abc"); props.setProperty("version", "123"); manifestFile.save(props); Properties loaded = manifestFile.load(); assertEquals(props, loaded); }
@Before @Override public void setUp() throws IOException { try { container = azurite.getContainer("oak-test"); archiveManager = new AzurePersistence(container.getDirectoryReference("oak")).createArchiveManager(true, false, new IOMonitorAdapter(), new FileStoreMonitorAdapter()); } catch (StorageException | InvalidKeyException | URISyntaxException e) { throw new IOException(e); } }
@Before @Override public void setUp() throws IOException { try { monitor = new TestFileStoreMonitor(); container = azurite.getContainer("oak-test"); archiveManager = new AzurePersistence(container.getDirectoryReference("oak")).createArchiveManager(true, false, new IOMonitorAdapter(), monitor); } catch (StorageException | InvalidKeyException | URISyntaxException e) { throw new IOException(e); } } }
@Override public NodeStore open() throws IOException { AzurePersistence azPersistence = null; try { azPersistence = new AzurePersistence(container.getDirectoryReference(dir)); } catch (URISyntaxException e) { throw new IllegalStateException(e); } FileStoreBuilder builder = FileStoreBuilder.fileStoreBuilder(Files.createTempDir()) .withCustomPersistence(azPersistence).withMemoryMapping(false); if (blob != null) { builder.withBlobStore(blob.open()); } try { fs = builder.build(); } catch (InvalidFileStoreVersionException e) { throw new IllegalStateException(e); } return SegmentNodeStoreBuilders.builder(fs).build(); }
@Override public Oak getOak(int clusterId) throws Exception { FileStoreBuilder fileStoreBuilder = fileStoreBuilder(parentPath) .withMaxFileSize(maxFileSize) .withSegmentCacheSize(segmentCacheSize) .withMemoryMapping(memoryMapping); if (azureConnectionString != null) { CloudStorageAccount cloud = CloudStorageAccount.parse(azureConnectionString); CloudBlobContainer container = cloud.createCloudBlobClient().getContainerReference(azureContainerName); container.createIfNotExists(); CloudBlobDirectory directory = container.getDirectoryReference(azureRootPath); fileStoreBuilder.withCustomPersistence(new AzurePersistence(directory)); } if (useBlobStore) { FileDataStore fds = new FileDataStore(); fds.setMinRecordLength(4092); fds.init(parentPath.getAbsolutePath()); BlobStore blobStore = new DataStoreBlobStore(fds); fileStoreBuilder.withBlobStore(blobStore); } FileStore fs = fileStoreBuilder.build(); return newOak(SegmentNodeStoreBuilders.builder(fs).build()); }
@Override public Oak getOak(int clusterId) throws Exception { FileStoreBuilder fileStoreBuilder = fileStoreBuilder(parentPath) .withMaxFileSize(maxFileSize) .withSegmentCacheSize(segmentCacheSize) .withMemoryMapping(memoryMapping); if (azureConnectionString != null) { CloudStorageAccount cloud = CloudStorageAccount.parse(azureConnectionString); CloudBlobContainer container = cloud.createCloudBlobClient().getContainerReference(azureContainerName); container.createIfNotExists(); CloudBlobDirectory directory = container.getDirectoryReference(azureRootPath); fileStoreBuilder.withCustomPersistence(new AzurePersistence(directory)); } if (useBlobStore) { FileDataStore fds = new FileDataStore(); fds.setMinRecordLength(4092); fds.init(parentPath.getAbsolutePath()); BlobStore blobStore = new DataStoreBlobStore(fds); fileStoreBuilder.withBlobStore(blobStore); } FileStore fs = fileStoreBuilder.build(); return newOak(SegmentNodeStoreBuilders.builder(fs).build()); }
@Before @Override public void setUp() throws Exception { container = azurite.getContainer("oak-test"); tarFiles = TarFiles.builder() .withDirectory(folder.newFolder()) .withTarRecovery((id, data, recovery) -> { // Intentionally left blank }) .withIOMonitor(new IOMonitorAdapter()) .withFileStoreMonitor(new FileStoreMonitorAdapter()) .withMaxFileSize(MAX_FILE_SIZE) .withPersistence(new AzurePersistence(container.getDirectoryReference("oak"))) .build(); } }
@Test public void testUncleanStop() throws URISyntaxException, IOException, InvalidFileStoreVersionException, CommitFailedException, StorageException { AzurePersistence p = new AzurePersistence(container.getDirectoryReference("oak")); FileStore fs = FileStoreBuilder.fileStoreBuilder(new File("target")).withCustomPersistence(p).build(); SegmentNodeStore segmentNodeStore = SegmentNodeStoreBuilders.builder(fs).build(); NodeBuilder builder = segmentNodeStore.getRoot().builder(); builder.setProperty("foo", "bar"); segmentNodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY); fs.close(); container.getBlockBlobReference("oak/data00000a.tar/closed").delete(); container.getBlockBlobReference("oak/data00000a.tar/data00000a.tar.brf").delete(); container.getBlockBlobReference("oak/data00000a.tar/data00000a.tar.gph").delete(); fs = FileStoreBuilder.fileStoreBuilder(new File("target")).withCustomPersistence(p).build(); segmentNodeStore = SegmentNodeStoreBuilders.builder(fs).build(); assertEquals("bar", segmentNodeStore.getRoot().getString("foo")); fs.close(); } }
@Before public void setup() throws IOException, InvalidFileStoreVersionException, CommitFailedException, URISyntaxException, InvalidKeyException, StorageException { SegmentNodeStorePersistence sharedPersistence = new AzurePersistence(azurite.getContainer("oak-test").getDirectoryReference("oak")); baseFileStore = FileStoreBuilder .fileStoreBuilder(folder.newFolder()) .withCustomPersistence(sharedPersistence) .build(); base = SegmentNodeStoreBuilders.builder(baseFileStore).build(); NodeBuilder builder = base.getRoot().builder(); builder.child("foo").child("bar").setProperty("version", "v1"); base.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY); baseFileStore.flush(); SegmentNodeStorePersistence localPersistence = new TarPersistence(folder.newFolder()); splitPersistence = new SplitPersistence(sharedPersistence, localPersistence); splitFileStore = FileStoreBuilder .fileStoreBuilder(folder.newFolder()) .withCustomPersistence(splitPersistence) .build(); split = SegmentNodeStoreBuilders.builder(splitFileStore).build(); }