private void setupState$(DataSourceDescription dsd, LocalStorageType storageType) { Id datasourceId = dsd.getId(); if ( zone.exists(datasourceId) ) { DataState dataState = zone.get(datasourceId); throw new DeltaConfigException("Local data source management already exists: "+dataState.getDatasourceName()); } DataState dataState = zone.create(datasourceId, dsd.getName(), dsd.getUri(), storageType); }
protected DataSourceDescription getDescription() { if ( description == null ) { description = getDescriptions().stream() .filter(dsd-> Objects.equals(dsd.getName(), dataSourceName) || Objects.equals(dsd.getUri(), dataSourceURI)) .findFirst().orElse(null); if ( description == null ) throw new CmdException("Source '"+dataSourceName+"' does not exist"); } return description; }
public String getDataSourceName() { return dsd.getName(); }
public Id getDsRef() { return dsd.getId() ; } }
@Test public void patchStore_1() { PatchStore ps = provider(); DataRegistry dataRegistry = ps.getDataRegistry(); DataSourceDescription dsdSetup = new DataSourceDescription(Id.create(), "ABC", "http://example/ABC"); PatchLog patchLog = ps.createLog(dsdSetup); Id logId = patchLog.getLogId(); assertEquals(dsdSetup.getId(), logId); assertNotNull(dataRegistry.getByName("ABC")); assertFalse(dataRegistry.isEmpty()); assertTrue(ps.logExists(dsdSetup.getId())); assertEquals(patchLog, ps.getLog(logId)); }
/** * Look for {@link DataSource DataSources} in a disk area given by {@code location}. * <p> * Scan the given area for directories (must have a config file), check they are enabled, * and deal with {@code log_type}. */ public static List<DataSourceDescription> scanForLogs(Path location, PatchStore ps) { // PatchStore's that rely on the scan of local directories and checking the "log_type" field. Pair<List<Path>, List<Path>> pair = scanDirectory(location); List<Path> dataSourcePaths = pair.getLeft(); List<Path> disabledDataSources = pair.getRight(); //dataSourcePaths.forEach(p->LOG.info("Data source paths: "+p)); disabledDataSources.forEach(p->LOG.info("Data source: "+p+" : Disabled")); List<DataSourceDescription> descriptions = ListUtils.toList (dataSourcePaths.stream() .map(p->{ // Extract name from disk name. String dsName = p.getFileName().toString(); // read config file. JsonObject sourceObj = JSON.read(p.resolve(FileNames.DS_CONFIG).toString()); DataSourceDescription dsd = DataSourceDescription.fromJson(sourceObj); if ( ! Objects.equals(dsName, dsd.getName()) ) throw new DeltaConfigException("Names do not match: directory="+dsName+", dsd="+dsd); return dsd; }) .filter(Objects::nonNull) ); return descriptions; }
if ( patchLogs.containsKey(dsd.getName()) ) { FmtLog.debug(LOGZK, "[%d] Found to exist", instance); return patchLogs.get(dsd.getName()); String dsName = dsd.getName(); if ( ! validateName(dsName) ) { String msg = String.format("Log name '%s' does not match regex '%s'", dsName, LogNameRegex); JsonObject dsdJson = dsd.asJson(); zkCreateSetJson(client, zkActiveLog, dsdJson);
public void recovery1() { PatchStore ps = provider(); // Match dsd2 below DataSourceDescription dsdSetup = new DataSourceDescription(Id.create(), "ABC", "http://example/ABC"); PatchLog patchLog = ps.createLog(dsdSetup); RDFPatch patch = RDFPatchOps.emptyPatch(); patchLog.append(patch); PatchLogInfo info = patchLog.getInfo(); Id id = patchLog.getLogId(); DataSourceDescription dsd = patchLog.getDescription(); DPS.resetSystem(); String name = dsd.getName(); PatchStore provider = provider(); // Same FileStore, different PatchLog? DataSourceDescription dsd2 = new DataSourceDescription(id, name, "http://example/ABC"); PatchLog patchLog1 = provider.connectLog(dsd2); PatchLogInfo info1 = patchLog1.getInfo(); assertEquals(info, info1); }
private DataSourceDescription descriptor(DataSource dataSource) { DataSourceDescription descr = new DataSourceDescription (dataSource.getId(), dataSource.getURI(), dataSource.getName()); return descr; }
private DataSourceDescription getDataSourceDescription(JsonObject arg) { JsonObject obj = rpc(DeltaConst.OP_DESCR_DS, arg); if ( obj.isEmpty() ) return null; return DataSourceDescription.fromJson(obj); }
public String getURI() { return dsDescription.getUri(); }
public String getName() { return dsDescription.getName(); }
public Id getId() { return dsDescription.getId(); }
@Test public void patchStore_2() { PatchStore ps = provider(); DataRegistry dataRegistry = ps.getDataRegistry(); DataSourceDescription dsdSetup = new DataSourceDescription(Id.create(), "ABC", "http://example/ABC"); PatchLog patchLog = ps.createLog(dsdSetup); assertEquals(dsdSetup.getId(), patchLog.getLogId()); Id logId = patchLog.getLogId(); assertFalse(dataRegistry.isEmpty()); assertNotNull(dataRegistry.getByName("ABC")); assertTrue(ps.logExists(dsdSetup.getId())); assertNotNull(ps.getLog(logId)); ps.release(patchLog); assertTrue(dataRegistry.isEmpty()); assertNull(dataRegistry.getByName("ABC")); assertFalse(ps.logExists(dsdSetup.getId())); assertNull(ps.getLog(logId)); }
@Override protected PatchLog patchLog() { try { String connectString = "localhost:" + server.getPort(); RetryPolicy policy = new ExponentialBackoffRetry(10000, 5); client = CuratorFrameworkFactory.builder() .connectString(connectString) .retryPolicy(policy) .build(); client.start(); //client.getConnectionStateListenable().addListener((c, newState)->System.out.println("** STATE CHANGED TO : " + newState)); client.blockUntilConnected(); LocalServerConfig config = LocalServers.configZk(connectString); PatchStore ps = new PatchStoreProviderZk().create(config); ps.initialize(new DataRegistry("TestPatchLogZk"), config); DataSourceDescription dsd = new DataSourceDescription(Id.create(), "ABC", "http://example/ABC"); PatchLog patchLog = ps.createLog(dsd); return patchLog; } catch (Exception ex) { ex.printStackTrace(); return null; } }
private List<DataSourceDescription> listDataSourcesZkPath(String logsPath) { List<DataSourceDescription> descriptions = new ArrayList<>(); List<String> logNames = Zk.zkSubNodes(client, logsPath); if ( logNames == null ) return Collections.emptyList(); for ( String name: logNames) { String logDsd = zkPath(ZkConst.pLogs, name, ZkConst.nDsd); JsonObject obj = zkFetchJson(client, logDsd); if ( obj == null ) { FmtLog.info(LOGZK, "[%d] listDataSourcesZkPath: %s: no DSD", instance, name); continue; } DataSourceDescription dsd = DataSourceDescription.fromJson(obj); descriptions.add(dsd); }; return descriptions; }