@Override public ModifiableConfiguration getJanusGraphConfiguration(String graphName, Class<?> test, String testMethodName) { return GraphDatabaseConfiguration.buildGraphConfiguration() .set(SimpleBulkPlacementStrategy.CONCURRENT_PARTITIONS,1) .set(GraphDatabaseConfiguration.CLUSTER_MAX_PARTITIONS, 2) .set(GraphDatabaseConfiguration.IDAUTHORITY_CAV_BITS,0); }
@Override public WriteConfiguration getConfiguration() { WriteConfiguration config = getBaseConfiguration(); ModifiableConfiguration modifiableConfiguration = new ModifiableConfiguration(GraphDatabaseConfiguration.ROOT_NS,config, BasicConfiguration.Restriction.NONE); modifiableConfiguration.set(BASIC_METRICS,true); modifiableConfiguration.set(METRICS_MERGE_STORES,false); modifiableConfiguration.set(PROPERTY_PREFETCHING,false); modifiableConfiguration.set(DB_CACHE, false); return config; }
@Override public WriteConfiguration getConfiguration() { WriteConfiguration config = getBaseConfiguration(); ModifiableConfiguration modifiableConfiguration = new ModifiableConfiguration(GraphDatabaseConfiguration.ROOT_NS,config, BasicConfiguration.Restriction.NONE); // Let GraphDatabaseConfiguration's config freezer set CLUSTER_PARTITION //modifiableConfiguration.set(GraphDatabaseConfiguration.CLUSTER_PARTITION,true); modifiableConfiguration.set(GraphDatabaseConfiguration.CLUSTER_MAX_PARTITIONS,numPartitions); //uses SimpleBulkPlacementStrategy by default modifiableConfiguration.set(SimpleBulkPlacementStrategy.CONCURRENT_PARTITIONS,3*numPartitions); return config; }
public void setAll(Map<ConfigElement.PathIdentifier,Object> options) { for (Map.Entry<ConfigElement.PathIdentifier,Object> entry : options.entrySet()) { Preconditions.checkArgument(entry.getKey().element.isOption()); set((ConfigOption) entry.getKey().element, entry.getValue(), entry.getKey().umbrellaElements); } }
@Override public TransactionBuilder customOption(String k, Object v) { if (null == writableCustomOptions) throw new IllegalStateException("This builder was not constructed with setCustomOption support"); writableCustomOptions.set((ConfigOption<Object>)ConfigElement.parse(ROOT_NS, k).element, v); return this; }
@Override public LogManager openLogManager(String senderId, boolean requiresOrderPreserving) throws BackendException { storeManager = openStorageManager(); ModifiableConfiguration config = GraphDatabaseConfiguration.buildGraphConfiguration(); config.set(GraphDatabaseConfiguration.UNIQUE_INSTANCE_ID,senderId); config.set(GraphDatabaseConfiguration.LOG_READ_INTERVAL, Duration.ofMillis(500L), LOG_NAME); //To ensure that the write order is preserved in reading, we need to ensure that all writes go to the same partition //otherwise readers will independently read from the partitions out-of-order by design to avoid having to synchronize config.set(KCVSLogManager.LOG_FIXED_PARTITION, requiresOrderPreserving, LOG_NAME); return new KCVSLogManager(storeManager,config.restrictTo(LOG_NAME)); }
@Override public Map<String, Object> getBaseConfiguration(String graphName, Class<?> test, String testMethodName, final LoadGraphWith.GraphData loadGraphWith) { ModifiableConfiguration conf = getJanusGraphConfiguration(graphName, test, testMethodName); conf.set(GraphDatabaseConfiguration.COMPUTER_RESULT_MODE, "persist"); conf.set(GraphDatabaseConfiguration.AUTO_TYPE, "tp3"); Map<String, Object> result = new HashMap<>(); conf.getAll().forEach( (key, value) -> result.put(ConfigElement.getPath(key.element, key.umbrellaElements), value)); result.put(Graph.GRAPH, JanusGraphFactory.class.getName()); return result; }
@Parameterized.Parameters public static Collection<Object[]> configs() { final List<Object[]> configurations = new ArrayList<>(); ModifiableConfiguration c = getBasicConfig(); configurations.add(new Object[]{c.getConfiguration()}); c = getBasicConfig(); c.set(IDAUTHORITY_CAV_BITS,9); c.set(IDAUTHORITY_CAV_TAG,511); configurations.add(new Object[]{c.getConfiguration()}); c = getBasicConfig(); c.set(IDAUTHORITY_CAV_RETRIES,10); c.set(IDAUTHORITY_WAIT, Duration.ofMillis(10L)); c.set(IDAUTHORITY_CAV_BITS,7); //c.set(IDAUTHORITY_RANDOMIZE_UNIQUEID,true); c.set(IDAUTHORITY_CONFLICT_AVOIDANCE, ConflictAvoidanceMode.GLOBAL_AUTO); configurations.add(new Object[]{c.getConfiguration()}); return configurations; }
public static Backend getBackend(WriteConfiguration config, boolean initialize) throws BackendException { final ModifiableConfiguration adjustedConfig = new ModifiableConfiguration(GraphDatabaseConfiguration.ROOT_NS,config.copy(), BasicConfiguration.Restriction.NONE); adjustedConfig.set(GraphDatabaseConfiguration.LOCK_LOCAL_MEDIATOR_GROUP, "tmp"); adjustedConfig.set(GraphDatabaseConfiguration.UNIQUE_INSTANCE_ID, "inst"); final Backend backend = new Backend(adjustedConfig); if (initialize) { backend.initialize(adjustedConfig); } return backend; }
private static ReadConfiguration getLocalConfiguration(String shortcutOrFile) { File file = new File(shortcutOrFile); if (file.exists()) return getLocalConfiguration(file); else { int pos = shortcutOrFile.indexOf(':'); if (pos<0) pos = shortcutOrFile.length(); String backend = shortcutOrFile.substring(0,pos); Preconditions.checkArgument(StandardStoreManager.getAllManagerClasses().containsKey(backend.toLowerCase()), "Backend shorthand unknown: %s", backend); String secondArg = null; if (pos+1<shortcutOrFile.length()) secondArg = shortcutOrFile.substring(pos + 1).trim(); BaseConfiguration config = new BaseConfiguration(); ModifiableConfiguration writeConfig = new ModifiableConfiguration(ROOT_NS,new CommonsConfiguration(config), BasicConfiguration.Restriction.NONE); writeConfig.set(STORAGE_BACKEND,backend); ConfigOption option = Backend.getOptionForShorthand(backend); if (option==null) { Preconditions.checkArgument(secondArg==null); } else if (option==STORAGE_DIRECTORY || option==STORAGE_CONF_FILE) { Preconditions.checkArgument(StringUtils.isNotBlank(secondArg),"Need to provide additional argument to initialize storage backend"); writeConfig.set(option,getAbsolutePath(secondArg)); } else if (option==STORAGE_HOSTS) { Preconditions.checkArgument(StringUtils.isNotBlank(secondArg),"Need to provide additional argument to initialize storage backend"); writeConfig.set(option,new String[]{secondArg}); } else throw new IllegalArgumentException("Invalid configuration option for backend "+option); return new CommonsConfiguration(config); } }
public static Configuration getJobConf(List<SliceQuery> queries, Long modulus, Long modVal) { ModifiableConfiguration conf2 = new ModifiableConfiguration(SimpleScanJob.ROOT_NS, new CommonsConfiguration(new BaseConfiguration()), BasicConfiguration.Restriction.NONE); if (null != queries) conf2.set(HEX_QUERIES, encodeQueries(queries)); if (null != modulus) conf2.set(KEY_FILTER_ID_MODULUS, modulus); if (null != modVal) conf2.set(KEY_FILTER_ID_MODULAR_VALUE, modVal); return conf2; } }
private void checkBackwardCompatibilityWithTitan(ModifiableConfiguration globalWrite, BasicConfiguration localBasicConfiguration, KCVSConfiguration keyColumnValueStoreConfiguration, ModifiableConfiguration overwrite) { String version = globalWrite.get(TITAN_COMPATIBLE_VERSIONS); Preconditions.checkArgument(version!=null,"JanusGraph version nor Titan compatibility have not been initialized"); if (!JanusGraphConstants.TITAN_COMPATIBLE_VERSIONS.contains(version)) { throw new JanusGraphException(String.format(INCOMPATIBLE_VERSION_EXCEPTION, version, JanusGraphConstants.VERSION)); } // When connecting to a store created by Titan the ID store name will not be in the // global configuration as it was not something which was configurable with Titan. // So to ensure compatibility override the default to titan_ids. boolean localIdStoreIsDefault = JanusGraphConstants.JANUSGRAPH_ID_STORE_NAME.equals(localBasicConfiguration.get(IDS_STORE_NAME)); boolean usingTitanIdStore = localIdStoreIsDefault || JanusGraphConstants.TITAN_ID_STORE_NAME.equals(localBasicConfiguration.get(IDS_STORE_NAME)); boolean existingKeyStore = keyColumnValueStoreConfiguration.get(IDS_STORE_NAME.getName(), IDS_STORE_NAME.getDatatype()) != null; Preconditions.checkArgument(usingTitanIdStore,"ID store for Titan compatibility has not been initialized to: " + JanusGraphConstants.TITAN_ID_STORE_NAME); if (!existingKeyStore) { log.info("Setting {} to {} for Titan compatibility", IDS_STORE_NAME.getName(), JanusGraphConstants.TITAN_ID_STORE_NAME); overwrite.set(IDS_STORE_NAME, JanusGraphConstants.TITAN_ID_STORE_NAME); } }
public void open() throws BackendException { manager = new KeyColumnValueStoreManager[CONCURRENCY]; idAuthorities = new IDAuthority[CONCURRENCY]; for (int i = 0; i < CONCURRENCY; i++) { ModifiableConfiguration sc = StorageSetup.getConfig(baseStoreConfiguration.copy()); //sc.set(GraphDatabaseConfiguration.INSTANCE_RID_SHORT,(short)i); sc.set(GraphDatabaseConfiguration.UNIQUE_INSTANCE_ID_SUFFIX, (short)i); if (!sc.has(UNIQUE_INSTANCE_ID)) { String uniqueGraphId = getOrGenerateUniqueInstanceId(sc); log.debug("Setting unique instance id: {}", uniqueGraphId); sc.set(UNIQUE_INSTANCE_ID, uniqueGraphId); } sc.set(GraphDatabaseConfiguration.CLUSTER_MAX_PARTITIONS,MAX_NUM_PARTITIONS); manager[i] = openStorageManager(); StoreFeatures storeFeatures = manager[i].getFeatures(); KeyColumnValueStore idStore = manager[i].openDatabase("ids"); if (storeFeatures.isKeyConsistent()) idAuthorities[i] = new ConsistentKeyIDAuthority(idStore, manager[i], sc); else throw new IllegalArgumentException("Cannot open id store"); } }
public JanusGraph getForceIndexGraph() throws BackendException { final ModifiableConfiguration adjustedConfig = new ModifiableConfiguration(GraphDatabaseConfiguration.ROOT_NS,getConfiguration(), BasicConfiguration.Restriction.NONE); adjustedConfig.set(GraphDatabaseConfiguration.FORCE_INDEX_USAGE, true); final WriteConfiguration writeConfig = adjustedConfig.getConfiguration(); TestGraphConfigs.applyOverrides(writeConfig); Preconditions.checkNotNull(writeConfig); return JanusGraphFactory.open(writeConfig); }
public void open() throws BackendException { manager = new KeyColumnValueStoreManager[CONCURRENCY]; tx = new StoreTransaction[CONCURRENCY][NUM_TX]; store = new KeyColumnValueStore[CONCURRENCY]; for (int i = 0; i < CONCURRENCY; i++) { final ModifiableConfiguration sc = GraphDatabaseConfiguration.buildGraphConfiguration(); sc.set(GraphDatabaseConfiguration.LOCK_LOCAL_MEDIATOR_GROUP,concreteClassName + i); sc.set(GraphDatabaseConfiguration.UNIQUE_INSTANCE_ID,"inst"+i); sc.set(GraphDatabaseConfiguration.LOCK_RETRY,10); sc.set(GraphDatabaseConfiguration.LOCK_EXPIRE, Duration.ofMillis(EXPIRE_MS)); manager[i] = openStorageManager(i, sc); StoreFeatures storeFeatures = manager[i].getFeatures(); store[i] = manager[i].openDatabase(DB_NAME); for (int j = 0; j < NUM_TX; j++) { tx[i][j] = manager[i].beginTransaction(getTxConfig()); log.debug("Began transaction of class {}", tx[i][j].getClass().getCanonicalName()); } if (!storeFeatures.hasLocking()) { Preconditions.checkArgument(storeFeatures.isKeyConsistent(),"Store needs to support some form of locking"); KeyColumnValueStore lockerStore = manager[i].openDatabase(DB_NAME + "_lock_"); ConsistentKeyLocker c = new ConsistentKeyLocker.Builder(lockerStore, manager[i]).fromConfig(sc).mediatorName(concreteClassName + i).build(); store[i] = new ExpectedValueCheckingStore(store[i], c); for (int j = 0; j < NUM_TX; j++) tx[i][j] = new ExpectedValueCheckingTransaction(tx[i][j], manager[i].beginTransaction(getConsistentTxConfig(manager[i])), GraphDatabaseConfiguration.STORAGE_READ_WAITTIME.getDefaultValue()); } } }
public void clopen(Object... settings) { config = getConfiguration(); if (mgmt!=null && mgmt.isOpen()) mgmt.rollback(); if (null != tx && tx.isOpen()) tx.commit(); if (settings!=null && settings.length>0) { final Map<TestConfigOption,Object> options = validateConfigOptions(settings); JanusGraphManagement janusGraphManagement = null; final ModifiableConfiguration modifiableConfiguration = new ModifiableConfiguration(GraphDatabaseConfiguration.ROOT_NS,config, BasicConfiguration.Restriction.LOCAL); for (final Map.Entry<TestConfigOption,Object> option : options.entrySet()) { if (option.getKey().option.isLocal()) { modifiableConfiguration.set(option.getKey().option,option.getValue(),option.getKey().umbrella); } else { if (janusGraphManagement==null) janusGraphManagement = graph.openManagement(); janusGraphManagement.set(ConfigElement.getPath(option.getKey().option,option.getKey().umbrella),option.getValue()); } } if (janusGraphManagement!=null) janusGraphManagement.commit(); modifiableConfiguration.close(); } if (null != graph && null != graph.tx() && graph.tx().isOpen()) graph.tx().commit(); if (null != graph && graph.isOpen()) graph.close(); Preconditions.checkNotNull(config); open(config); }
private StandardScanner.Builder buildStoreIndexScanJob(String storeName) { TimestampProvider provider = configuration.get(TIMESTAMP_PROVIDER); ModifiableConfiguration jobConfig = GraphDatabaseConfiguration.buildJobConfiguration(); jobConfig.set(JOB_START_TIME,provider.getTime().toEpochMilli()); return scanner.build() .setStoreName(storeName) .setTimestampProvider(provider) .setJobConfiguration(jobConfig) .setGraphConfiguration(configuration) .setNumProcessingThreads(1) .setWorkBlockSize(10000); }