@Override public String get(String path) { ConfigElement.PathIdentifier pp = ConfigElement.parse(config.getRootNamespace(),path); if (pp.element.isNamespace()) { ConfigNamespace ns = (ConfigNamespace)pp.element; StringBuilder s = new StringBuilder(); if (ns.isUmbrella() && !pp.lastIsUmbrella) { for (String sub : config.getContainedNamespaces(ns,pp.umbrellaElements)) { s.append("+ ").append(sub).append("\n"); } else { Object value; if (config.has((ConfigOption)pp.element,pp.umbrellaElements) || ((ConfigOption) pp.element).getDefaultValue()!=null) { value = config.get((ConfigOption)pp.element,pp.umbrellaElements); } else { return "null";
@Override public WriteConfiguration getConfiguration() { WriteConfiguration config = getBaseConfiguration(); ModifiableConfiguration modifiableConfiguration = new ModifiableConfiguration(GraphDatabaseConfiguration.ROOT_NS,config, BasicConfiguration.Restriction.NONE); modifiableConfiguration.set(BASIC_METRICS,true); modifiableConfiguration.set(METRICS_MERGE_STORES,false); modifiableConfiguration.set(PROPERTY_PREFETCHING,false); modifiableConfiguration.set(DB_CACHE, false); return config; }
/** * Closes this configuration handler */ public void close() { config.close(); }
public static WriteConfiguration addPermanentCache(ModifiableConfiguration conf) { conf.set(DB_CACHE, true); conf.set(DB_CACHE_TIME, 0L); return conf.getConfiguration(); }
@Override public LogManager openLogManager(String senderId, boolean requiresOrderPreserving) throws BackendException { storeManager = openStorageManager(); ModifiableConfiguration config = GraphDatabaseConfiguration.buildGraphConfiguration(); config.set(GraphDatabaseConfiguration.UNIQUE_INSTANCE_ID,senderId); config.set(GraphDatabaseConfiguration.LOG_READ_INTERVAL, Duration.ofMillis(500L), LOG_NAME); //To ensure that the write order is preserved in reading, we need to ensure that all writes go to the same partition //otherwise readers will independently read from the partitions out-of-order by design to avoid having to synchronize config.set(KCVSLogManager.LOG_FIXED_PARTITION, requiresOrderPreserving, LOG_NAME); return new KCVSLogManager(storeManager,config.restrictTo(LOG_NAME)); }
ModifiableConfiguration overwrite = new ModifiableConfiguration(ROOT_NS,new CommonsConfiguration(), BasicConfiguration.Restriction.NONE); localConfiguration = new ModifiableConfiguration(ROOT_NS, new CommonsConfiguration(), BasicConfiguration.Restriction.LOCAL); localConfiguration.setAll(getLocalSubset(localBasicConfiguration.getAll())); overwrite.set(LOCK_LOCAL_MEDIATOR_GROUP, storeManager.getName()); ModifiableConfiguration globalWrite = new ModifiableConfiguration(ROOT_NS,keyColumnValueStoreConfiguration, BasicConfiguration.Restriction.GLOBAL); if (!globalWrite.isFrozen()) { globalWrite.setAll(getGlobalSubset(localBasicConfiguration.getAll())); Preconditions.checkArgument(!globalWrite.has(INITIAL_JANUSGRAPH_VERSION),"Database has already been initialized but not frozen"); globalWrite.set(INITIAL_JANUSGRAPH_VERSION,JanusGraphConstants.VERSION); Preconditions.checkArgument(!globalWrite.has(INITIAL_STORAGE_VERSION),"Database has already been initialized but not frozen"); globalWrite.set(INITIAL_STORAGE_VERSION,JanusGraphConstants.STORAGE_VERSION); final TimestampProviders backendPreference; if (f.hasTimestamps() && null != (backendPreference = f.getPreferredTimestamps())) { globalWrite.set(TIMESTAMP_PROVIDER, backendPreference); log.info("Set timestamps to {} according to storage backend preference", LoggerUtil.sanitizeAndLaunder(globalWrite.get(TIMESTAMP_PROVIDER))); } else { globalWrite.set(TIMESTAMP_PROVIDER, TIMESTAMP_PROVIDER.getDefaultValue()); log.info("Set default timestamp provider {}", LoggerUtil.sanitizeAndLaunder(globalWrite.get(TIMESTAMP_PROVIDER)));
@Override public ModifiableConfiguration getJanusGraphConfiguration(String graphName, Class<?> test, String testMethodName) { return GraphDatabaseConfiguration.buildGraphConfiguration() .set(SimpleBulkPlacementStrategy.CONCURRENT_PARTITIONS,1) .set(GraphDatabaseConfiguration.CLUSTER_MAX_PARTITIONS, 2) .set(GraphDatabaseConfiguration.IDAUTHORITY_CAV_BITS,0); }
@Override public void setConf(final Configuration config) { super.setConf(config); // Copy some JanusGraph configuration keys to the Hadoop Configuration keys used by Cassandra's ColumnFamilyInputFormat ConfigHelper.setInputInitialAddress(config, janusgraphConf.get(GraphDatabaseConfiguration.STORAGE_HOSTS)[0]); if (janusgraphConf.has(GraphDatabaseConfiguration.STORAGE_PORT)) ConfigHelper.setInputRpcPort(config, String.valueOf(janusgraphConf.get(GraphDatabaseConfiguration.STORAGE_PORT))); if (janusgraphConf.has(GraphDatabaseConfiguration.AUTH_USERNAME)) ConfigHelper.setInputKeyspaceUserName(config, janusgraphConf.get(GraphDatabaseConfiguration.AUTH_USERNAME)); if (janusgraphConf.has(GraphDatabaseConfiguration.AUTH_PASSWORD)) ConfigHelper.setInputKeyspacePassword(config, janusgraphConf.get(GraphDatabaseConfiguration.AUTH_PASSWORD)); // Copy keyspace, force the CF setting to edgestore, honor widerows when set final boolean wideRows = config.getBoolean(INPUT_WIDEROWS_CONFIG, false); // Use the setInputColumnFamily overload that includes a widerows argument; using the overload without this argument forces it false ConfigHelper.setInputColumnFamily(config, janusgraphConf.get(AbstractCassandraStoreManager.CASSANDRA_KEYSPACE), mrConf.get(JanusGraphHadoopConfiguration.COLUMN_FAMILY_NAME), wideRows); log.debug("Set keyspace: {}", janusgraphConf.get(AbstractCassandraStoreManager.CASSANDRA_KEYSPACE)); // Set the column slice bounds via Faunus' vertex query filter final SlicePredicate predicate = new SlicePredicate(); final int rangeBatchSize = config.getInt(RANGE_BATCH_SIZE_CONFIG, Integer.MAX_VALUE); predicate.setSlice_range(getSliceRange(rangeBatchSize)); // TODO stop slicing the whole row ConfigHelper.setInputSlicePredicate(config, predicate); }
final Map<String, Object> configSub = janusgraphConf.getSubset(HBaseStoreManager.HBASE_CONFIGURATION_NAMESPACE); for (Map.Entry<String, Object> entry : configSub.entrySet()) { log.info("HBase configuration: setting {}={}", entry.getKey(), entry.getValue()); config.set(TableInputFormat.INPUT_TABLE, janusgraphConf.get(HBaseStoreManager.HBASE_TABLE)); config.set(HConstants.ZOOKEEPER_QUORUM, janusgraphConf.get(GraphDatabaseConfiguration.STORAGE_HOSTS)[0]); if (janusgraphConf.has(GraphDatabaseConfiguration.STORAGE_PORT)) config.set(HConstants.ZOOKEEPER_CLIENT_PORT, String.valueOf(janusgraphConf.get(GraphDatabaseConfiguration.STORAGE_PORT))); config.set("autotype", "none"); log.debug("hbase.security.authentication={}", config.get("hbase.security.authentication")); String cfName = mrConf.get(JanusGraphHadoopConfiguration.COLUMN_FAMILY_NAME); if (janusgraphConf.get(HBaseStoreManager.SHORT_CF_NAMES)) { try { final BiMap<String,String> shortCfMap = HBaseStoreManager.createShortCfMap(janusgraphConf);
/** * Opens a {@link JanusGraph} database configured according to the provided configuration. * This method shouldn't be called by end users; it is used by internal server processes to * open graphs defined at server start that do not include the graphname property. * * @param configuration Configuration for the graph database * @param backupName Backup name for graph * @return JanusGraph graph database */ public static JanusGraph open(ReadConfiguration configuration, String backupName) { final ModifiableConfiguration config = new ModifiableConfiguration(ROOT_NS, (WriteConfiguration) configuration, BasicConfiguration.Restriction.NONE); final String graphName = config.has(GRAPH_NAME) ? config.get(GRAPH_NAME) : backupName; final JanusGraphManager jgm = JanusGraphManagerUtility.getInstance(); if (null != graphName) { Preconditions.checkState(jgm != null, JANUS_GRAPH_MANAGER_EXPECTED_STATE_MSG); return (JanusGraph) jgm.openGraph(graphName, gName -> new StandardJanusGraph(new GraphDatabaseConfiguration(configuration))); } else { if (jgm != null) { log.warn("You should supply \"graph.graphname\" in your .properties file configuration if you are opening " + "a graph that has not already been opened at server start, i.e. it was " + "defined in your YAML file. This will ensure the graph is tracked by the JanusGraphManager, " + "which will enable autocommit and rollback functionality upon all gremlin script executions. " + "Note that JanusGraphFactory#open(String === shortcut notation) does not support consuming the property " + "\"graph.graphname\" so these graphs should be accessed dynamically by supplying a .properties file here " + "or by using the ConfiguredGraphFactory."); } return new StandardJanusGraph(new GraphDatabaseConfiguration(configuration)); } }
public void open() throws BackendException { manager = new KeyColumnValueStoreManager[CONCURRENCY]; idAuthorities = new IDAuthority[CONCURRENCY]; for (int i = 0; i < CONCURRENCY; i++) { ModifiableConfiguration sc = StorageSetup.getConfig(baseStoreConfiguration.copy()); //sc.set(GraphDatabaseConfiguration.INSTANCE_RID_SHORT,(short)i); sc.set(GraphDatabaseConfiguration.UNIQUE_INSTANCE_ID_SUFFIX, (short)i); if (!sc.has(UNIQUE_INSTANCE_ID)) { String uniqueGraphId = getOrGenerateUniqueInstanceId(sc); log.debug("Setting unique instance id: {}", uniqueGraphId); sc.set(UNIQUE_INSTANCE_ID, uniqueGraphId); } sc.set(GraphDatabaseConfiguration.CLUSTER_MAX_PARTITIONS,MAX_NUM_PARTITIONS); manager[i] = openStorageManager(); StoreFeatures storeFeatures = manager[i].getFeatures(); KeyColumnValueStore idStore = manager[i].openDatabase("ids"); if (storeFeatures.isKeyConsistent()) idAuthorities[i] = new ConsistentKeyIDAuthority(idStore, manager[i], sc); else throw new IllegalArgumentException("Cannot open id store"); } }
public JanusGraph getForceIndexGraph() throws BackendException { final ModifiableConfiguration adjustedConfig = new ModifiableConfiguration(GraphDatabaseConfiguration.ROOT_NS,getConfiguration(), BasicConfiguration.Restriction.NONE); adjustedConfig.set(GraphDatabaseConfiguration.FORCE_INDEX_USAGE, true); final WriteConfiguration writeConfig = adjustedConfig.getConfiguration(); TestGraphConfigs.applyOverrides(writeConfig); Preconditions.checkNotNull(writeConfig); return JanusGraphFactory.open(writeConfig); }
@Override public void forceCloseInstance(String instanceId) { Preconditions.checkArgument(!graph.getConfiguration().getUniqueGraphId().equals(instanceId), "Cannot force close this current instance [%s]. Properly shut down the graph instead.", instanceId); Preconditions.checkArgument(modifyConfig.has(REGISTRATION_TIME, instanceId), "Instance [%s] is not currently open", instanceId); Instant registrationTime = modifyConfig.get(REGISTRATION_TIME, instanceId); Preconditions.checkArgument(registrationTime.compareTo(txStartTime) < 0, "The to-be-closed instance [%s] was started after this transaction" + "which indicates a successful restart and can hence not be closed: %s vs %s", instanceId, registrationTime, txStartTime); modifyConfig.remove(REGISTRATION_TIME, instanceId); }
public void clopen(Object... settings) { config = getConfiguration(); if (mgmt!=null && mgmt.isOpen()) mgmt.rollback(); if (null != tx && tx.isOpen()) tx.commit(); if (settings!=null && settings.length>0) { final Map<TestConfigOption,Object> options = validateConfigOptions(settings); JanusGraphManagement janusGraphManagement = null; final ModifiableConfiguration modifiableConfiguration = new ModifiableConfiguration(GraphDatabaseConfiguration.ROOT_NS,config, BasicConfiguration.Restriction.LOCAL); for (final Map.Entry<TestConfigOption,Object> option : options.entrySet()) { if (option.getKey().option.isLocal()) { modifiableConfiguration.set(option.getKey().option,option.getValue(),option.getKey().umbrella); } else { if (janusGraphManagement==null) janusGraphManagement = graph.openManagement(); janusGraphManagement.set(ConfigElement.getPath(option.getKey().option,option.getKey().umbrella),option.getValue()); } } if (janusGraphManagement!=null) janusGraphManagement.commit(); modifiableConfiguration.close(); } if (null != graph && null != graph.tx() && graph.tx().isOpen()) graph.tx().commit(); if (null != graph && graph.isOpen()) graph.close(); Preconditions.checkNotNull(config); open(config); }
public static ModifiableConfiguration getConfig(WriteConfiguration config) { return new ModifiableConfiguration(ROOT_NS,config, BasicConfiguration.Restriction.NONE); }
private Log openLog(String logManagerName, String logName) { try { final ModifiableConfiguration configuration = new ModifiableConfiguration(GraphDatabaseConfiguration.ROOT_NS,config.copy(), BasicConfiguration.Restriction.NONE); configuration.set(GraphDatabaseConfiguration.UNIQUE_INSTANCE_ID, "reader"); configuration.set(GraphDatabaseConfiguration.LOG_READ_INTERVAL, Duration.ofMillis(500L), logManagerName); if (logStoreManager==null) { logStoreManager = Backend.getStorageManager(configuration); } final StoreFeatures f = logStoreManager.getFeatures(); final boolean part = f.isDistributed() && f.isKeyOrdered(); if (part) { for (final String partitionedLogName : new String[]{USER_LOG,TRANSACTION_LOG,MANAGEMENT_LOG}) configuration.set(KCVSLogManager.LOG_MAX_PARTITIONS,8,partitionedLogName); } assert logStoreManager!=null; if (!logManagers.containsKey(logManagerName)) { //Open log manager - only supports KCVSLog final Configuration logConfig = configuration.restrictTo(logManagerName); Preconditions.checkArgument(logConfig.get(LOG_BACKEND).equals(LOG_BACKEND.getDefaultValue())); logManagers.put(logManagerName,new KCVSLogManager(logStoreManager,logConfig)); } assert logManagers.containsKey(logManagerName); return logManagers.get(logManagerName).openLog(logName); } catch (final BackendException e) { throw new JanusGraphException("Could not open log: "+ logName,e); } }
private void checkBackwardCompatibilityWithTitan(ModifiableConfiguration globalWrite, BasicConfiguration localBasicConfiguration, KCVSConfiguration keyColumnValueStoreConfiguration, ModifiableConfiguration overwrite) { String version = globalWrite.get(TITAN_COMPATIBLE_VERSIONS); Preconditions.checkArgument(version!=null,"JanusGraph version nor Titan compatibility have not been initialized"); if (!JanusGraphConstants.TITAN_COMPATIBLE_VERSIONS.contains(version)) { throw new JanusGraphException(String.format(INCOMPATIBLE_VERSION_EXCEPTION, version, JanusGraphConstants.VERSION)); } // When connecting to a store created by Titan the ID store name will not be in the // global configuration as it was not something which was configurable with Titan. // So to ensure compatibility override the default to titan_ids. boolean localIdStoreIsDefault = JanusGraphConstants.JANUSGRAPH_ID_STORE_NAME.equals(localBasicConfiguration.get(IDS_STORE_NAME)); boolean usingTitanIdStore = localIdStoreIsDefault || JanusGraphConstants.TITAN_ID_STORE_NAME.equals(localBasicConfiguration.get(IDS_STORE_NAME)); boolean existingKeyStore = keyColumnValueStoreConfiguration.get(IDS_STORE_NAME.getName(), IDS_STORE_NAME.getDatatype()) != null; Preconditions.checkArgument(usingTitanIdStore,"ID store for Titan compatibility has not been initialized to: " + JanusGraphConstants.TITAN_ID_STORE_NAME); if (!existingKeyStore) { log.info("Setting {} to {} for Titan compatibility", IDS_STORE_NAME.getName(), JanusGraphConstants.TITAN_ID_STORE_NAME); overwrite.set(IDS_STORE_NAME, JanusGraphConstants.TITAN_ID_STORE_NAME); } }
final Map<String, Object> configSub = janusgraphConf.getSubset(HBaseStoreManager.HBASE_CONFIGURATION_NAMESPACE); for (Map.Entry<String, Object> entry : configSub.entrySet()) { log.info("HBase configuration: setting {}={}", entry.getKey(), entry.getValue()); String cfName = mrConf.get(JanusGraphHadoopConfiguration.COLUMN_FAMILY_NAME); if (janusgraphConf.get(HBaseStoreManager.SHORT_CF_NAMES)) { try { final BiMap<String, String> shortCfMap = HBaseStoreManager.createShortCfMap(janusgraphConf); final String snapshotName = janusgraphConf.get(HBaseStoreManager.HBASE_SNAPSHOT); final String restoreDirString = janusgraphConf.get(HBaseStoreManager.HBASE_SNAPSHOT_RESTORE_DIR);
@Override public Map<String, Object> getBaseConfiguration(String graphName, Class<?> test, String testMethodName, final LoadGraphWith.GraphData loadGraphWith) { ModifiableConfiguration conf = getJanusGraphConfiguration(graphName, test, testMethodName); conf.set(GraphDatabaseConfiguration.COMPUTER_RESULT_MODE, "persist"); conf.set(GraphDatabaseConfiguration.AUTO_TYPE, "tp3"); Map<String, Object> result = new HashMap<>(); conf.getAll().forEach( (key, value) -> result.put(ConfigElement.getPath(key.element, key.umbrellaElements), value)); result.put(Graph.GRAPH, JanusGraphFactory.class.getName()); return result; }
@Override public UserModifiableConfiguration set(String path, Object value) { ConfigElement.PathIdentifier pp = ConfigElement.parse(config.getRootNamespace(),path); Preconditions.checkArgument(pp.element.isOption(),"Need to provide configuration option - not namespace: %s",path); ConfigOption option = (ConfigOption)pp.element; verifier.verifyModification(option); if (option.getDatatype().isArray()) { Class arrayType = option.getDatatype().getComponentType(); Object arr; if (value.getClass().isArray()) { int size = Array.getLength(value); arr = Array.newInstance(arrayType,size); for (int i=0;i<size;i++) { Array.set(arr,i,convertBasic(Array.get(value,i),arrayType)); } } else { arr = Array.newInstance(arrayType,1); Array.set(arr,0,convertBasic(value,arrayType)); } value = arr; } else { value = convertBasic(value,option.getDatatype()); } config.set(option,value,pp.umbrellaElements); return this; }