@Deprecated public KiWiStore(String name, String jdbcUrl, String db_user, String db_password, KiWiDialect dialect, String defaultContext, String inferredContext) { this(new KiWiConfiguration(name,jdbcUrl,db_user,db_password,dialect, defaultContext, inferredContext)); }
/** * Check availability of the Database. * @param config the {@link KiWiConfiguration} to test * @throws AssumptionViolatedException if the database is not available. */ public static void checkDatabaseAvailability(KiWiConfiguration config) throws AssumptionViolatedException { checkDatabaseAvailability(config.getJdbcUrl(), config.getDbUser(), config.getDbPassword(), config.getDialect()); }
private void setupCaches() { setupMapConfig(NODE_CACHE, configuration.getNodeCacheSize()); setupMapConfig(TRIPLE_CACHE, configuration.getTripleCacheSize()); setupMapConfig(URI_CACHE, configuration.getUriCacheSize()); setupMapConfig(BNODE_CACHE, configuration.getBNodeCacheSize()); setupMapConfig(LITERAL_CACHE, configuration.getLiteralCacheSize()); setupMapConfig(NS_PREFIX_CACHE, configuration.getNamespaceCacheSize()); setupMapConfig(NS_URI_CACHE, configuration.getNamespaceCacheSize()); }
public InfinispanRemoteCacheManager(KiWiConfiguration configuration) { this.configuration = configuration; Configuration remoteCfg = new ConfigurationBuilder() .addServer() .host(configuration.getClusterAddress()) .port(configuration.getClusterPort()) .marshaller(new CustomJBossMarshaller()) .socketTimeout(configuration.getClusterTimeout()) .connectionTimeout(configuration.getClusterTimeout()) .build(); cacheManager = new RemoteCacheManager(remoteCfg); log.info("initialised Infinispan remote cache manager (servers: {})", configuration.getClusterAddress()); }
KiWiConfiguration configuration = new KiWiConfiguration(configurationService.getStringConfiguration(KiWiOptions.CLUSTERING_NAME, "Marmotta") + " KiWi", jdbcUrl, dbUser, dbPass, dialect, configurationService.getDefaultContext(), configurationService.getInferredContext()); configuration.setQueryLoggingEnabled(configurationService.getBooleanConfiguration(KiWiOptions.DEBUG_SLOWQUERIES, false)); configuration.setTripleBatchCommit(configurationService.getBooleanConfiguration(KiWiOptions.TRIPLES_BATCHCOMMIT, true)); configuration.setTripleBatchSize(configurationService.getIntConfiguration(KiWiOptions.TRIPLES_BATCHSIZE, 10000)); configuration.setDatacenterId(configurationService.getIntConfiguration(KiWiOptions.DATACENTER_ID,0)); configuration.setFulltextEnabled(configurationService.getBooleanConfiguration(KiWiOptions.FULLTEXT_ENABLED, true)); configuration.setFulltextLanguages(configurationService.getListConfiguration(KiWiOptions.FULLTEXT_LANGUAGES, ImmutableList.of("en"))); configuration.setClustered(configurationService.getBooleanConfiguration(KiWiOptions.CLUSTERING_ENABLED, false)); configuration.setClusterName(configurationService.getStringConfiguration(KiWiOptions.CLUSTERING_NAME, "Marmotta")); configuration.setLiteralCacheSize(configurationService.getIntConfiguration(KiWiOptions.CACHING_LITERAL_SIZE, 100000)); configuration.setBNodeCacheSize(configurationService.getIntConfiguration(KiWiOptions.CACHING_BNODE_SIZE, 10000)); configuration.setUriCacheSize(configurationService.getIntConfiguration(KiWiOptions.CACHING_URI_SIZE, 500000)); configuration.setTripleCacheSize(configurationService.getIntConfiguration(KiWiOptions.CACHING_TRIPLE_SIZE, 100000)); configuration.setClusterPort(configurationService.getIntConfiguration(KiWiOptions.CLUSTERING_PORT, 46655)); configuration.setClusterAddress(configurationService.getStringConfiguration(KiWiOptions.CLUSTERING_ADDRESS, "228.6.7.8")); configuration.setCachingBackend(CachingBackends.valueOf(configurationService.getStringConfiguration(KiWiOptions.CLUSTERING_BACKEND, "GUAVA"))); configuration.setCacheMode(CacheMode.valueOf(configurationService.getStringConfiguration(KiWiOptions.CLUSTERING_MODE,"LOCAL"))); if(configuration.isClustered()) { configuration.setRegistryStrategy(RegistryStrategy.CACHE); } else { configuration.setRegistryStrategy(RegistryStrategy.LOCAL);
KiWiConfiguration configuration = new KiWiConfiguration("Marmotta KiWi",dbUrl,db_user,db_pass,dialect); if(!StringUtils.isBlank(cluster)){ log.debug(" - cluster: {}", cluster); configuration.setClustered(true); configuration.setClusterName(cluster); String clusterAddress = (String)config.get(CLUSTER_ADDRESS); if(!StringUtils.isBlank(clusterAddress)){ configuration.setClusterAddress( strSubstitutor.replace(clusterAddress)); log.debug(" - cluster address: {}", configuration.getClusterAddress()); int port = ((Number)clusterPort).intValue(); if(port > 0){ configuration.setClusterPort(port); strSubstitutor.replace(clusterPort)); if(port > 0) { configuration.setClusterPort(port); log.debug(" - cluster port ({})", configuration.getClusterPort()); String cachingBackend = (String)config.get(CACHING_BACKEND); if(StringUtils.isBlank(cachingBackend)){ configuration.setCachingBackend( CachingBackends.valueOf(DEFAULT_CACHING_BACKEND)); } else { try { configuration.setCachingBackend(CachingBackends.valueOf(
public HazelcastCacheManager(KiWiConfiguration configuration) { this.configuration = configuration; hcConfiguration = new Config(); if(configuration.isClustered()) { hcConfiguration.getNetworkConfig().getJoin().getMulticastConfig().setEnabled(true); hcConfiguration.getNetworkConfig().getJoin().getMulticastConfig().setMulticastPort(configuration.getClusterPort()); hcConfiguration.getNetworkConfig().getJoin().getMulticastConfig().setMulticastGroup(configuration.getClusterAddress()); } else { hcConfiguration.getNetworkConfig().getJoin().getMulticastConfig().setEnabled(false); } hcConfiguration.getGroupConfig().setName(configuration.getClusterName()); hcConfiguration.setClassLoader(HazelcastCacheManager.class.getClassLoader()); hcConfiguration.setProperty("hazelcast.logging.type", "slf4j"); setupSerializers(); setupCaches(); hazelcast = Hazelcast.newHazelcastInstance(hcConfiguration); if(!configuration.isClustered()) { log.info("initialised Hazelcast local cache manager"); } else { log.info("initialised Hazelcast distributed cache manager (cluster name: {})", configuration.getClusterName()); if(configuration.getCacheMode() != CacheMode.DISTRIBUTED) { log.warn("Hazelcast only supports distributed cache mode (mode configuration was {})", configuration.getCacheMode()); } } }
public KiWiDialect getDialect() { return configuration.getDialect(); }
String jgroupsXml = IOUtils.toString(InfinispanEmbeddedCacheManager.class.getResourceAsStream("/jgroups-kiwi.xml")); jgroupsXml = jgroupsXml.replaceAll("mcast_addr=\"[0-9.]+\"", String.format("mcast_addr=\"%s\"", config.getClusterAddress())); jgroupsXml = jgroupsXml.replaceAll("mcast_port=\"[0-9]+\"", String.format("mcast_port=\"%d\"", config.getClusterPort())); .transport() .defaultTransport() .clusterName(config.getClusterName()) .machineId("instance-" + config.getDatacenterId()) .addProperty("configurationXml", jgroupsXml) .distributedSyncTimeout(config.getClusterTimeout()) .globalJmxStatistics() .jmxDomain("org.apache.marmotta.kiwi") .stateTransfer() .fetchInMemoryState(false) .timeout(config.getClusterTimeout()) .eviction() .strategy(EvictionStrategy.LIRS)
KiWiConfiguration kiwi = new KiWiConfiguration("kiwiLoader", dbCon, dbUser, dbPasswd, getDialect(dbCon).newInstance()); if(kiwi.getDialect() instanceof PostgreSQLDialect) { log.info("- using PostgreSQL bulk loader ... "); loaderConfiguration.setCommitBatchSize(100000); handler = new KiWiPostgresHandler(store,loaderConfiguration); } else if(kiwi.getDialect() instanceof MySQLDialect) { log.info("- using MySQL bulk loader ... "); loaderConfiguration.setCommitBatchSize(100000);
public static KiWiConfiguration createKiWiConfig(String database, KiWiDialect dialect) { final KiWiConfiguration config; if(!(dialect instanceof H2Dialect) && System.getProperty(database.toLowerCase()+".url") != null) { config = new KiWiConfiguration( database, System.getProperty(database.toLowerCase()+".url"), System.getProperty(database.toLowerCase()+".user","kiwi"), System.getProperty(database.toLowerCase()+".pass","kiwi"), dialect); } else if (dialect instanceof H2Dialect) { config = new KiWiConfiguration( "default-H2", "jdbc:h2:mem:kiwitest;MVCC=true;DB_CLOSE_ON_EXIT=TRUE;DB_CLOSE_DELAY=-1", "kiwi", "kiwi", dialect); } else { return null; } config.setDefaultContext("http://localhost/context/default"); config.setInferredContext("http://localhost/context/inferred"); return config; }
/** * Return a connection from the connection pool which already has the auto-commit disabled. * * @return a fresh JDBC connection from the connection pool * @throws SQLException in case a new connection could not be established */ public KiWiConnection getConnection() throws SQLException { if (!initialized) { throw new SQLException("persistence backend not initialized; call initialise before acquiring a connection"); } if (connectionPool == null) { throw new SQLException("connection pool is closed, database connections not available"); } KiWiConnection con = new KiWiConnection(this,configuration.getDialect(),cacheManager); if(getDialect().isBatchSupported()) { con.setBatchCommit(configuration.isTripleBatchCommit()); con.setBatchSize(configuration.getTripleBatchSize()); } return con; }
private void prepareFulltext(KiWiConfiguration configuration) { try { if(configuration.isFulltextEnabled()) { try (KiWiConnection connection = parent.getPersistence().getConnection()) { if (configuration.getDialect() instanceof PostgreSQLDialect) { if (configuration.getFulltextLanguages() != null) { StringBuilder script = new StringBuilder(); for (String line : IOUtils.readLines(PostgreSQLDialect.class.getResourceAsStream("create_fulltext_index.sql"))) { for (String lang : configuration.getFulltextLanguages()) { if (connection.getMetadata("ft.idx." + lang) == null) { String pg_configuration = POSTGRES_LANG_MAPPINGS.get(lang); if (configuration.getFulltextLanguages() != null) { if (connection.getMetadata("ft.idx.generic") == null) { StringBuilder script = new StringBuilder();
public EHCacheManager(KiWiConfiguration configuration) { cacheManager = net.sf.ehcache.CacheManager.newInstance(EHCacheManager.class.getResource("/ehcache-kiwi.xml")); if(configuration.getClusterName() != null) { cacheManager.setName(configuration.getClusterName()); } }
protected boolean isClustered() { return config.getCacheMode() == org.apache.marmotta.kiwi.config.CacheMode.DISTRIBUTED || config.getCacheMode() == org.apache.marmotta.kiwi.config.CacheMode.REPLICATED; }
/** * Return the prefix -> namespace cache from the cache manager. Used for looking up namespaces * @return */ public Map getNamespacePrefixCache() { if(nsPrefixCache == null) { if(isClustered()) { Configuration nsprefixConfiguration = new ConfigurationBuilder().read(defaultConfiguration) .clustering() .cacheMode(CacheMode.REPL_ASYNC) .eviction() .maxEntries(config.getNamespaceCacheSize()) .expiration() .lifespan(1, TimeUnit.DAYS) .build(); cacheManager.defineConfiguration(NS_PREFIX_CACHE, nsprefixConfiguration); } else { Configuration nsprefixConfiguration = new ConfigurationBuilder().read(defaultConfiguration) .eviction() .maxEntries(config.getNamespaceCacheSize()) .expiration() .lifespan(1, TimeUnit.HOURS) .build(); cacheManager.defineConfiguration(NS_PREFIX_CACHE, nsprefixConfiguration); } nsPrefixCache = cacheManager.getCache(NS_PREFIX_CACHE).getAdvancedCache().withFlags(Flag.SKIP_LOCKING, Flag.SKIP_CACHE_LOAD, Flag.SKIP_REMOTE_LOOKUP); } return nsPrefixCache; }
/** * Return the anonId -> KiWiAnonResource cache from the cache manager. This cache is used when constructing new * KiWiAnonResources to avoid a database lookup. * * @return */ public Map getBNodeCache() { if(bnodeCache == null) { Configuration bnodeConfiguration = new ConfigurationBuilder().read(defaultConfiguration) .eviction() .maxEntries(config.getBNodeCacheSize()) .build(); cacheManager.defineConfiguration(BNODE_CACHE, bnodeConfiguration); bnodeCache = new AsyncMap(cacheManager.getCache(BNODE_CACHE).getAdvancedCache().withFlags(Flag.SKIP_LOCKING, Flag.SKIP_CACHE_LOAD, Flag.SKIP_REMOTE_LOOKUP)); } return bnodeCache; }
/** * Return the literal cache key -> KiWiLiteral cache from the cache manager. This cache is used when constructing new * KiWiLiterals to avoid a database lookup. * * @see org.apache.marmotta.commons.sesame.model.LiteralCommons#createCacheKey(String, java.util.Locale, String) * @return */ public Map getLiteralCache() { if(literalCache == null) { Configuration literalConfiguration = new ConfigurationBuilder().read(defaultConfiguration) .eviction() .maxEntries(config.getLiteralCacheSize()) .build(); cacheManager.defineConfiguration(LITERAL_CACHE, literalConfiguration); literalCache = new AsyncMap(cacheManager.getCache(LITERAL_CACHE).getAdvancedCache().withFlags(Flag.SKIP_LOCKING, Flag.SKIP_CACHE_LOAD, Flag.SKIP_REMOTE_LOOKUP)); } return literalCache; }
/** * Return the triple id -> triple cache from the cache manager. This cache is used for speeding up the * construction of query results. * * @return */ public Map getTripleCache() { if(tripleCache == null) { Configuration tripleConfiguration = new ConfigurationBuilder().read(defaultConfiguration) .clustering() .cacheMode(CacheMode.LOCAL) .eviction() .maxEntries(config.getTripleCacheSize()) .expiration() .lifespan(60, TimeUnit.MINUTES) .maxIdle(30, TimeUnit.MINUTES) .build(); cacheManager.defineConfiguration(TRIPLE_CACHE, tripleConfiguration); tripleCache = new AsyncMap(cacheManager.getCache(TRIPLE_CACHE).getAdvancedCache().withFlags(Flag.SKIP_LOCKING, Flag.SKIP_CACHE_LOAD, Flag.SKIP_REMOTE_LOOKUP)); } return tripleCache; }
KiWiConfiguration configuration = new KiWiConfiguration(configurationService.getStringConfiguration(KiWiOptions.CLUSTERING_NAME, "Marmotta") + " KiWi", jdbcUrl, dbUser, dbPass, dialect, configurationService.getDefaultContext(), configurationService.getInferredContext()); configuration.setQueryLoggingEnabled(configurationService.getBooleanConfiguration(KiWiOptions.DEBUG_SLOWQUERIES, false)); configuration.setTripleBatchCommit(configurationService.getBooleanConfiguration(KiWiOptions.TRIPLES_BATCHCOMMIT, true)); configuration.setTripleBatchSize(configurationService.getIntConfiguration(KiWiOptions.TRIPLES_BATCHSIZE, 10000)); configuration.setDatacenterId(configurationService.getIntConfiguration(KiWiOptions.DATACENTER_ID,0)); configuration.setFulltextEnabled(configurationService.getBooleanConfiguration(KiWiOptions.FULLTEXT_ENABLED, true)); configuration.setFulltextLanguages(configurationService.getListConfiguration(KiWiOptions.FULLTEXT_LANGUAGES, ImmutableList.of("en"))); configuration.setClustered(configurationService.getBooleanConfiguration(KiWiOptions.CLUSTERING_ENABLED, false)); configuration.setClusterName(configurationService.getStringConfiguration(KiWiOptions.CLUSTERING_NAME, "Marmotta")); configuration.setLiteralCacheSize(configurationService.getIntConfiguration(KiWiOptions.CACHING_LITERAL_SIZE, 100000)); configuration.setBNodeCacheSize(configurationService.getIntConfiguration(KiWiOptions.CACHING_BNODE_SIZE, 10000)); configuration.setUriCacheSize(configurationService.getIntConfiguration(KiWiOptions.CACHING_URI_SIZE, 500000)); configuration.setTripleCacheSize(configurationService.getIntConfiguration(KiWiOptions.CACHING_TRIPLE_SIZE, 100000)); configuration.setClusterPort(configurationService.getIntConfiguration(KiWiOptions.CLUSTERING_PORT, 46655)); configuration.setClusterAddress(configurationService.getStringConfiguration(KiWiOptions.CLUSTERING_ADDRESS, "228.6.7.8")); configuration.setCachingBackend(CachingBackends.valueOf(configurationService.getStringConfiguration(KiWiOptions.CLUSTERING_BACKEND, "GUAVA"))); configuration.setCacheMode(CacheMode.valueOf(configurationService.getStringConfiguration(KiWiOptions.CLUSTERING_MODE,"LOCAL"))); if(configuration.isClustered()) { configuration.setRegistryStrategy(RegistryStrategy.CACHE); } else { configuration.setRegistryStrategy(RegistryStrategy.LOCAL);