public CommonsConfiguration() { this(new BaseConfiguration()); }
public CommonsConfiguration() { this(new BaseConfiguration()); }
public static final ModifiableConfiguration buildJobConfiguration() { return new ModifiableConfiguration(JOB_NS, new CommonsConfiguration(new BaseConfiguration()), BasicConfiguration.Restriction.NONE); }
public static final ModifiableConfiguration buildGraphConfiguration() { return new ModifiableConfiguration(ROOT_NS, new CommonsConfiguration(new BaseConfiguration()), BasicConfiguration.Restriction.NONE); }
public static ModifiableConfiguration buildJobConfiguration() { return new ModifiableConfiguration(JOB_NS, new CommonsConfiguration(new BaseConfiguration()), BasicConfiguration.Restriction.NONE); }
public static ModifiableConfiguration buildGraphConfiguration() { return new ModifiableConfiguration(ROOT_NS, new CommonsConfiguration(new BaseConfiguration()), BasicConfiguration.Restriction.NONE); }
private static ModifiableConfiguration getIndexJobConf(String indexName, String relationType) { ModifiableConfiguration mc = new ModifiableConfiguration(GraphDatabaseConfiguration.JOB_NS, new CommonsConfiguration(new BaseConfiguration()), BasicConfiguration.Restriction.NONE); mc.set(com.thinkaurelius.titan.graphdb.olap.job.IndexUpdateJob.INDEX_NAME, indexName); mc.set(com.thinkaurelius.titan.graphdb.olap.job.IndexUpdateJob.INDEX_RELATION_TYPE, relationType); mc.set(GraphDatabaseConfiguration.JOB_START_TIME, System.currentTimeMillis()); return mc; }
@Override public WriteConfiguration copy() { BaseConfiguration copy = new BaseConfiguration(); copy.copy(config); return new CommonsConfiguration(copy); }
@Override public WriteConfiguration copy() { BaseConfiguration copy = new BaseConfiguration(); copy.copy(config); return new CommonsConfiguration(copy); }
public static Configuration getJobConf(List<SliceQuery> queries, Long modulus, Long modVal) { ModifiableConfiguration conf2 = new ModifiableConfiguration(SimpleScanJob.ROOT_NS, new CommonsConfiguration(new BaseConfiguration()), BasicConfiguration.Restriction.NONE); if (null != queries) conf2.set(HEX_QUERIES, encodeQueries(queries)); if (null != modulus) conf2.set(KEY_FILTER_ID_MODULUS, modulus); if (null != modVal) conf2.set(KEY_FILTER_ID_MODULAR_VALUE, modVal); return conf2; } }
public org.apache.commons.configuration.Configuration getConfigurationAtOpen() { org.apache.commons.configuration.Configuration result = new BaseConfiguration(); for (String k : configurationAtOpen.getKeys("")) { result.setProperty(k, configurationAtOpen.get(k, Object.class)); } return result; }
public static Configuration getJobConf(List<SliceQuery> queries, Long modulus, Long modVal) { ModifiableConfiguration conf2 = new ModifiableConfiguration(SimpleScanJob.ROOT_NS, new CommonsConfiguration(new BaseConfiguration()), BasicConfiguration.Restriction.NONE); if (null != queries) conf2.set(HEX_QUERIES, encodeQueries(queries)); if (null != modulus) conf2.set(KEY_FILTER_ID_MODULUS, modulus); if (null != modVal) conf2.set(KEY_FILTER_ID_MODULAR_VALUE, modVal); return conf2; } }
private static ReadConfiguration getLocalConfiguration(String shortcutOrFile) { File file = new File(shortcutOrFile); if (file.exists()) return getLocalConfiguration(file); else { int pos = shortcutOrFile.indexOf(':'); if (pos<0) pos = shortcutOrFile.length(); String backend = shortcutOrFile.substring(0,pos); Preconditions.checkArgument(StandardStoreManager.getAllManagerClasses().containsKey(backend.toLowerCase()), "Backend shorthand unknown: %s", backend); String secondArg = null; if (pos+1<shortcutOrFile.length()) secondArg = shortcutOrFile.substring(pos + 1).trim(); BaseConfiguration config = new BaseConfiguration(); ModifiableConfiguration writeConfig = new ModifiableConfiguration(ROOT_NS,new CommonsConfiguration(config), BasicConfiguration.Restriction.NONE); writeConfig.set(STORAGE_BACKEND,backend); ConfigOption option = Backend.getOptionForShorthand(backend); if (option==null) { Preconditions.checkArgument(secondArg==null); } else if (option==STORAGE_DIRECTORY || option==STORAGE_CONF_FILE) { Preconditions.checkArgument(StringUtils.isNotBlank(secondArg),"Need to provide additional argument to initialize storage backend"); writeConfig.set(option,getAbsolutePath(secondArg)); } else if (option==STORAGE_HOSTS) { Preconditions.checkArgument(StringUtils.isNotBlank(secondArg),"Need to provide additional argument to initialize storage backend"); writeConfig.set(option,new String[]{secondArg}); } else throw new IllegalArgumentException("Invalid configuration option for backend "+option); return new CommonsConfiguration(config); } }
public org.apache.commons.configuration.Configuration getConfigurationAtOpen() { org.apache.commons.configuration.Configuration result = new BaseConfiguration(); for (String k : configurationAtOpen.getKeys("")) { result.setProperty(k, configurationAtOpen.get(k, Object.class)); } return result; }
@Test public void testHelixExternalViewBasedRoutingTable() throws Exception { URL resourceUrl = getClass().getClassLoader().getResource("SampleExternalView.json"); Assert.assertNotNull(resourceUrl); String fileName = resourceUrl.getFile(); byte[] externalViewBytes = IOUtils.toByteArray(new FileInputStream(fileName)); ExternalView externalView = new ExternalView((ZNRecord) new ZNRecordSerializer().deserialize(externalViewBytes)); String tableName = externalView.getResourceName(); List<InstanceConfig> instanceConfigs = getInstanceConfigs(externalView); int numSegmentsInEV = externalView.getPartitionSet().size(); int numServersInEV = instanceConfigs.size(); HelixExternalViewBasedRouting routing = new HelixExternalViewBasedRouting(null, null, new BaseConfiguration()); routing.markDataResourceOnline(generateTableConfig(tableName), externalView, instanceConfigs); for (int i = 0; i < NUM_ROUNDS; i++) { Map<String, List<String>> routingTable = routing.getRoutingTable(new RoutingTableLookupRequest(tableName)); Assert.assertEquals(routingTable.size(), numServersInEV); int numSegments = 0; for (List<String> segmentsForServer : routingTable.values()) { int numSegmentsForServer = segmentsForServer.size(); Assert.assertTrue( numSegmentsForServer >= MIN_NUM_SEGMENTS_PER_SERVER && numSegmentsForServer <= MAX_NUM_SEGMENTS_PER_SERVER); numSegments += numSegmentsForServer; } Assert.assertEquals(numSegments, numSegmentsInEV); } }
@Test public void testLocalNodeUsingExtAndIndexDirectory() throws BackendException, InterruptedException { String baseDir = Joiner.on(File.separator).join("target", "es", "jvmlocal_ext2"); assertFalse(new File(baseDir + File.separator + "data").exists()); CommonsConfiguration cc = new CommonsConfiguration(new BaseConfiguration()); cc.set("index." + INDEX_NAME + ".elasticsearch.ext.node.data", "true"); cc.set("index." + INDEX_NAME + ".elasticsearch.ext.node.client", "false"); cc.set("index." + INDEX_NAME + ".elasticsearch.ext.node.local", "true"); ModifiableConfiguration config = new ModifiableConfiguration(GraphDatabaseConfiguration.ROOT_NS, cc, BasicConfiguration.Restriction.NONE); config.set(INTERFACE, ElasticSearchSetup.NODE.toString(), INDEX_NAME); config.set(INDEX_DIRECTORY, baseDir, INDEX_NAME); Configuration indexConfig = config.restrictTo(INDEX_NAME); IndexProvider idx = new ElasticSearchIndex(indexConfig); simpleWriteAndQuery(idx); idx.close(); assertTrue(new File(baseDir + File.separator + "data").exists()); }
@Test public void testLocalNodeUsingExt() throws BackendException, InterruptedException { String baseDir = Joiner.on(File.separator).join("target", "es", "jvmlocal_ext"); assertFalse(new File(baseDir + File.separator + "data").exists()); CommonsConfiguration cc = new CommonsConfiguration(new BaseConfiguration()); cc.set("index." + INDEX_NAME + ".elasticsearch.ext.node.data", "true"); cc.set("index." + INDEX_NAME + ".elasticsearch.ext.node.client", "false"); cc.set("index." + INDEX_NAME + ".elasticsearch.ext.node.local", "true"); cc.set("index." + INDEX_NAME + ".elasticsearch.ext.path.data", baseDir + File.separator + "data"); cc.set("index." + INDEX_NAME + ".elasticsearch.ext.path.work", baseDir + File.separator + "work"); cc.set("index." + INDEX_NAME + ".elasticsearch.ext.path.logs", baseDir + File.separator + "logs"); ModifiableConfiguration config = new ModifiableConfiguration(GraphDatabaseConfiguration.ROOT_NS, cc, BasicConfiguration.Restriction.NONE); config.set(INTERFACE, ElasticSearchSetup.NODE.toString(), INDEX_NAME); Configuration indexConfig = config.restrictTo(INDEX_NAME); IndexProvider idx = new ElasticSearchIndex(indexConfig); simpleWriteAndQuery(idx); idx.close(); assertTrue(new File(baseDir + File.separator + "data").exists()); }
@Test public void testDynamicRouting() throws Exception { String tableNameWithType = "testTable_OFFLINE"; BalancedRandomRoutingTableBuilder routingTableBuilder = new BalancedRandomRoutingTableBuilder(); TableConfig tableConfig = RoutingTableBuilderTestUtil.getDynamicComputingTableConfig(tableNameWithType); routingTableBuilder.init(new BaseConfiguration(), tableConfig, null, null); // Create external view ExternalView externalView = getDummyExternalView(); // Create instance configs List<InstanceConfig> instanceConfigList = getDummyInstanceConfigs(); // Build routing table routingTableBuilder.computeOnExternalViewChange("dummy", externalView, instanceConfigList); RoutingTableLookupRequest request = new RoutingTableLookupRequest(tableNameWithType); Map<String, List<String>> routingTable = routingTableBuilder.getRoutingTable(request, null); Set<String> segmentsInRoutingTable = new HashSet<>(); for (List<String> segments : routingTable.values()) { segmentsInRoutingTable.addAll(segments); } // Check that we picked all segments from the table Set<String> expectedSegments = externalView.getPartitionSet(); Assert.assertEquals(segmentsInRoutingTable, expectedSegments); }
@Test public void testIndexCreationOptions() throws InterruptedException, BackendException { final int shards = 77; ElasticsearchRunner esr = new ElasticsearchRunner(".", "indexCreationOptions.yml"); esr.start(); CommonsConfiguration cc = new CommonsConfiguration(new BaseConfiguration()); cc.set("index." + INDEX_NAME + ".elasticsearch.create.ext.number_of_shards", String.valueOf(shards)); cc.set("index." + INDEX_NAME + ".elasticsearch.ext.cluster.name", "indexCreationOptions"); ModifiableConfiguration config = new ModifiableConfiguration(GraphDatabaseConfiguration.ROOT_NS, cc, BasicConfiguration.Restriction.NONE); config.set(INTERFACE, ElasticSearchSetup.NODE.toString(), INDEX_NAME); Configuration indexConfig = config.restrictTo(INDEX_NAME); IndexProvider idx = new ElasticSearchIndex(indexConfig); simpleWriteAndQuery(idx); ImmutableSettings.Builder settingsBuilder = ImmutableSettings.settingsBuilder(); settingsBuilder.put("discovery.zen.ping.multicast.enabled", "false"); settingsBuilder.put("discovery.zen.ping.unicast.hosts", "localhost,127.0.0.1:9300"); settingsBuilder.put("cluster.name", "indexCreationOptions"); NodeBuilder nodeBuilder = NodeBuilder.nodeBuilder().settings(settingsBuilder.build()); nodeBuilder.client(true).data(false).local(false); Node n = nodeBuilder.build().start(); GetSettingsResponse response = n.client().admin().indices().getSettings(new GetSettingsRequest().indices("titan")).actionGet(); assertEquals(String.valueOf(shards), response.getSetting("titan", "index.number_of_shards")); idx.close(); n.stop(); esr.stop(); }
ElasticsearchRunner esr = new ElasticsearchRunner(".", "networkNodeUsingExt.yml"); esr.start(); CommonsConfiguration cc = new CommonsConfiguration(new BaseConfiguration()); cc.set("index." + INDEX_NAME + ".elasticsearch.ext.node.data", "false"); cc.set("index." + INDEX_NAME + ".elasticsearch.ext.node.client", "true");