public static synchronized DistributedQueryRunner createAccumuloQueryRunner(Map<String, String> extraProperties) throws Exception { DistributedQueryRunner queryRunner = new DistributedQueryRunner(createSession(), 4, extraProperties); queryRunner.installPlugin(new TpchPlugin()); queryRunner.createCatalog("tpch", "tpch"); queryRunner.installPlugin(new AccumuloPlugin()); Map<String, String> accumuloProperties = ImmutableMap.<String, String>builder() .put(AccumuloConfig.INSTANCE, connector.getInstance().getInstanceName()) .put(AccumuloConfig.ZOOKEEPERS, connector.getInstance().getZooKeepers()) .put(AccumuloConfig.USERNAME, MAC_USER) .put(AccumuloConfig.PASSWORD, MAC_PASSWORD) .put(AccumuloConfig.ZOOKEEPER_METADATA_ROOT, "/presto-accumulo-test") .build(); queryRunner.createCatalog("accumulo", "accumulo", accumuloProperties); if (!tpchLoaded) { copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, createSession(), TpchTable.getTables()); connector.tableOperations().addSplits("tpch.orders", ImmutableSortedSet.of(new Text(new LexicoderRowSerializer().encode(BIGINT, 7500L)))); tpchLoaded = true; } return queryRunner; }
@Override public String getInstanceName() { return instance.getInstanceName(); }
@Override public void run() { synchronized (BasicServlet.class) { if (cachedInstanceName == null) { cachedInstanceName = HdfsZooInstance.getInstance().getInstanceName(); } } } }, 1000);
@Override public void run() { synchronized (Monitor.class) { if (cachedInstanceName.get().equals(DEFAULT_INSTANCE_NAME)) { final String instanceName = HdfsZooInstance.getInstance().getInstanceName(); if (null != instanceName) { cachedInstanceName.set(instanceName); } } } } }, 0);
@Override public String toString() { return new StringBuilder().append(instance.getInstanceID()).append("/").append(connector.getInstance().getInstanceName()).append("/").append("/") .append(table).append("/").append(user).toString(); }
public static void main(String[] args) { Instance instance = HdfsZooInstance.getInstance(); System.out.println("Instance Name: " + instance.getInstanceName()); System.out.println("Instance ID: " + instance.getInstanceID()); System.out.println("ZooKeepers: " + instance.getZooKeepers()); System.out.println("Masters: " + Joiner.on(", ").join(instance.getMasterLocations())); } }
public static void main(String[] args) { Instance instance = HdfsZooInstance.getInstance(); System.out.println("Instance Name: " + instance.getInstanceName()); System.out.println("Instance ID: " + instance.getInstanceID()); System.out.println("ZooKeepers: " + instance.getZooKeepers()); System.out.println("Masters: " + StringUtil.join(instance.getMasterLocations(), ", ")); }
/** * Check authenticationToken works for this user. */ private void checkCredentials() { try { if (!connector.securityOperations().authenticateUser(connector.whoami(), authenticationToken)) throw new IllegalArgumentException("instance " + connector.getInstance().getInstanceName() + ": bad username " + connector.whoami() + " with token " + authenticationToken); } catch (AccumuloException | AccumuloSecurityException e) { throw new IllegalArgumentException("instance " + connector.getInstance().getInstanceName() + ": error with username " + connector.whoami() + " with token " + authenticationToken, e); } }
public String getDefaultPrompt() { return connector.whoami() + "@" + connector.getInstance().getInstanceName() + (getTableName().isEmpty() ? "" : " ") + getTableName() + "> "; }
protected String loadDataAndCreateQuery(final String sparql, final Collection<Statement> statements) throws Exception { requireNonNull(sparql); requireNonNull(statements); // Register the PCJ with Rya. final Instance accInstance = super.getAccumuloConnector().getInstance(); final Connector accumuloConn = super.getAccumuloConnector(); final RyaClient ryaClient = AccumuloRyaClientFactory.build(new AccumuloConnectionDetails(ACCUMULO_USER, ACCUMULO_PASSWORD.toCharArray(), accInstance.getInstanceName(), accInstance.getZooKeepers()), accumuloConn); final String pcjId = ryaClient.getCreatePCJ().createPCJ(RYA_INSTANCE_NAME, sparql, Sets.newHashSet(ExportStrategy.KAFKA)); loadData(statements); // The PCJ Id is the topic name the results will be written to. return pcjId; }
protected String loadDataAndCreateQuery(final String sparql, final Collection<Statement> statements) throws Exception { requireNonNull(sparql); requireNonNull(statements); // Register the PCJ with Rya. final Instance accInstance = super.getAccumuloConnector().getInstance(); final Connector accumuloConn = super.getAccumuloConnector(); final RyaClient ryaClient = AccumuloRyaClientFactory.build(new AccumuloConnectionDetails(ACCUMULO_USER, ACCUMULO_PASSWORD.toCharArray(), accInstance.getInstanceName(), accInstance.getZooKeepers()), accumuloConn); final String pcjId = ryaClient.getCreatePCJ().createPCJ(RYA_INSTANCE_NAME, sparql, Sets.newHashSet(ExportStrategy.KAFKA)); loadData(statements); // The PCJ Id is the topic name the results will be written to. return pcjId; }
public synchronized ScannerBase newRfileScanner(String tableName, Set<Authorizations> auths, Query setting) { Configuration conf = new Configuration(); Connector con = cxn; final String instanceName = con.getInstance().getInstanceName(); final String zookeepers = con.getInstance().getZooKeepers(); AccumuloHelper.setInstanceName(conf, instanceName); AccumuloHelper.setUsername(conf, con.whoami()); AccumuloHelper.setZooKeepers(conf, zookeepers); BulkInputFormat.setZooKeeperInstance(conf, instanceName, zookeepers); AccumuloHelper.setPassword(conf, config.getAccumuloPassword().getBytes()); BulkInputFormat.setMemoryInput(conf, con.whoami(), config.getAccumuloPassword().getBytes(), tableName, auths.iterator().next()); conf.set(MultiRfileInputformat.CACHE_METADATA, "true"); ScannerBase baseScanner = new RfileScanner(con, conf, tableName, auths, 1); instances.add(baseScanner); return baseScanner; }
public void printInfo() throws IOException { reader.print("\n" + SHELL_DESCRIPTION + "\n" + "- \n" + "- version: " + Constants.VERSION + "\n" + "- instance name: " + connector.getInstance().getInstanceName() + "\n" + "- instance id: " + connector.getInstance().getInstanceID() + "\n" + "- \n" + "- type 'help' for a list of available commands\n" + "- \n"); reader.flush(); }
public static IteratorSetting iteratorSetting(int priority, int sleepAfterFirstWrite, long batchWriterTimeout, long batchWriterMaxMemory, int numEntriesToWrite, String tableName, Connector connector, AuthenticationToken token, boolean clearCacheAfterFirstWrite, boolean splitAfterFirstWrite) { return iteratorSetting(priority, sleepAfterFirstWrite, batchWriterTimeout, batchWriterMaxMemory, numEntriesToWrite, tableName, connector.getInstance().getZooKeepers(), connector.getInstance().getInstanceName(), connector.getInstance().getZooKeepersSessionTimeOut(), connector.whoami(), token, clearCacheAfterFirstWrite, splitAfterFirstWrite); }
@Override public void initialize(InputSplit inSplit, TaskAttemptContext attempt) throws IOException { super.initialize(inSplit, attempt); rowIterator = new RowIterator(scannerIterator); currentK = new Text(); try { conf = new AccumuloGraphConfiguration(); conf.setZooKeeperHosts(EdgeInputFormat.getInstance(attempt).getZooKeepers()); conf.setInstanceName(EdgeInputFormat.getInstance(attempt).getInstanceName()); conf.setUser(EdgeInputFormat.getPrincipal(attempt)); conf.setTokenWithFallback(EdgeInputFormat.getToken(attempt)); conf.setGraphName(attempt.getConfiguration().get(GRAPH_NAME)); if (EdgeInputFormat.getInstance(attempt) instanceof MockInstance) { conf.setInstanceType(InstanceType.Mock); } parent = AccumuloGraph.open(conf.getConfiguration()); } catch (AccumuloException e) { throw new AccumuloGraphException(e); } }
@Override public void initialize(InputSplit inSplit, TaskAttemptContext attempt) throws IOException { super.initialize(inSplit, attempt); rowIterator = new RowIterator(scannerIterator); currentK = new Text(); try { conf = new AccumuloGraphConfiguration(); conf.setZooKeeperHosts(VertexInputFormat.getInstance(attempt).getZooKeepers()); conf.setInstanceName(VertexInputFormat.getInstance(attempt).getInstanceName()); conf.setUser(VertexInputFormat.getPrincipal(attempt)); conf.setPassword(VertexInputFormat.getToken(attempt)); conf.setGraphName(attempt.getConfiguration().get(GRAPH_NAME)); if (VertexInputFormat.getInstance(attempt) instanceof MockInstance) { conf.setInstanceType(InstanceType.Mock); } parent = AccumuloGraph.open(conf.getConfiguration()); } catch (AccumuloException e) { throw new AccumuloGraphException(e); } }
@Override public void initialize(InputSplit inSplit, TaskAttemptContext attempt) throws IOException { super.initialize(inSplit, attempt); rowIterator = new RowIterator(scannerIterator); currentK = new Text(); try { conf = new AccumuloGraphConfiguration(); conf.setZooKeeperHosts(EdgeInputFormat.getInstance(attempt).getZooKeepers()); conf.setInstanceName(EdgeInputFormat.getInstance(attempt).getInstanceName()); conf.setUser(EdgeInputFormat.getPrincipal(attempt)); conf.setPassword(EdgeInputFormat.getToken(attempt)); conf.setGraphName(attempt.getConfiguration().get(GRAPH_NAME)); if (EdgeInputFormat.getInstance(attempt) instanceof MockInstance) { conf.setInstanceType(InstanceType.Mock); } parent = AccumuloGraph.open(conf.getConfiguration()); } catch (AccumuloException e) { throw new AccumuloGraphException(e); } }
/** * Central place to set common split configuration not handled by split constructors. The * intention is to make it harder to miss optional setters in future refactor. */ public static void updateSplit(RangeInputSplit split, Instance instance, InputTableConfig tableConfig, String principal, AuthenticationToken token, Authorizations auths, Level logLevel) { split.setInstanceName(instance.getInstanceName()); split.setZooKeepers(instance.getZooKeepers()); DeprecationUtil.setMockInstance(split, DeprecationUtil.isMockInstance(instance)); split.setPrincipal(principal); split.setToken(token); split.setAuths(auths); split.setFetchedColumns(tableConfig.getFetchedColumns()); split.setIterators(tableConfig.getIterators()); split.setLogLevel(logLevel); split.setSamplerConfiguration(tableConfig.getSamplerConfiguration()); }
protected AccumuloRdfConfiguration makeConfig(final String instanceName, final String zookeepers) { final AccumuloRdfConfiguration conf = new AccumuloRdfConfiguration(); conf.setTablePrefix(RYA_INSTANCE_NAME); // Accumulo connection information. conf.setAccumuloUser(AccumuloExportITBase.ACCUMULO_USER); conf.setAccumuloPassword(AccumuloExportITBase.ACCUMULO_PASSWORD); conf.setAccumuloInstance(super.getAccumuloConnector().getInstance().getInstanceName()); conf.setAccumuloZookeepers(super.getAccumuloConnector().getInstance().getZooKeepers()); conf.setAuths(""); // PCJ configuration information. conf.set(ConfigUtils.USE_PCJ, "true"); conf.set(ConfigUtils.USE_PCJ_UPDATER_INDEX, "true"); conf.set(ConfigUtils.FLUO_APP_NAME, super.getFluoConfiguration().getApplicationName()); conf.set(ConfigUtils.PCJ_STORAGE_TYPE, PrecomputedJoinIndexerConfig.PrecomputedJoinStorageType.ACCUMULO.toString()); conf.set(ConfigUtils.PCJ_UPDATER_TYPE, PrecomputedJoinIndexerConfig.PrecomputedJoinUpdaterType.FLUO.toString()); conf.setDisplayQueryPlan(true); return conf; }
protected AccumuloRdfConfiguration makeConfig(final String instanceName, final String zookeepers) { final AccumuloRdfConfiguration conf = new AccumuloRdfConfiguration(); conf.setTablePrefix(RYA_INSTANCE_NAME); // Accumulo connection information. conf.setAccumuloUser(AccumuloExportITBase.ACCUMULO_USER); conf.setAccumuloPassword(AccumuloExportITBase.ACCUMULO_PASSWORD); conf.setAccumuloInstance(super.getAccumuloConnector().getInstance().getInstanceName()); conf.setAccumuloZookeepers(super.getAccumuloConnector().getInstance().getZooKeepers()); conf.setAuths(""); // PCJ configuration information. conf.set(ConfigUtils.USE_PCJ, "true"); conf.set(ConfigUtils.USE_PCJ_UPDATER_INDEX, "true"); conf.set(ConfigUtils.FLUO_APP_NAME, super.getFluoConfiguration().getApplicationName()); conf.set(ConfigUtils.PCJ_STORAGE_TYPE, PrecomputedJoinIndexerConfig.PrecomputedJoinStorageType.ACCUMULO.toString()); conf.set(ConfigUtils.PCJ_UPDATER_TYPE, PrecomputedJoinIndexerConfig.PrecomputedJoinUpdaterType.FLUO.toString()); conf.setDisplayQueryPlan(true); return conf; }