/** * Uses underlying parameter map to build a FluoConfiguration object * @return - FluoConfiguration for creating a FluoClient */ public FluoConfiguration getFluoConfiguration() { final FluoConfiguration config = new FluoConfiguration(); config.setMiniStartAccumulo(false); config.setAccumuloInstance(params.get(CONF_ACCUMULO_INSTANCE_NAME)); config.setAccumuloUser(params.get(CONF_EXPORTER_USERNAME)); config.setAccumuloPassword(params.get(CONF_EXPORTER_PASSWORD)); config.setInstanceZookeepers(params.get(CONF_FLUO_INSTANCE_ZOOKEEPERS)); config.setAccumuloZookeepers(params.get(CONF_ZOOKEEPER_SERVERS)); config.setApplicationName(params.get(CONF_FLUO_APP_NAME)); config.setAccumuloTable(params.get(CONF_FLUO_TABLE_NAME)); return config; }
@Test public void testBasic() { try (FluoClient client = FluoFactory.newClient(config)) { client.newSnapshot(); } FluoConfiguration fluoConfig = new FluoConfiguration(); fluoConfig.setApplicationName(config.getApplicationName()); fluoConfig.setInstanceZookeepers(config.getInstanceZookeepers()); try (FluoClient client = FluoFactory.newClient(fluoConfig)) { client.newSnapshot(); } try (FluoClientImpl client = new FluoClientImpl(fluoConfig)) { client.newSnapshot(); FluoConfiguration sharedConfig = client.getSharedConfiguration(); Assert.assertEquals(config.getAccumuloTable(), sharedConfig.getAccumuloTable()); Assert.assertEquals(config.getAccumuloInstance(), sharedConfig.getAccumuloInstance()); Assert.assertEquals(config.getAccumuloUser(), sharedConfig.getAccumuloUser()); Assert.assertEquals(config.getZookeeperTimeout(), sharedConfig.getZookeeperTimeout()); Assert.assertEquals(config.getTransactionRollbackTime(), sharedConfig.getTransactionRollbackTime()); } }
/** * Returns the zookeeper application name string. * * @return The zookeeper application string. */ public String getAppZookeepers() { return getInstanceZookeepers() + "/" + getApplicationName(); }
public void validate() { // keep in alphabetical order getAccumuloInstance(); getAccumuloPassword(); getAccumuloTable(); getAccumuloUser(); getAccumuloZookeepers(); getApplicationName(); getAppZookeepers(); getConnectionRetryTimeout(); getLoaderQueueSize(); getLoaderThreads(); getObserverSpecifications(); getTransactionRollbackTime(); getWorkerThreads(); getZookeeperTimeout(); }
@Before public void setUpFluo() throws Exception { config = new FluoConfiguration(); config.setApplicationName("mini-test" + testCounter.getAndIncrement()); config.setAccumuloInstance(clientInfo.getInstanceName()); config.setAccumuloUser(USER); config.setAccumuloPassword(PASSWORD); config.setAccumuloZookeepers(clientInfo.getZooKeepers()); config.setInstanceZookeepers(clientInfo.getZooKeepers() + "/fluo"); config.setAccumuloTable(getNextTableName()); config.setWorkerThreads(5); setupObservers(config); config.setMiniStartAccumulo(false); setConfig(config); setAppConfig(config.getAppConfiguration()); config.setTransactionRollbackTime(1, TimeUnit.SECONDS); try (FluoAdmin admin = FluoFactory.newAdmin(config)) { InitializationOptions opts = new InitializationOptions().setClearZookeeper(true).setClearTable(true); admin.initialize(opts); } config.getAppConfiguration().clear(); client = FluoFactory.newClient(config); miniFluo = FluoFactory.newMiniFluo(config); }
private void startMiniAccumulo() { try { // start mini accumulo cluster MiniAccumuloConfig cfg = new MiniAccumuloConfig(new File(config.getMiniDataDir()), PASSWORD); cluster = new MiniAccumuloCluster(cfg); cluster.start(); log.debug("Started MiniAccumulo(accumulo=" + cluster.getInstanceName() + " zk=" + cluster.getZooKeepers() + ")"); // configuration that must overridden config.setAccumuloInstance(cluster.getInstanceName()); config.setAccumuloUser(USER); config.setAccumuloPassword(PASSWORD); config.setAccumuloZookeepers(cluster.getZooKeepers()); config.setInstanceZookeepers(cluster.getZooKeepers() + "/fluo"); // configuration that only needs to be set if not by user if ((config.containsKey(FluoConfiguration.ACCUMULO_TABLE_PROP) == false) || config.getAccumuloTable().trim().isEmpty()) { config.setAccumuloTable("fluo"); } InitializationOptions opts = new InitializationOptions(); try (FluoAdmin admin = FluoFactory.newAdmin(config)) { admin.initialize(opts); } File miniProps = new File(clientPropsPath(config)); config.getClientConfiguration().save(miniProps); log.debug("Wrote MiniFluo client properties to {}", miniProps.getAbsolutePath()); } catch (Exception e) { throw new RuntimeException(e); } }
/** * A utility method that will set the configuration needed by Fluo from a given MiniCluster */ public static void configureFromMAC(FluoConfiguration fluoConfig, MiniAccumuloCluster cluster) { fluoConfig.setMiniStartAccumulo(false); fluoConfig.setAccumuloInstance(cluster.getInstanceName()); fluoConfig.setAccumuloUser("root"); fluoConfig.setAccumuloPassword(cluster.getConfig().getRootPassword()); fluoConfig.setInstanceZookeepers(cluster.getZooKeepers() + "/fluo"); fluoConfig.setAccumuloZookeepers(cluster.getZooKeepers()); }
@Override public void initialize(InitializationOptions opts) throws AlreadyInitializedException, TableExistsException { if (!config.hasRequiredAdminProps()) { throw new IllegalArgumentException("Admin configuration is missing required properties"); !ZookeeperUtil.parseRoot(config.getInstanceZookeepers()).equals("/"), "The Zookeeper connection string (set by 'fluo.connection.zookeepers') " + " must have a chroot suffix."); config.getObserverJarsUrl().isEmpty() || config.getObserverInitDir().isEmpty(), "Only one of 'fluo.observer.init.dir' and 'fluo.observer.jars.url' can be set"); "Fluo application already initialized at " + config.getAppZookeepers()); boolean tableExists = conn.tableOperations().exists(config.getAccumuloTable()); if (tableExists && !opts.getClearTable()) { throw new TableExistsException("Accumulo table already exists " + config.getAccumuloTable()); config.getAccumuloTable()); try { conn.tableOperations().delete(config.getAccumuloTable()); } catch (Exception e) { throw new RuntimeException(e); if (rootCurator.checkExists().forPath(appRootDir) != null) { logger.info("Clearing Fluo '{}' application in Zookeeper at {}", config.getApplicationName(), config.getAppZookeepers()); rootCurator.delete().deletingChildrenIfNeeded().forPath(appRootDir);
config.load(applicationPropsFile); config.setApplicationName(opts.getApplicationName()); opts.overrideFluoConfig(config); if (config.containsKey(propKey)) { System.out.println(config.getString(propKey)); if (!config.hasRequiredAdminProps()) { System.err.println("Error - Required properties are not set in " + opts.getAppPropsPath()); System.exit(-1); config.validate(); } catch (Exception e) { System.err.println("Error - Invalid configuration due to " + e.getMessage()); System.out.println("Updating configuration for the Fluo '" + config.getApplicationName() + "' application in Zookeeper using " + opts.getAppPropsPath()); admin.updateSharedConfig(); initOpts.setClearZookeeper(true); } else if (admin.zookeeperInitialized()) { System.out.print("A Fluo '" + config.getApplicationName() + "' application is already initialized in Zookeeper at " + config.getAppZookeepers() + " - Would you like to clear and reinitialize Zookeeper" + " for this application (y/n)? "); initOpts.setClearTable(true); } else if (admin.accumuloTableExists()) { System.out.print("The Accumulo table '" + config.getAccumuloTable()
public static void main(String[] args) { if (args.length != 1) { System.out.println("ERROR - Expected usage: ValidateAppName <fluoApplicationName>"); System.exit(-1); } FluoConfiguration config = new FluoConfiguration(); try { config.setApplicationName(args[0]); } catch (IllegalArgumentException e) { System.out.println("ERROR - " + e.getMessage()); System.exit(-1); } } }
FluoConfiguration localConfig = new FluoConfiguration(config); localConfig.setProperty("fluo.test123", "${fluo.connection.application.name}"); Assert.assertEquals(localConfig.getApplicationName(), localConfig.getString("fluo.test123")); client.tableOperations().getLocalityGroups(config.getAccumuloTable()); Assert.assertEquals("Unexpected locality group count.", 1, localityGroups.size()); Entry<String, Set<Text>> localityGroup = localityGroups.entrySet().iterator().next(); Assert.assertEquals(localConfig.getApplicationName(), sharedConfig.getString("fluo.test123")); Assert.assertEquals(localConfig.getApplicationName(), sharedConfig.getApplicationName());
private void resetFluoConfig() { fluoConfig = new FluoConfiguration(); configureFromMAC(fluoConfig, cluster); fluoConfig.setApplicationName("fluo-it"); fluoConfig.setAccumuloTable("fluo" + tableCounter.getAndIncrement()); } }
@Override public List<AutoCloseable> start(Params params) { SimpleConfiguration config = new FluoConfiguration(params.getConfiguration()).getReporterConfiguration("csv"); String dir = config.getString("dir", ""); if (!config.getBoolean("enable", false) || dir.isEmpty()) { return Collections.emptyList(); } TimeUnit rateUnit = TimeUnit.valueOf(config.getString("rateUnit", "seconds").toUpperCase()); TimeUnit durationUnit = TimeUnit.valueOf(config.getString("durationUnit", "milliseconds").toUpperCase()); CsvReporter reporter = CsvReporter.forRegistry(params.getMetricRegistry()) .convertDurationsTo(durationUnit).convertRatesTo(rateUnit).build(new File(dir)); reporter.start(config.getInt("frequency", 60), TimeUnit.SECONDS); log.info("Reporting metrics as csv to directory {}", dir); return Collections.singletonList((AutoCloseable) reporter); }
@Override public void remove() { if (!config.hasRequiredAdminProps()) { throw new IllegalArgumentException("Admin configuration is missing required properties"); !ZookeeperUtil.parseRoot(config.getInstanceZookeepers()).equals("/"), "The Zookeeper connection string (set by 'fluo.connection.zookeepers') " + " must have a chroot suffix."); boolean tableExists = conn.tableOperations().exists(config.getAccumuloTable()); logger.info("The Accumulo table '{}' will be dropped", config.getAccumuloTable()); try { conn.tableOperations().delete(config.getAccumuloTable()); } catch (Exception e) { throw new RuntimeException(e); if (rootCurator.checkExists().forPath(appRootDir) != null) { logger.info("Clearing Fluo '{}' application in Zookeeper at {}", config.getApplicationName(), config.getAppZookeepers()); rootCurator.delete().deletingChildrenIfNeeded().forPath(appRootDir); logger.error("An error occurred deleting Zookeeper root of [" + config.getAppZookeepers() + "], error=[" + e.getMessage() + "]"); throw new RuntimeException(e);
public FluoConfiguration getAppConfiguration(String appName, boolean debug) { verifyAppInstall(appName); String propsPath = getAppPropsPath(appName); FluoConfiguration config = new FluoConfiguration(new File(propsPath)); if (!config.getApplicationName().equals(appName)) { throw new FluoException("Application name in config '" + config.getApplicationName() + "' does not match given appName: " + appName); } if (debug) { System.out.println("Connecting to Fluo instance (" + config.getInstanceZookeepers() + ") using config (" + stripFluoHomeDir(propsPath) + ")"); } return config; }
if (!config.hasRequiredAdminProps()) { System.err.println("Error - Required properties are not set in " + propsPath); System.exit(-1); config.validate(); } catch (IllegalArgumentException e) { System.err System.err.println("Error - The Fluo '" + config.getApplicationName() + "' application" + " is already running and must be stopped before running 'fluo init'. " + " Aborted initialization."); System.out.println("Updating configuration for the Fluo '" + config.getApplicationName() + "' application in Zookeeper using " + propsPath); admin.updateSharedConfig(); initOpts.setClearZookeeper(true); } else if (admin.zookeeperInitialized()) { System.out.print("A Fluo '" + config.getApplicationName() + "' application is already initialized in Zookeeper at " + config.getAppZookeepers() + " - Would you like to clear and reinitialize Zookeeper" + " for this application (y/n)? "); initOpts.setClearTable(true); } else if (admin.accumuloTableExists()) { System.out.print("The Accumulo table '" + config.getAccumuloTable() + "' already exists - Would you like to drop and recreate this table (y/n)? "); if (readYes()) {
final FluoConfiguration config = new FluoConfiguration(); config.setApplicationName("IntegrationTests"); config.setMiniDataDir(miniDataDir.getAbsolutePath()); config.addObservers(observers);
private static Environment initOracleEnv(FluoConfiguration config, int port) { FluoConfiguration c = new FluoConfiguration(config); c.setProperty(FluoConfigurationImpl.ORACLE_PORT_PROP, port); return new Environment(c); }
public FluoConfiguration getFluoConfiguration(boolean debug) { verifyFluoInstall(); String propsPath = getFluoPropsPath(); FluoConfiguration config = new FluoConfiguration(new File(propsPath)); if (debug) { System.out.println("Connecting to Fluo instance (" + config.getInstanceZookeepers() + ") using config (" + stripFluoHomeDir(propsPath) + ")"); } return config; }
public static void verifyAppInitialized(FluoConfiguration config) { if (!FluoAdminImpl.isInitialized(config)) { System.out.println("A Fluo '" + config.getApplicationName() + "' application has not " + "been initialized yet in Zookeeper at " + config.getAppZookeepers()); System.exit(-1); } }