private static Environment initOracleEnv(FluoConfiguration config, int port) { FluoConfiguration c = new FluoConfiguration(config); c.setProperty(FluoConfigurationImpl.ORACLE_PORT_PROP, port); return new Environment(c); }
private static int countNotifications(Environment env) throws Exception { Scanner scanner = env.getAccumuloClient().createScanner(env.getTable(), env.getAuthorizations()); Notification.configureScanner(scanner); return Iterables.size(scanner); }
@Override public void waitForObservers() { try { Scanner scanner = env.getConnector().createScanner(env.getTable(), env.getAuthorizations()); Notification.configureScanner(scanner); while (true) { long ts1 = env.getSharedResources().getOracleClient().getStamp().getTxTimestamp(); long ntfyCount = Iterables.size(scanner); long ts2 = env.getSharedResources().getOracleClient().getStamp().getTxTimestamp(); if (ntfyCount == 0 && ts1 == (ts2 - 1)) { break; } long sleepTime = ntfyCount / 2; sleepTime = Math.min(Math.max(10, sleepTime), 10000); Uninterruptibles.sleepUninterruptibly(sleepTime, TimeUnit.MILLISECONDS); } } catch (Exception e) { throw new RuntimeException(e); } }
public MetricsReporter getMetricsReporter() { return new MetricsReporterImpl(getConfiguration(), getSharedResources().getMetricRegistry(), getMetricsReporterID()); }
@VisibleForTesting public long countNotifications(Environment env) { Scanner scanner = null; try { scanner = env.getConnector().createScanner(env.getTable(), env.getAuthorizations()); } catch (TableNotFoundException e) { log.error("An exception was thrown -", e); throw new FluoException(e); } Notification.configureScanner(scanner); return Iterables.size(scanner); }
/** * Configure properties needed to connect to a Fluo application * * @param conf Job configuration * @param config use {@link FluoConfiguration} to configure programmatically */ public static void configure(Job conf, SimpleConfiguration config) { try { FluoConfiguration fconfig = new FluoConfiguration(config); try (Environment env = new Environment(fconfig)) { long ts = env.getSharedResources().getTimestampTracker().allocateTimestamp().getTxTimestamp(); conf.getConfiguration().setLong(TIMESTAMP_CONF_KEY, ts); ByteArrayOutputStream baos = new ByteArrayOutputStream(); config.save(baos); conf.getConfiguration().set(PROPS_CONF_KEY, new String(baos.toByteArray(), StandardCharsets.UTF_8)); AccumuloInputFormat.setClientInfo(conf, AccumuloUtil.getClientInfo(fconfig)); AccumuloInputFormat.setInputTableName(conf, env.getTable()); AccumuloInputFormat.setScanAuthorizations(conf, env.getAuthorizations()); } } catch (Exception e) { throw new RuntimeException(e); } }
@Override public void waitForObservers() { try { Scanner scanner = env.getAccumuloClient().createScanner(env.getTable(), env.getAuthorizations()); Notification.configureScanner(scanner); while (true) { long ts1 = env.getSharedResources().getOracleClient().getStamp().getTxTimestamp(); long ntfyCount = Iterables.size(scanner); long ts2 = env.getSharedResources().getOracleClient().getStamp().getTxTimestamp(); if (ntfyCount == 0 && ts1 == (ts2 - 1)) { break; } long sleepTime = ntfyCount / 2; sleepTime = Math.min(Math.max(10, sleepTime), 10000); Uninterruptibles.sleepUninterruptibly(sleepTime, TimeUnit.MILLISECONDS); } } catch (Exception e) { throw new RuntimeException(e); } }
public CommitData createCommitData() { CommitData cd = new CommitData(); cd.cw = env.getSharedResources().getConditionalWriter(); cd.acw = env.getSharedResources().getAsyncConditionalWriter(); cd.bacw = env.getSharedResources().getBulkAsyncConditionalWriter(); return cd; }
public OracleClient(Environment env) { this.env = env; responseTimer = MetricsUtil.getTimer(env.getConfiguration(), env.getSharedResources().getMetricRegistry(), env.getMetricNames().getOracleResponseTime()); stampsHistogram = MetricsUtil.getHistogram(env.getConfiguration(), env.getSharedResources().getMetricRegistry(), env.getMetricNames().getOracleClientStamps()); timestampRetriever = new TimestampRetriever(); thread = new Thread(timestampRetriever); thread.setDaemon(true); thread.start(); }
public SharedResources(Environment env) throws TableNotFoundException { this.env = env; curator = CuratorUtil.newAppCurator(env.getConfiguration()); curator.start(); int numTservers = env.getAccumuloClient().instanceOperations().getTabletServers().size(); int numBWThreads = FluoConfigurationImpl.getNumBWThreads(env.getConfiguration(), numTservers); bw = env.getAccumuloClient().createBatchWriter(env.getTable(), new BatchWriterConfig().setMaxWriteThreads(numBWThreads)); sbw = new SharedBatchWriter(bw); int numCWThreads = FluoConfigurationImpl.getNumCWThreads(env.getConfiguration(), numTservers); cw = env.getAccumuloClient().createConditionalWriter(env.getTable(), new ConditionalWriterConfig().setAuthorizations(env.getAuthorizations()) .setMaxWriteThreads(numCWThreads)); bulkCw = env.getAccumuloClient().createConditionalWriter(env.getTable(), new ConditionalWriterConfig().setAuthorizations(env.getAuthorizations()) .setMaxWriteThreads(numCWThreads)); txInfoCache = new TxInfoCache(env); visCache = new VisibilityCache(env.getConfiguration()); metricRegistry = new MetricRegistry(); int commitThreads = env.getConfiguration().getInt(FluoConfigurationImpl.ASYNC_COMMIT_THREADS, FluoConfigurationImpl.ASYNC_COMMIT_THREADS_DEFAULT); asyncCommitExecutor = FluoExecutors.newFixedThreadPool(commitThreads, "async-commits"); commitThreads = env.getConfiguration().getInt(FluoConfigurationImpl.SYNC_COMMIT_THREADS, FluoConfigurationImpl.SYNC_COMMIT_THREADS_DEFAULT); syncCommitExecutor = FluoExecutors.newFixedThreadPool(commitThreads, "sync-commits"); acw = new AsyncConditionalWriter(env, cw); bulkAcw = new AsyncConditionalWriter(env, bulkCw); }
@Override public void start() { try { env = new Environment(config); reporters = ReporterUtil.setupReporters(env); appIdCache = CuratorUtil.startAppIdWatcher(env); log.info("Starting Worker for Fluo '{}' application with the following configuration:", config.getApplicationName()); env.getConfiguration().print(); np = new NotificationProcessor(env); notificationFinder = NotificationFinderFactory.newNotificationFinder(env.getConfiguration()); notificationFinder.init(env, np); notificationFinder.start(); } catch (Exception e) { throw new FluoException(e); } }
public SharedResources(Environment env) throws TableNotFoundException { this.env = env; curator = CuratorUtil.newAppCurator(env.getConfiguration()); curator.start(); int numTservers = env.getConnector().instanceOperations().getTabletServers().size(); int numBWThreads = FluoConfigurationImpl.getNumBWThreads(env.getConfiguration(), numTservers); bw = env.getConnector().createBatchWriter(env.getTable(), new BatchWriterConfig().setMaxWriteThreads(numBWThreads)); sbw = new SharedBatchWriter(bw); int numCWThreads = FluoConfigurationImpl.getNumCWThreads(env.getConfiguration(), numTservers); cw = env.getConnector().createConditionalWriter(env.getTable(), new ConditionalWriterConfig() .setAuthorizations(env.getAuthorizations()).setMaxWriteThreads(numCWThreads)); bulkCw = env.getConnector().createConditionalWriter(env.getTable(), new ConditionalWriterConfig() .setAuthorizations(env.getAuthorizations()).setMaxWriteThreads(numCWThreads)); txInfoCache = new TxInfoCache(env); visCache = new VisibilityCache(env.getConfiguration()); metricRegistry = new MetricRegistry(); int commitThreads = env.getConfiguration().getInt(FluoConfigurationImpl.ASYNC_COMMIT_THREADS, FluoConfigurationImpl.ASYNC_COMMIT_THREADS_DEFAULT); asyncCommitExecutor = FluoExecutors.newFixedThreadPool(commitThreads, "async-commits"); commitThreads = env.getConfiguration().getInt(FluoConfigurationImpl.SYNC_COMMIT_THREADS, FluoConfigurationImpl.SYNC_COMMIT_THREADS_DEFAULT); syncCommitExecutor = FluoExecutors.newFixedThreadPool(commitThreads, "sync-commits"); acw = new AsyncConditionalWriter(env, cw); bulkAcw = new AsyncConditionalWriter(env, bulkCw); }
public static byte[] serializeTableSplits(Environment env) { List<Bytes> splits; try { splits = env.getAccumuloClient().tableOperations().listSplits(env.getTable()).stream() .map(ByteUtil::toBytes).collect(Collectors.toList()); } catch (TableNotFoundException | AccumuloSecurityException | AccumuloException e) { throw new RuntimeException(e); } return serialize(splits); } }
final Column statCheck = new Column("stat", "check"); Environment env = new Environment(config); Assert.assertEquals(0, countNotifications(env)); env.getAccumuloClient().tableOperations().flush(env.getTable(), null, null, true);
@Test public void testWeakNotification() throws Exception { Environment env = new Environment(config); Assert.assertTrue(tx6.preCommit(cd6)); Stamp commitTs5 = env.getSharedResources().getOracleClient().getStamp(); Assert.assertTrue(tx5.commitPrimaryColumn(cd5, commitTs5)); Stamp commitTs6 = env.getSharedResources().getOracleClient().getStamp(); Assert.assertTrue(tx6.commitPrimaryColumn(cd6, commitTs6)); Assert.assertEquals("39", tx7.gets("r1", STAT_COUNT)); env.close();
public NotificationProcessor(Environment env) { int numThreads = env.getConfiguration().getWorkerThreads(); this.env = env; this.queue = new PriorityBlockingQueue<>(); this.executor = FluoExecutors.newFixedThreadPool(numThreads, queue, "ntfyProc"); this.tracker = new NotificationTracker(); this.observers = env.getConfiguredObservers().getObservers(env); env.getSharedResources().getMetricRegistry().register( env.getMetricNames().getNotificationQueued(), (Gauge<Integer>) () -> queue.size()); }
@Override public void close() { env.close(); } }
@Test public void testVisibility() throws Exception { aClient.securityOperations().changeUserAuthorizations(USER, new Authorizations("A", "B", "C")); env.setAuthorizations(new Authorizations("A", "B", "C")); Column balanceCol = new Column("account", "balance", "A|B"); final TestTransaction tx = new TestTransaction(env); tx.set("bob", balanceCol, "10"); tx.set("joe", balanceCol, "20"); tx.set("jill", balanceCol, "60"); tx.done(); FluoConfiguration fc = new FluoConfiguration(config); Environment env2 = new Environment(fc); env2.setAuthorizations(new Authorizations("B")); final TestTransaction tx2 = new TestTransaction(env2); Assert.assertEquals("10", tx2.gets("bob", balanceCol)); Assert.assertEquals("20", tx2.gets("joe", balanceCol)); Assert.assertEquals("60", tx2.gets("jill", balanceCol)); tx2.done(); env2.close(); Environment env3 = new Environment(fc); env3.setAuthorizations(new Authorizations("C")); final TestTransaction tx3 = new TestTransaction(env3); Assert.assertNull(tx3.gets("bob", balanceCol)); Assert.assertNull(tx3.gets("joe", balanceCol)); Assert.assertNull(tx3.gets("jill", balanceCol)); tx3.done(); env3.close(); }
@Override public void setWeakNotification(Bytes row, Column col) { checkIfOpen(); Objects.requireNonNull(row); Objects.requireNonNull(col); if (!env.getConfiguredObservers().getObservedColumns(WEAK).contains(col)) { throw new IllegalArgumentException("Column not configured for weak notifications " + col); } env.getSharedResources().getVisCache().validate(col); Set<Column> columns = weakNotifications.get(row); if (columns == null) { columns = new HashSet<>(); weakNotifications.put(row, columns); } columns.add(col); }
public static byte[] serializeTableSplits(Environment env) { List<Bytes> splits; try { splits = env.getConnector().tableOperations().listSplits(env.getTable()).stream() .map(ByteUtil::toBytes).collect(Collectors.toList()); } catch (TableNotFoundException | AccumuloSecurityException | AccumuloException e) { throw new RuntimeException(e); } return serialize(splits); } }