public FluoClientImpl(FluoConfiguration config) { this.config = config; if (!config.hasRequiredClientProps()) { String msg = "Client configuration is missing required properties"; log.error(msg); throw new IllegalArgumentException(msg); } try { this.env = new Environment(config); } catch (Exception e) { throw new IllegalStateException(e); } reporter = setupReporters(env, "client", reporterCounter); }
@Override public void waitForObservers() { try { Scanner scanner = env.getConnector().createScanner(env.getTable(), env.getAuthorizations()); Notification.configureScanner(scanner); while (isProcessing(scanner)) { Thread.sleep(100); } } catch (Exception e) { throw new RuntimeException(e); } } }
@Override public void close() { env.close(); try { reporter.close(); } catch (Exception e) { throw new RuntimeException(e); } } }
public OracleClient(Environment env) { this.env = env; responseTimer = MetricsUtil.getTimer(env.getConfiguration(), env.getSharedResources().getMetricRegistry(), env.getMetricNames().getOracleResponseTime()); stampsHistogram = MetricsUtil.getHistogram(env.getConfiguration(), env.getSharedResources() .getMetricRegistry(), env.getMetricNames().getOracleClientStamps()); timestampRetriever = new TimestampRetriever(); thread = new Thread(timestampRetriever); thread.setDaemon(true); thread.start(); }
private List<TabletInfo<T>> listSplits() throws TableNotFoundException, AccumuloSecurityException, AccumuloException { List<Text> splits = new ArrayList<>(env.getConnector().tableOperations().listSplits(env.getTable())); Collections.sort(splits); List<TabletInfo<T>> tablets = new ArrayList<>(splits.size() + 1); for (int i = 0; i < splits.size(); i++) { tablets .add(new TabletInfo<>(i == 0 ? null : splits.get(i - 1), splits.get(i), supplier.get())); } tablets.add(new TabletInfo<>(splits.size() == 0 ? null : splits.get(splits.size() - 1), null, supplier.get())); listSplitsTime = System.currentTimeMillis(); return tablets; }
/** * Configure properties needed to connect to a Fluo application * * @param conf Job configuration * @param config use {@link FluoConfiguration} to configure programmatically */ @SuppressWarnings("deprecation") public static void configure(Job conf, Configuration config) { try { FluoConfiguration fconfig = new FluoConfiguration(config); try (Environment env = new Environment(fconfig)) { long ts = env.getSharedResources().getTimestampTracker().allocateTimestamp().getTxTimestamp(); conf.getConfiguration().setLong(TIMESTAMP_CONF_KEY, ts); ByteArrayOutputStream baos = new ByteArrayOutputStream(); ConfigurationConverter.getProperties(config).store(baos, ""); conf.getConfiguration().set(PROPS_CONF_KEY, new String(baos.toByteArray(), "UTF8")); AccumuloInputFormat.setZooKeeperInstance(conf, fconfig.getAccumuloInstance(), fconfig.getAccumuloZookeepers()); AccumuloInputFormat.setConnectorInfo(conf, fconfig.getAccumuloUser(), new PasswordToken( fconfig.getAccumuloPassword())); AccumuloInputFormat.setInputTableName(conf, env.getTable()); AccumuloInputFormat.setScanAuthorizations(conf, env.getAuthorizations()); } } catch (Exception e) { throw new RuntimeException(e); } }
public SharedResources(Environment env) throws TableNotFoundException { this.env = env; curator = CuratorUtil.newAppCurator(env.getConfiguration()); curator.start(); int numTservers = env.getConnector().instanceOperations().getTabletServers().size(); int numBWThreads = FluoConfigurationImpl.getNumBWThreads(env.getConfiguration(), numTservers); bw = env.getConnector().createBatchWriter(env.getTable(), new BatchWriterConfig().setMaxWriteThreads(numBWThreads)); sbw = new SharedBatchWriter(bw); int numCWThreads = FluoConfigurationImpl.getNumCWThreads(env.getConfiguration(), numTservers); cw = env.getConnector().createConditionalWriter( env.getTable(), new ConditionalWriterConfig().setAuthorizations(env.getAuthorizations()) .setMaxWriteThreads(numCWThreads)); bulkCw = env.getConnector().createConditionalWriter( env.getTable(), new ConditionalWriterConfig().setAuthorizations(env.getAuthorizations()) .setMaxWriteThreads(numCWThreads)); txInfoCache = new TxInfoCache(env); visCache = new VisibilityCache(); metricRegistry = new MetricRegistry(); }
@Override public MetricRegistry getMetricRegistry() { return env.getSharedResources().getMetricRegistry(); }
try { scanner = env.getConnector().createScanner(env.getTable(), env.getAuthorizations()); } catch (TableNotFoundException e) { env.getSharedResources().getBatchWriter() .writeMutationAsync(notification.newDelete(env)); return true;
try (Environment env = new Environment(config); AutoCloseable reporters = ReporterUtil.setupReporters(env); NodeCache appIdCache = ClusterUtil.startAppIdWatcher(env)) { log.info("Starting Oracle for Fluo '{}' application with the following configuration:", config.getApplicationName()); env.getConfiguration().print();
@Override public Configuration getConfiguration() { return env.getConfiguration(); }
public void waitUntilFinished(FluoConfiguration config) { FluoConfiguration waitConfig = new FluoConfiguration(config); waitConfig.setClientRetryTimeout(500); try (Environment env = new Environment(waitConfig)) { log.info("The wait command will exit when all notifications are processed"); while (true) { long ts1 = env.getSharedResources().getOracleClient().getStamp().getTxTimestamp(); long ntfyCount = countNotifications(env); long ts2 = env.getSharedResources().getOracleClient().getStamp().getTxTimestamp(); if (ntfyCount == 0 && ts1 == (ts2 - 1)) { log.info("All processing has finished!");
TransactorCache transactorCache = env.getSharedResources().getTransactorCache(); if (System.currentTimeMillis() - startTime > env.getConfiguration() .getTransactionRollbackTime()) { locksToRecover = locks; TxInfoCache txiCache = env.getSharedResources().getTxInfoCache(); env.getSharedResources().getBatchWriter().writeMutations(new ArrayList<>(mutations.values()));
@Override public void setWeakNotification(Bytes row, Column col) { checkIfOpen(); Preconditions.checkNotNull(row); Preconditions.checkNotNull(col); if (!env.getWeakObservers().containsKey(col)) { throw new IllegalArgumentException("Column not configured for weak notifications " + col); } env.getSharedResources().getVisCache().validate(col); Set<Column> columns = weakNotifications.get(row); if (columns == null) { columns = new HashSet<>(); weakNotifications.put(row, columns); } columns.add(col); }
/** * Configure properties needed to connect to a Fluo application * * @param conf Job configuration * @param config use {@link io.fluo.api.config.FluoConfiguration} to configure programmatically */ @SuppressWarnings("deprecation") public static void configure(Job conf, Configuration config) { try { FluoConfiguration fconfig = new FluoConfiguration(config); try (Environment env = new Environment(fconfig)) { long ts = env.getSharedResources().getTimestampTracker().allocateTimestamp().getTxTimestamp(); conf.getConfiguration().setLong(TIMESTAMP_CONF_KEY, ts); ByteArrayOutputStream baos = new ByteArrayOutputStream(); ConfigurationConverter.getProperties(config).store(baos, ""); conf.getConfiguration().set(PROPS_CONF_KEY, new String(baos.toByteArray(), "UTF8")); AccumuloInputFormat.setZooKeeperInstance(conf, fconfig.getAccumuloInstance(), fconfig.getAccumuloZookeepers()); AccumuloInputFormat.setConnectorInfo(conf, fconfig.getAccumuloUser(), new PasswordToken( fconfig.getAccumuloPassword())); AccumuloInputFormat.setInputTableName(conf, env.getTable()); AccumuloInputFormat.setScanAuthorizations(conf, env.getAuthorizations()); } } catch (Exception e) { throw new RuntimeException(e); } }
GcTimestampTracker() throws Exception { this.curator = env.getSharedResources().getCurator(); }
public OracleServer(Environment env) throws Exception { this.env = env; stampsHistogram = MetricsUtil.getHistogram(env.getConfiguration(), env.getSharedResources() .getMetricRegistry(), env.getMetricNames().getOracleServerStamps()); this.cnxnListener = new CuratorCnxnListener(); this.maxTsPath = ZookeeperPath.ORACLE_MAX_TIMESTAMP; this.oraclePath = ZookeeperPath.ORACLE_SERVER; }
try (Environment env = new Environment(config); AutoCloseable reporters = ReporterUtil.setupReporters(env); NodeCache appIdCache = ClusterUtil.startAppIdWatcher(env)) { log.info("Starting Worker for Fluo '{}' application with the following configuration:", config.getApplicationName()); env.getConfiguration().print(); NotificationFinderFactory.newNotificationFinder(env.getConfiguration()); notificationFinder.init(env, np); notificationFinder.start();
public TimestampTracker(Environment env, TransactorID tid) { this(env, tid, env.getConfiguration().getLong(FluoConfigurationImpl.ZK_UPDATE_PERIOD_PROP, FluoConfigurationImpl.ZK_UPDATE_PERIOD_MS_DEFAULT)); }
@VisibleForTesting public long countNotifications(Environment env) { Scanner scanner = null; try { scanner = env.getConnector().createScanner(env.getTable(), env.getAuthorizations()); } catch (TableNotFoundException e) { log.error("An exception was thrown -", e); throw new FluoException(e); } Notification.configureScanner(scanner); long count = 0; for (Iterator<Map.Entry<Key, Value>> iterator = scanner.iterator(); iterator.hasNext(); iterator .next()) { count++; } return count; }