private Set<String> getSortedWALogs() throws IOException { AccumuloConfiguration conf = instance.getConfiguration(); Path recoveryDir = new Path(Constants.getRecoveryDir(conf)); Set<String> sortedWALogs = new HashSet<String>(); if (fs.exists(recoveryDir)) { for (FileStatus status : fs.listStatus(recoveryDir)) { if (isUUID(status.getPath().getName())) { sortedWALogs.add(status.getPath().getName()); } else { log.debug("Ignoring file " + status.getPath() + " because it doesn't look like a uuid"); } } } return sortedWALogs; }
@Override public void run() { List<TabletLocation> tabletsToAssignMapFileTo = Collections.emptyList(); try { tabletsToAssignMapFileTo = findOverlappingTablets(instance.getConfiguration(), fs, locator, mapFile, credentials); } catch (Exception ex) { log.warn("Unable to find tablets that overlap file " + mapFile.toString()); } log.debug("Map file " + mapFile + " found to overlap " + tabletsToAssignMapFileTo.size() + " tablets"); if (tabletsToAssignMapFileTo.size() == 0) { List<KeyExtent> empty = Collections.emptyList(); completeFailures.put(mapFile, empty); } else assignments.put(mapFile, tabletsToAssignMapFileTo); } };
private InetSocketAddress startStatsService(String hostname) throws UnknownHostException { Processor<Iface> processor = new Processor<Iface>(TraceWrap.service(this)); int port = instance.getConfiguration().getPort(Property.GC_PORT); long maxMessageSize = instance.getConfiguration().getMemoryInBytes(Property.GENERAL_MAX_MESSAGE_SIZE); try { port = TServerUtils.startTServer(port, processor, this.getClass().getSimpleName(), "GC Monitor Service", 2, 1000, maxMessageSize).port; } catch (Exception ex) { log.fatal(ex, ex); throw new RuntimeException(ex); } return new InetSocketAddress(hostname, port); }
public void init(FileSystem fs, Instance instance, TCredentials credentials, boolean noTrash) throws IOException { this.fs = TraceFileSystem.wrap(fs); this.credentials = credentials; this.instance = instance; gcStartDelay = instance.getConfiguration().getTimeInMillis(Property.GC_CYCLE_START); long gcDelay = instance.getConfiguration().getTimeInMillis(Property.GC_CYCLE_DELAY); numDeleteThreads = instance.getConfiguration().getCount(Property.GC_DELETE_THREADS); log.info("start delay: " + (offline ? 0 + " sec (offline)" : gcStartDelay + " milliseconds")); log.info("time delay: " + gcDelay + " milliseconds"); log.info("safemode: " + safemode); log.info("offline: " + offline); log.info("verbose: " + verbose); log.info("memory threshold: " + CANDIDATE_MEMORY_PERCENTAGE + " of " + Runtime.getRuntime().maxMemory() + " bytes"); log.info("delete threads: " + numDeleteThreads); if (!noTrash) { this.trash = new Trash(fs, fs.getConf()); } }
@Override public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { FileSplit split = (FileSplit) inputSplit; Configuration job = taskAttemptContext.getConfiguration(); Path file = split.getPath(); FileSystem fs = file.getFileSystem(job); Instance instance = MRUtils.AccumuloProps.getInstance(taskAttemptContext); fileSKVIterator = RFileOperations.getInstance().openReader(file.toString(), ALLRANGE, new HashSet<ByteSequence>(), false, fs, job, instance.getConfiguration()); }
@Override public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { FileSplit split = (FileSplit) inputSplit; Configuration job = taskAttemptContext.getConfiguration(); Path file = split.getPath(); FileSystem fs = file.getFileSystem(job); Instance instance = MRUtils.AccumuloProps.getInstance(taskAttemptContext); fileSKVIterator = RFileOperations.getInstance().openReader(file.toString(), ALLRANGE, new HashSet<ByteSequence>(), false, fs, job, instance.getConfiguration()); }
private int scanServers(Map<String,String> fileToServerMap) throws Exception { AccumuloConfiguration conf = instance.getConfiguration(); Path walRoot = new Path(Constants.getWalDirectory(conf)); for (FileStatus status : fs.listStatus(walRoot)) { String name = status.getPath().getName(); if (status.isDir()) { for (FileStatus file : fs.listStatus(new Path(walRoot, name))) { if (isUUID(file.getPath().getName())) fileToServerMap.put(file.getPath().getName(), name); else { log.info("Ignoring file " + file.getPath() + " because it doesn't look like a uuid"); } } } else if (isUUID(name)) { // old-style WAL are not under a directory fileToServerMap.put(name, ""); } else { log.info("Ignoring file " + name + " because it doesn't look like a uuid"); } } int count = 0; return count; }
private void updatePlugins(String property) { if (property.equals(Property.MASTER_TABLET_BALANCER.getKey())) { TabletBalancer balancer = createInstanceFromPropertyName(instance.getConfiguration(), Property.MASTER_TABLET_BALANCER, TabletBalancer.class, new DefaultLoadBalancer()); balancer.init(serverConfig); tabletBalancer = balancer; log.info("tablet balancer changed to " + tabletBalancer.getClass().getName()); } }
private List<KeyExtent> assignMapFiles(TCredentials credentials, String location, Map<KeyExtent,List<PathSize>> assignmentsPerTablet) throws AccumuloException, AccumuloSecurityException { try { long timeInMillis = instance.getConfiguration().getTimeInMillis(Property.TSERV_BULK_TIMEOUT); TabletClientService.Iface client = ThriftUtil.getTServerClient(location, instance.getConfiguration(), timeInMillis); try { HashMap<KeyExtent,Map<String,org.apache.accumulo.core.data.thrift.MapFileInfo>> files = new HashMap<KeyExtent,Map<String,org.apache.accumulo.core.data.thrift.MapFileInfo>>(); for (Entry<KeyExtent,List<PathSize>> entry : assignmentsPerTablet.entrySet()) { HashMap<String,org.apache.accumulo.core.data.thrift.MapFileInfo> tabletFiles = new HashMap<String,org.apache.accumulo.core.data.thrift.MapFileInfo>(); files.put(entry.getKey(), tabletFiles); for (PathSize pathSize : entry.getValue()) { org.apache.accumulo.core.data.thrift.MapFileInfo mfi = new org.apache.accumulo.core.data.thrift.MapFileInfo(pathSize.estSize); tabletFiles.put(pathSize.path.toUri().getPath().toString(), mfi); } } log.debug("Asking " + location + " to bulk load " + files); List<TKeyExtent> failures = client.bulkImport(Tracer.traceInfo(), credentials, tid, Translator.translate(files, Translators.KET), setTime); return Translator.translate(failures, Translators.TKET); } finally { ThriftUtil.returnClient((TServiceClient) client); } } catch (ThriftSecurityException e) { throw new AccumuloSecurityException(e.user, e.code, e); } catch (Throwable t) { t.printStackTrace(); throw new AccumuloException(t); } }
private int removeFiles(Map<String,ArrayList<String>> serverToFileMap, Set<String> sortedWALogs, final GCStatus status) { AccumuloConfiguration conf = instance.getConfiguration(); for (Entry<String,ArrayList<String>> entry : serverToFileMap.entrySet()) { if (entry.getKey().length() == 0) {
tabletsToAssignMapFileTo.addAll(findOverlappingTablets(instance.getConfiguration(), fs, locator, entry.getKey(), ke, credentials)); keListIter.remove(); } catch (Exception ex) {
if (offline) { try { scanner = new OfflineMetadataScanner(instance.getConfiguration(), fs); } catch (IOException e) { throw new IllegalStateException("Unable to create offline metadata scanner", e);
long gcDelay = instance.getConfiguration().getTimeInMillis(Property.GC_CYCLE_DELAY); log.debug("Sleeping for " + gcDelay + " milliseconds"); Thread.sleep(gcDelay);
protected Connector createAccumuloConnection(BackupRestoreOptionsBase options) throws AccumuloSecurityException, AccumuloException { String instanceName = options.getAccumuloInstanceName(); String zooServers = options.getZookeeperServers(); Instance inst = new ZooKeeperInstance(instanceName, zooServers); ConfigurationCopy conf = new ConfigurationCopy(inst.getConfiguration()); conf.set(Property.INSTANCE_DFS_URI, options.getHadoopFsDefaultFS()); inst.setConfiguration(conf); AuthenticationToken authenticationToken = new PasswordToken(options.getAccumuloPassword()); return inst.getConnector(options.getAccumuloUserName(), authenticationToken); }
UtilWaitThread.sleep(instance.getConfiguration().getTimeInMillis(Property.MONITOR_LOCK_CHECK_INTERVAL));