public static List<String> bulkLoad(ServerContext context, long tid, String tableId, List<String> files, boolean setTime) throws IOException { AssignmentStats stats = new BulkImporter(context, tid, tableId, setTime).importFiles(files); List<String> result = new ArrayList<>(); for (Path p : stats.completeFailures.keySet()) { result.add(p.toString()); } return result; }
private Map<Path,List<KeyExtent>> assignMapFiles(VolumeManager fs, Map<Path,List<TabletLocation>> assignments, Collection<Path> paths, int numThreads, int numMapThreads) { timer.start(Timers.EXAMINE_MAP_FILES); Map<Path,List<AssignmentInfo>> assignInfo = estimateSizes(fs, assignments, paths, numMapThreads); timer.stop(Timers.EXAMINE_MAP_FILES); Map<Path,List<KeyExtent>> ret; timer.start(Timers.IMPORT_MAP_FILES); ret = assignMapFiles(assignInfo, locationsOf(assignments), numThreads); timer.stop(Timers.IMPORT_MAP_FILES); return ret; }
public static List<TabletLocation> findOverlappingTablets(ServerContext context, VolumeManager fs, TabletLocator locator, Path file, KeyExtent failed) throws Exception { locator.invalidateCache(failed); Text start = getStartRowForExtent(failed); return findOverlappingTablets(context, fs, locator, file, start, failed.getEndRow()); }
Map<Path,List<KeyExtent>> assignmentFailures = assignMapFiles(fs, assignments, paths, numAssignThreads, numThreads); assignmentStats.assignmentsFailed(assignmentFailures); try { tabletsToAssignMapFileTo .addAll(findOverlappingTablets(context, fs, locator, entry.getKey(), ke)); keListIter.remove(); } catch (Exception ex) { Map<Path,List<KeyExtent>> assignmentFailures2 = assignMapFiles(fs, assignments, paths, numAssignThreads, numThreads); assignmentStats.assignmentsFailed(assignmentFailures2); Set<Path> failedFailures = processFailures(completeFailures); assignmentStats.unrecoveredMapFiles(failedFailures); printReport(paths); return assignmentStats; } finally {
public static List<TabletLocation> findOverlappingTablets(ServerContext context, VolumeManager fs, TabletLocator locator, Path file) throws Exception { return findOverlappingTablets(context, fs, locator, file, null, null); }
@Override public List<String> bulkImportFiles(TInfo tinfo, final TCredentials credentials, final long tid, final String tableId, final List<String> files, final String errorDir, final boolean setTime) throws ThriftSecurityException, ThriftTableOperationException, TException { try { if (!security.canPerformSystemActions(credentials)) throw new AccumuloSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED); bulkImportStatus.updateBulkImportStatus(files, BulkImportState.INITIAL); log.debug("Got request to bulk import files to table({}): {}", tableId, files); bulkImportStatus.updateBulkImportStatus(files, BulkImportState.PROCESSING); try { return BulkImporter.bulkLoad(context, tid, tableId, files, setTime); } finally { bulkImportStatus.removeBulkImportStatus(files); } } catch (AccumuloSecurityException e) { throw e.asThriftException(); } catch (Exception ex) { throw new TException(ex); } }
Map<Path,List<KeyExtent>> assignmentFailures = assignMapFiles(acuConf, instance, conf, credentials, fs, tableId, assignments, paths, numAssignThreads, numThreads); assignmentStats.assignmentsFailed(assignmentFailures); tabletsToAssignMapFileTo.addAll(findOverlappingTablets(instance.getConfiguration(), fs, locator, entry.getKey(), ke, credentials)); keListIter.remove(); } catch (Exception ex) { Map<Path,List<KeyExtent>> assignmentFailures2 = assignMapFiles(acuConf, instance, conf, credentials, fs, tableId, assignments, paths, numAssignThreads, numThreads); assignmentStats.assignmentsFailed(assignmentFailures2); Set<Path> failedFailures = processFailures(conf, fs, failureDir, completeFailures); assignmentStats.unrecoveredMapFiles(failedFailures); printReport(); return assignmentStats; } finally {
@Override public void run() { List<TabletLocation> tabletsToAssignMapFileTo = Collections.emptyList(); try { tabletsToAssignMapFileTo = findOverlappingTablets(context, fs, locator, mapFile); } catch (Exception ex) { log.warn("Unable to find tablets that overlap file " + mapFile, ex); } log.debug("Map file {} found to overlap {} tablets", mapFile, tabletsToAssignMapFileTo.size()); if (tabletsToAssignMapFileTo.size() == 0) { List<KeyExtent> empty = Collections.emptyList(); completeFailures.put(mapFile, empty); } else assignments.put(mapFile, tabletsToAssignMapFileTo); } };
@Override public List<String> call() throws Exception { return BulkImporter.bulkLoad(new ServerConfiguration(instance).getConfiguration(), instance, credentials, tid, tableId, files, errorDir, setTime); } });
Map<Path,List<KeyExtent>> assignmentFailures = assignMapFiles(context, conf, fs, tableId, assignments, paths, numAssignThreads, numThreads); assignmentStats.assignmentsFailed(assignmentFailures); try { tabletsToAssignMapFileTo .addAll(findOverlappingTablets(context, fs, locator, entry.getKey(), ke)); keListIter.remove(); } catch (Exception ex) { Map<Path,List<KeyExtent>> assignmentFailures2 = assignMapFiles(context, conf, fs, tableId, assignments, paths, numAssignThreads, numThreads); assignmentStats.assignmentsFailed(assignmentFailures2); Set<Path> failedFailures = processFailures(completeFailures); assignmentStats.unrecoveredMapFiles(failedFailures); printReport(paths); return assignmentStats; } finally {
private Map<Path,List<KeyExtent>> assignMapFiles(AccumuloConfiguration acuConf, Instance instance, Configuration conf, TCredentials credentials, FileSystem fs, String tableId, Map<Path,List<TabletLocation>> assignments, Collection<Path> paths, int numThreads, int numMapThreads) { timer.start(Timers.EXAMINE_MAP_FILES); Map<Path,List<AssignmentInfo>> assignInfo = estimateSizes(acuConf, conf, fs, assignments, paths, numMapThreads); timer.stop(Timers.EXAMINE_MAP_FILES); Map<Path,List<KeyExtent>> ret; timer.start(Timers.IMPORT_MAP_FILES); ret = assignMapFiles(credentials, tableId, assignInfo, locationsOf(assignments), numThreads); timer.stop(Timers.IMPORT_MAP_FILES); return ret; }
public static List<String> bulkLoad(ClientContext context, long tid, String tableId, List<String> files, String errorDir, boolean setTime) throws IOException, AccumuloException, AccumuloSecurityException, ThriftTableOperationException { AssignmentStats stats = new BulkImporter(context, tid, tableId, setTime).importFiles(files, new Path(errorDir)); List<String> result = new ArrayList<>(); for (Path p : stats.completeFailures.keySet()) { result.add(p.toString()); } return result; }
public static List<TabletLocation> findOverlappingTablets(ClientContext context, VolumeManager fs, TabletLocator locator, Path file, KeyExtent failed) throws Exception { locator.invalidateCache(failed); Text start = getStartRowForExtent(failed); return findOverlappingTablets(context, fs, locator, file, start, failed.getEndRow()); }
public static List<TabletLocation> findOverlappingTablets(AccumuloConfiguration acuConf, FileSystem fs, TabletLocator locator, Path file, TCredentials credentials) throws Exception { return findOverlappingTablets(acuConf, fs, locator, file, null, null, credentials); }
@Override public List<String> call() throws Exception { bulkImportStatus.updateBulkImportStatus(files, BulkImportState.PROCESSING); try { return BulkImporter.bulkLoad(context, tid, tableId, files, errorDir, setTime); } finally { bulkImportStatus.removeBulkImportStatus(files); } } });
private Map<Path,List<KeyExtent>> assignMapFiles(ClientContext context, Configuration conf, VolumeManager fs, String tableId, Map<Path,List<TabletLocation>> assignments, Collection<Path> paths, int numThreads, int numMapThreads) { timer.start(Timers.EXAMINE_MAP_FILES); Map<Path,List<AssignmentInfo>> assignInfo = estimateSizes(context.getConfiguration(), conf, fs, assignments, paths, numMapThreads); timer.stop(Timers.EXAMINE_MAP_FILES); Map<Path,List<KeyExtent>> ret; timer.start(Timers.IMPORT_MAP_FILES); ret = assignMapFiles(tableId, assignInfo, locationsOf(assignments), numThreads); timer.stop(Timers.IMPORT_MAP_FILES); return ret; }
public static List<String> bulkLoad(AccumuloConfiguration conf, Instance instance, TCredentials creds, long tid, String tableId, List<String> files, String errorDir, boolean setTime) throws IOException, AccumuloException, AccumuloSecurityException, ThriftTableOperationException { AssignmentStats stats = new BulkImporter(conf, instance, creds, tid, tableId, setTime).importFiles(files, new Path(errorDir)); List<String> result = new ArrayList<String>(); for (Path p : stats.completeFailures.keySet()) { result.add(p.toString()); } return result; }
public static List<TabletLocation> findOverlappingTablets(AccumuloConfiguration acuConf, FileSystem fs, TabletLocator locator, Path file, KeyExtent failed, TCredentials credentials) throws Exception { locator.invalidateCache(failed); Text start = getStartRowForExtent(failed); return findOverlappingTablets(acuConf, fs, locator, file, start, failed.getEndRow(), credentials); }
public static List<TabletLocation> findOverlappingTablets(ClientContext context, VolumeManager fs, TabletLocator locator, Path file) throws Exception { return findOverlappingTablets(context, fs, locator, file, null, null); }
@Override public void run() { List<TabletLocation> tabletsToAssignMapFileTo = Collections.emptyList(); try { tabletsToAssignMapFileTo = findOverlappingTablets(context, fs, locator, mapFile); } catch (Exception ex) { log.warn("Unable to find tablets that overlap file " + mapFile.toString(), ex); } log.debug("Map file " + mapFile + " found to overlap " + tabletsToAssignMapFileTo.size() + " tablets"); if (tabletsToAssignMapFileTo.size() == 0) { List<KeyExtent> empty = Collections.emptyList(); completeFailures.put(mapFile, empty); } else assignments.put(mapFile, tabletsToAssignMapFileTo); } };