@GET @Path ("/stash-time") @RequiresPermissions ("system|comp_control") public Map<StashTimeKey, StashRunTimeInfo> getStashTimesForPlacement(@QueryParam ("placement") String placement) { return Strings.isNullOrEmpty(placement) ? _compactionControlSource.getAllStashTimes() : _compactionControlSource.getStashTimesForPlacement(placement); } }
@Override public void deleteStashTime(String id, String dataCenter) { try { for (CompactionControlSource compactionControlSource : _compactionControlSourceListProvider.get()) { compactionControlSource.deleteStashTime(id, dataCenter); } } catch (Exception e) { _log.error("Failed to delete stash timestamp info for id: {}", id, e); throw Throwables.propagate(e); } }
@Override public StashRunTimeInfo getStashTime(String id, String dataCenter) { try { return _localCompactionControl.getStashTime(id, dataCenter); } catch (Exception e) { _log.error("Failed to get stash timestamp info for id: {}", id, e); throw Throwables.propagate(e); } }
@Override public Map<StashTimeKey, StashRunTimeInfo> getStashTimesForPlacement(String placement) { try { return _localCompactionControl.getStashTimesForPlacement(placement); } catch (Exception e) { _log.error("Failed to get all stash timestamps info for placement: {}", placement, e); throw Throwables.propagate(e); } } }
@VisibleForTesting void deleteExpiredStashTimes(long currentTimeInMillis) { try { _log.debug("Checking for expired stash times at {}", currentTimeInMillis); Map<StashTimeKey, StashRunTimeInfo> expiredStashTimes = _compactionControlSource.getAllStashTimes().entrySet().stream() .filter(entry -> entry.getValue().getExpiredTimestamp() < currentTimeInMillis).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); for (Map.Entry<StashTimeKey, StashRunTimeInfo> expiredStashTimeInfo : expiredStashTimes.entrySet()) { // If we are deleting the entries here, then there could be a problem which we may want to know. So setting it as a warn. _log.warn("Deleting the stash time entry for id: {} and datacenter: {}", expiredStashTimeInfo.getKey().getId(), expiredStashTimeInfo.getKey().getDatacenter()); _compactionControlSource.deleteStashTime(expiredStashTimeInfo.getKey().getId(), expiredStashTimeInfo.getValue().getDataCenter()); } } catch (Exception e) { _log.error("Unexpected exception deleting the expired stash times", e); } } }
_compactionControlSource.updateStashTime(scanId, compactionControlTime, Lists.newArrayList(status.getOptions().getPlacements()), expireTime, _dataCenters.getSelf().getName()); } catch (Exception e) { _log.error("Failed to update the stash time for scan {}", scanId, e); _compactionControlSource.deleteStashTime(scanId, _dataCenters.getSelf().getName()); } catch (Exception ex) { _log.error("Failed to delete the stash time for scan {}", scanId, ex);
@Override public Map<StashTimeKey, StashRunTimeInfo> getAllStashTimes() { try { return _localCompactionControl.getAllStashTimes(); } catch (Exception e) { _log.error("Failed to get all stash timestamps info", e); throw Throwables.propagate(e); } }
@Override public void updateStashTime(String id, long timestamp, List<String> placements, long expiredTimestamp, String datacenter) { try { for (CompactionControlSource compactionControlSource : _compactionControlSourceListProvider.get()) { compactionControlSource.updateStashTime(id, timestamp, placements, expiredTimestamp, datacenter); } } catch (Exception e) { _log.error("Failed to update stash timestamp info for id: {}", id, e); throw Throwables.propagate(e); } }
private Expanded expand(final Record record, boolean ignoreRecent, final ReadConsistency consistency) { long fullConsistencyTimeStamp = _dataWriterDao.getFullConsistencyTimestamp(record.getKey().getTable()); long rawConsistencyTimeStamp = _dataWriterDao.getRawConsistencyTimestamp(record.getKey().getTable()); Map<StashTimeKey, StashRunTimeInfo> stashTimeInfoMap = _compactionControlSource.getStashTimesForPlacement(record.getKey().getTable().getAvailability().getPlacement()); // we will consider the earliest timestamp found as our compactionControlTimestamp. // we are also filtering out any expired timestamps. (CompactionControlMonitor should do this for us, but for now it's running every hour. So, just to fill that gap, we are filtering here.) // If no timestamps are found, then taking minimum value because we want all the deltas after the compactionControlTimestamp to be deleted as per the compaction rules as usual. long compactionControlTimestamp = stashTimeInfoMap.isEmpty() ? Long.MIN_VALUE : stashTimeInfoMap.values().stream().filter(s -> s.getExpiredTimestamp() > System.currentTimeMillis()).map(StashRunTimeInfo::getTimestamp).min(Long::compareTo).orElse(Long.MIN_VALUE); return expand(record, fullConsistencyTimeStamp, rawConsistencyTimeStamp, compactionControlTimestamp, ignoreRecent, consistency); }
private void updateMetrics(long currentTimeInMillis) { Map<StashTimeKey, StashRunTimeInfo> stashTimeInfoMap = _compactionControlSource.getAllStashTimes(); if (stashTimeInfoMap.size() > 0) { long oldestCompactionControlTime = stashTimeInfoMap.values().stream() .map(StashRunTimeInfo::getTimestamp) .min(Ordering.natural()) .orElse(currentTimeInMillis); _lag = currentTimeInMillis - oldestCompactionControlTime; } else { _lag = 0L; } }
@POST @Path ("/stash-time/{id}") @RequiresPermissions ("system|comp_control") public SuccessResponse updateStashTime(@PathParam ("id") String id, @QueryParam ("timestamp") LongParam timestampInMillisParam, @QueryParam ("placement") List<String> placements, @QueryParam ("expiredTimestamp") LongParam expiredTimestampInMillisParam, @QueryParam ("dataCenter") String dataCenter) { checkArgument(timestampInMillisParam != null, "timestamp is required"); checkArgument(!placements.isEmpty(), "Placement is required"); checkArgument(expiredTimestampInMillisParam != null, "expired timestamp is required."); _compactionControlSource.updateStashTime(id, timestampInMillisParam.get(), placements, expiredTimestampInMillisParam.get(), dataCenter); return SuccessResponse.instance(); }
private Expanded expand(final Record record, boolean ignoreRecent, final ReadConsistency consistency) { long fullConsistencyTimeStamp = _dataWriterDao.getFullConsistencyTimestamp(record.getKey().getTable()); long rawConsistencyTimeStamp = _dataWriterDao.getRawConsistencyTimestamp(record.getKey().getTable()); Map<StashTimeKey, StashRunTimeInfo> stashTimeInfoMap = _compactionControlSource.getStashTimesForPlacement(record.getKey().getTable().getAvailability().getPlacement()); // we will consider the earliest timestamp found as our compactionControlTimestamp. // we are also filtering out any expired timestamps. (CompactionControlMonitor should do this for us, but for now it's running every hour. So, just to fill that gap, we are filtering here.) // If no timestamps are found, then taking minimum value because we want all the deltas after the compactionControlTimestamp to be deleted as per the compaction rules as usual. long compactionControlTimestamp = stashTimeInfoMap.isEmpty() ? Long.MIN_VALUE : stashTimeInfoMap.values().stream().filter(s -> s.getExpiredTimestamp() > System.currentTimeMillis()).map(StashRunTimeInfo::getTimestamp).min(Long::compareTo).orElse(Long.MIN_VALUE); return expand(record, fullConsistencyTimeStamp, rawConsistencyTimeStamp, compactionControlTimestamp, ignoreRecent, consistency); }
@DELETE @Path ("/stash-time/{id}") @RequiresPermissions ("system|comp_control") public SuccessResponse deleteStashTime(@PathParam ("id") String id, @QueryParam ("dataCenter") String dataCenter) { checkArgument(!Strings.isNullOrEmpty(id), "id is required"); checkArgument(!Strings.isNullOrEmpty(dataCenter), "datacenter is required"); _compactionControlSource.deleteStashTime(id, dataCenter); return SuccessResponse.instance(); }
@GET @Path ("/stash-time/{id}") @RequiresPermissions ("system|comp_control") public Response getStashTime(@PathParam ("id") String id, @QueryParam ("dataCenter") String dataCenter) { checkArgument(!Strings.isNullOrEmpty(id), "id is required"); checkArgument(!Strings.isNullOrEmpty(dataCenter), "datacenter is required"); StashRunTimeInfo stashTimeInfo = _compactionControlSource.getStashTime(id, dataCenter); if (stashTimeInfo == null) { return Response.status(Response.Status.NOT_FOUND).build(); } return Response.ok(stashTimeInfo).build(); }
Map<StashTimeKey, StashRunTimeInfo> stashTimeInfoMap = _compactionControlSource.getStashTimesForPlacement(placement); Instant cutoffTime = stashTimeInfoMap.values().stream() .map(StashRunTimeInfo::getTimestamp)
private void cleanupScan(String id) { // Remove this scan from the active set if (_activeScans.remove(id)) { notifyActiveScanCountChanged(); } try { // Remove the table snapshots set for this scan _dataTools.clearStashTokenRangeSnapshot(id); } catch (Exception e) { _log.error("Failed to clean up table set for scan {}", id, e); } try { // Delete the entry of the scan start time in Zookeeper. _compactionControlSource.deleteStashTime(id, _dataCenters.getSelf().getName()); } catch (Exception e) { _log.error("Failed to delete the stash time for scan {}", id, e); } }
public void cancel(String id) { _scanStatusDAO.setCanceled(id); // Notify the workflow the scan status was updated _scanWorkflow.scanStatusUpdated(id); try { // Delete the entry of the scan start time in Zookeeper. _compactionControlSource.deleteStashTime(id, _dataCenters.getSelf().getName()); } catch (Exception e) { _log.error("Failed to delete the stash time for scan {}", id, e); } } }