int scanActiveLogs(Path dir) throws IOException { RemoteIterator<FileStatus> iter = list(dir); int logsToScanCount = 0; while (iter.hasNext()) { FileStatus stat = iter.next(); String name = stat.getPath().getName(); ApplicationId appId = parseApplicationId(name); if (appId != null) { LOG.debug("scan logs for {} in {}", appId, stat.getPath()); logsToScanCount++; AppLogs logs = getAndSetActiveLog(appId, stat.getPath()); executor.execute(new ActiveLogParser(logs)); } else { logsToScanCount += scanActiveLogs(stat.getPath()); } } return logsToScanCount; }
private AppLogs getAndSetAppLogs(ApplicationId applicationId) throws IOException { LOG.debug("Looking for app logs mapped for app id {}", applicationId); AppLogs appLogs = appIdLogMap.get(applicationId); if (appLogs == null) { AppState appState = AppState.UNKNOWN; Path appDirPath = getDoneAppPath(applicationId); if (fs.exists(appDirPath)) { appState = AppState.COMPLETED; } else { appDirPath = getActiveAppPath(applicationId); if (fs.exists(appDirPath)) { appState = AppState.ACTIVE; } } if (appState != AppState.UNKNOWN) { appLogs = new AppLogs(applicationId, appDirPath, appState); LOG.debug("Create and try to add new appLogs to appIdLogMap for {}", applicationId); AppLogs oldAppLogs = appIdLogMap.putIfAbsent(applicationId, appLogs); if (oldAppLogs != null) { appLogs = oldAppLogs; } } } return appLogs; }
@Override protected void serviceInit(Configuration conf) throws Exception { metrics = EntityGroupFSTimelineStoreMetrics.create(); summaryStore = createSummaryStore(); addService(summaryStore); cacheIdPlugins = loadPlugIns(conf); yarnClient = createAndInitYarnClient(conf); addIfService(yarnClient); activeRootPath = new Path(conf.get( YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_ACTIVE_DIR,
long now = Time.now(); RemoteIterator<FileStatus> iter = list(dirpath); while (iter.hasNext()) { FileStatus stat = iter.next(); ApplicationId appId = parseApplicationId(dirpath.getName()); if (appId != null) { // Application log dir if (shouldCleanAppLogDir(dirpath, now, fs, retainMillis)) { try { LOG.info("Deleting {}", dirpath); cleanLogs(stat.getPath(), fs, retainMillis);
if (appLogs == null) { AppState appState = AppState.UNKNOWN; Path appDirPath = getDoneAppPath(applicationId); if (fs.exists(appDirPath)) { appState = AppState.COMPLETED; } else { appDirPath = getActiveAppPath(applicationId); if (fs.exists(appDirPath)) { appState = AppState.ACTIVE; } else { RemoteIterator<FileStatus> iter = list(activeRootPath); while (iter.hasNext()) { Path child = new Path(iter.next().getPath().getName(), LOG.debug("Create and try to add new appLogs to appIdLogMap for {}", applicationId); appLogs = createAndPutAppLogsIfAbsent( applicationId, appDirPath, appState);
@SuppressWarnings("serial") @Override protected void serviceInit(Configuration conf) throws Exception { summaryStore = createSummaryStore(); summaryStore.init(conf); long logRetainSecs = conf.getLong( .TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_APP_CACHE_SIZE_DEFAULT); LOG.info("Application cache size is {}", appCacheMaxSize); cacheIdPlugins = loadPlugIns(conf); yarnClient = createAndInitYarnClient(conf); CallerContext.setCurrent( new CallerContext.Builder(ATS_V15_SERVER_DFS_CALLER_CTXT).build());
LOG.info("Starting {}", getName()); summaryStore.start(); Configuration conf = getConfig(); aclManager = new TimelineACLsManager(conf); aclManager.setTimelineStore(summaryStore); summaryTdm = new TimelineDataManager(summaryStore, aclManager); summaryTdm.init(conf); addService(summaryTdm);
private TimelineStore getCachedStore(TimelineEntityGroupId groupId, List<EntityCacheItem> cacheItems) throws IOException { EntityCacheItem cacheItem; synchronized (this.cachedLogs) { // Note that the content in the cache log storage may be stale. cacheItem = this.cachedLogs.get(groupId); if (cacheItem == null) { LOG.debug("Set up new cache item for id {}", groupId); cacheItem = new EntityCacheItem(groupId, getConfig()); AppLogs appLogs = getAndSetAppLogs(groupId.getApplicationId()); if (appLogs != null) { LOG.debug("Set applogs {} for group id {}", appLogs, groupId); cacheItem.setAppLogs(appLogs); this.cachedLogs.put(groupId, cacheItem); } else { LOG.warn("AppLogs for groupId {} is set to null!", groupId); } } } TimelineStore store = null; if (cacheItem.getAppLogs() != null) { AppLogs appLogs = cacheItem.getAppLogs(); LOG.debug("try refresh cache {} {}", groupId, appLogs.getAppId()); cacheItems.add(cacheItem); store = cacheItem.refreshCache(aclManager, metrics); } else { LOG.warn("AppLogs for group id {} is null", groupId); } return store; }
ApplicationId appId = parseApplicationId(dirpath.getName()); boolean shouldClean = (appId != null); RemoteIterator<FileStatus> iter = fs.listStatusIterator(dirpath); cleanLogs(stat.getPath());
@Test public void testScanActiveLogsAndMoveToDonePluginRead() throws Exception { EntityGroupFSTimelineStore store = null; ApplicationId appId = ApplicationId.fromString("application_1501509265053_0002"); String user = UserGroupInformation.getCurrentUser().getShortUserName(); Path userBase = new Path(testActiveDirPath, user); Path userAppRoot = new Path(userBase, appId.toString()); Path attemotDirPath = new Path(userAppRoot, getAttemptDirName(appId)); try { store = createAndStartTimelineStore(AppState.COMPLETED); String logFileName = EntityGroupFSTimelineStore.ENTITY_LOG_PREFIX + EntityGroupPlugInForTest.getStandardTimelineGroupId(appId); createTestFiles(appId, attemotDirPath, logFileName); store.scanActiveLogs(); TimelineEntity entity = store.getEntity(entityNew.getEntityId(), entityNew.getEntityType(), EnumSet.allOf(Field.class)); assertNotNull(entity); assertEquals(entityNew.getEntityId(), entity.getEntityId()); assertEquals(entityNew.getEntityType(), entity.getEntityType()); } finally { if (store != null) { store.stop(); } fs.delete(userBase, true); } }
@InterfaceAudience.Private @VisibleForTesting void scanActiveLogs() throws IOException { RemoteIterator<FileStatus> iter = fs.listStatusIterator(activeRootPath); while (iter.hasNext()) { FileStatus stat = iter.next(); ApplicationId appId = parseApplicationId(stat.getPath().getName()); if (appId != null) { LOG.debug("scan logs for {} in {}", appId, stat.getPath()); AppLogs logs = getAndSetActiveLog(appId, stat.getPath()); executor.execute(new ActiveLogParser(logs)); } } }
store.getConfig().get( YarnConfiguration.TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_CLASSES)); List<TimelineEntityGroupPlugin> currPlugins = store.getPlugins(); for (TimelineEntityGroupPlugin plugin : currPlugins) { ClassLoader pluginClassLoader = plugin.getClass().getClassLoader(); config); cacheItem.setAppLogs(appLogs); store.setCachedLogs( EntityGroupPlugInForTest.getStandardTimelineGroupId(mainTestAppId), cacheItem);
@Override public void run() { LOG.debug("Active scan starting"); try { scanActiveLogs(); } catch (Exception e) { LOG.error("Error scanning active files", e); } LOG.debug("Active scan complete"); } }
@Override public void run() { LOG.debug("Cleaner starting"); long startTime = Time.monotonicNow(); try { cleanLogs(doneRootPath, fs, logRetainMillis); } catch (Exception e) { Throwable t = extract(e); if (t instanceof InterruptedException) { LOG.info("Cleaner interrupted"); } else { LOG.error("Error cleaning files", e); } } finally { metrics.addLogCleanTime(Time.monotonicNow() - startTime); } LOG.debug("Cleaner finished"); } }
/** * Get the application state. * @param appId application ID * @return the state or {@link AppState#UNKNOWN} if it could not * be determined * @throws IOException on IO problems */ @VisibleForTesting protected AppState getAppState(ApplicationId appId) throws IOException { return getAppState(appId, yarnClient); }
private List<TimelineStore> getTimelineStoresFromCacheIds( Set<TimelineEntityGroupId> groupIds, String entityType) throws IOException { List<TimelineStore> stores = new LinkedList<TimelineStore>(); // For now we just handle one store in a context. We return the first // non-null storage for the group ids. for (TimelineEntityGroupId groupId : groupIds) { TimelineStore storeForId = getCachedStore(groupId); if (storeForId != null) { LOG.debug("Adding {} as a store for the query", storeForId.getName()); stores.add(storeForId); } } if (stores.size() == 0) { LOG.debug("Using summary store for {}", entityType); stores.add(this.summaryStore); } return stores; }
private TimelineStore getCachedStore(TimelineEntityGroupId groupId) throws IOException { EntityCacheItem cacheItem; synchronized (this.cachedLogs) { // Note that the content in the cache log storage may be stale. cacheItem = this.cachedLogs.get(groupId); if (cacheItem == null) { LOG.debug("Set up new cache item for id {}", groupId); cacheItem = new EntityCacheItem(getConfig(), fs); AppLogs appLogs = getAndSetAppLogs(groupId.getApplicationId()); if (appLogs != null) { LOG.debug("Set applogs {} for group id {}", appLogs, groupId); cacheItem.setAppLogs(appLogs); this.cachedLogs.put(groupId, cacheItem); } else { LOG.warn("AppLogs for groupId {} is set to null!", groupId); } } } TimelineStore store = null; if (cacheItem.getAppLogs() != null) { AppLogs appLogs = cacheItem.getAppLogs(); LOG.debug("try refresh cache {} {}", groupId, appLogs.getAppId()); store = cacheItem.refreshCache(groupId, aclManager, jsonFactory, objMapper); } else { LOG.warn("AppLogs for group id {} is null", groupId); } return store; }
@Override public void run() { LOG.debug("Active scan starting"); try { int scanned = scanActiveLogs(); LOG.debug("Scanned {} active applications", scanned); } catch (Exception e) { Throwable t = extract(e); if (t instanceof InterruptedException) { LOG.info("File scanner interrupted"); } else { LOG.error("Error scanning active files", t); } } LOG.debug("Active scan complete"); } }