@Autowired public DataCleanerJobEngine(ClusterManagerFactory clusterManagerFactory, DescriptorProvider descriptorProvider, ApplicationContext applicationContext) { super(FileFilters.ANALYSIS_XML.getExtension()); _clusterManagerFactory = clusterManagerFactory; _descriptorProvider = descriptorProvider; _applicationContext = applicationContext; _runningJobs = new ConcurrentHashMap<String, AnalysisResultFuture>(); }
@Override public String getName() { final int extensionLength = FileFilters.ANALYSIS_XML.getExtension().length(); final String filename = _file.getName(); return filename.substring(0, filename.length() - extensionLength); }
@Override public Type getType() { if (getName().endsWith(FileFilters.ANALYSIS_XML.getExtension())) { return Type.ANALYSIS_JOB; } else if (getName().endsWith(FileFilters.ANALYSIS_RESULT_SER.getExtension())) { return Type.ANALYSIS_RESULT; } else if (getName().endsWith(FileFilters.ANALYSIS_TIMELINE_XML.getExtension())) { return Type.TIMELINE_SPEC; } return Type.OTHER; }
if (name.endsWith(FileFilters.ANALYSIS_XML.getExtension())) { return ICON_FILE_ANALYSIS_JOB; if (name.endsWith(FileFilters.ANALYSIS_RESULT_SER.getExtension())) { return ICON_FILE_ANALYSIS_RESULT; if (name.endsWith(FileFilters.ODB.getExtension())) { return ICON_FILE_ODB; if (name.endsWith(FileFilters.XML.getExtension())) { return ICON_FILE_XML; if (name.endsWith(FileFilters.DBF.getExtension())) { return ICON_FILE_DBASE; if (name.endsWith(FileFilters.CSV.getExtension()) || name.endsWith(FileFilters.TSV.getExtension()) || name .endsWith(FileFilters.DAT.getExtension()) || name.endsWith(FileFilters.TXT.getExtension())) { return ICON_FILE_CSV; if (name.endsWith(FileFilters.MDB.getExtension()) || name.endsWith(FileFilters.ACCDB.getExtension())) { return ICON_FILE_ACCESS; if (name.endsWith(FileFilters.XLS.getExtension()) || name.endsWith(FileFilters.XLSX.getExtension())) { return ICON_FILE_EXCEL;
private void addTimelines(List<TimelineIdentifier> result, DashboardGroup group, RepositoryFolder repositoryFolder) { final String extension = FileFilters.ANALYSIS_TIMELINE_XML.getExtension(); final List<RepositoryFile> files = repositoryFolder.getFiles(null, extension); for (RepositoryFile file : files) { final String timelineName = file.getName().substring(0, file.getName().length() - extension.length()); final TimelineIdentifier timelineIdentifier = new TimelineIdentifier(timelineName, file.getQualifiedPath(), group); result.add(timelineIdentifier); } }
public OpenAnalysisJobMenuItem(final FileObject file, final OpenAnalysisJobActionListener openAnalysisJobActionListener) { super(); _file = file; _openAnalysisJobActionListener = openAnalysisJobActionListener; final String title; final String filename = file.getName().getBaseName(); final String jobExtension = FileFilters.ANALYSIS_XML.getExtension(); final String resultExtension = FileFilters.ANALYSIS_RESULT_SER.getExtension(); if (filename.toLowerCase().endsWith(jobExtension)) { title = filename.substring(0, filename.length() - jobExtension.length()); setIcon(ICON_JOB); } else if (filename.toLowerCase().endsWith(resultExtension)) { title = filename.substring(0, filename.length() - resultExtension.length()); setIcon(ICON_RESULT); } else { title = filename; setIcon(ICON_FILE); } setText(title); addActionListener(this); }
@Override public void propertyChange(final PropertyChangeEvent evt) { if (JFileChooser.SELECTED_FILE_CHANGED_PROPERTY.equals(evt.getPropertyName())) { _file = _fileChooser.getSelectedFileObject(); if (_file != null && _file.getName().getBaseName().endsWith(FileFilters.ANALYSIS_XML.getExtension())) { showFileInformation(); } else { setVisible(false); } } }
@RolesAllowed(SecurityRoles.VIEWER) @RequestMapping(method = RequestMethod.GET, produces = "application/json") @ResponseBody public List<Map<String, String>> resultsFolderJson(@PathVariable("tenant") String tenant) { final TenantContext context = _tenantContextFactory.getContext(tenant); final RepositoryFolder resultsFolder = context.getResultFolder(); final List<Map<String, String>> result = new ArrayList<Map<String, String>>(); { final List<RepositoryFile> files = resultsFolder.getFiles(null, FileFilters.ANALYSIS_RESULT_SER.getExtension()); for (RepositoryFile file : files) { Map<String, String> map = new HashMap<String, String>(); map.put("filename", file.getName()); map.put("repository_path", file.getQualifiedPath()); result.add(map); } } return result; } }
protected static List<RepositoryFile> getResultsForJob(String jobName, RepositoryFolder resultsFolder) { final String extension = FileFilters.ANALYSIS_RESULT_SER.getExtension(); final String prefix = jobName + "-"; final List<RepositoryFile> candidatesByFilename = resultsFolder.getFiles(prefix, extension); final List<RepositoryFile> files = CollectionUtils.filter(candidatesByFilename, new Predicate<RepositoryFile>() { @Override public Boolean eval(RepositoryFile file) { // check that the remainding part of the file is ONLY a // timestamp - or else it might be a name conflict // between similarly named jobs. String timestampPart = file.getName(); timestampPart = timestampPart.substring(prefix.length()); timestampPart = timestampPart.substring(0, timestampPart.length() - extension.length()); try { Long.parseLong(timestampPart); return true; } catch (NumberFormatException e) { return false; } } }); return files; }
public ExecutionLoggerImpl(ExecutionLog execution, RepositoryFolder resultFolder, ApplicationEventPublisher eventPublisher) { _execution = execution; _resultFolder = resultFolder; _eventPublisher = eventPublisher; _erronuous = new AtomicBoolean(false); _executionLogWriter = new JaxbExecutionLogWriter(); _log = new StringBuilder(); final String resultId = execution.getResultId(); final String logFilename = resultId + FileFilters.ANALYSIS_EXECUTION_LOG_XML.getExtension(); final RepositoryFile existingLogFile = resultFolder.getFile(logFilename); if (existingLogFile == null) { _logFile = resultFolder.createFile(logFilename, new Action<OutputStream>() { @Override public void run(OutputStream out) throws Exception { _executionLogWriter.write(_execution, out); } }); } else { _logFile = existingLogFile; } }
@Override public List<TimelineIdentifier> getTimelines(final TenantIdentifier tenant, final DashboardGroup group) { final RepositoryFolder timelinesFolder = _tenantContextFactory.getContext(tenant).getTimelineFolder(); final List<RepositoryFile> files; final String groupName = (group == null ? null : group.getName()); if (group == null || groupName == null || "".equals(groupName)) { files = timelinesFolder.getFiles(); } else { RepositoryFolder groupFolder = timelinesFolder.getFolder(groupName); files = groupFolder.getFiles(); } final List<TimelineIdentifier> result = new ArrayList<TimelineIdentifier>(); for (RepositoryFile file : files) { if (file.getType() == Type.TIMELINE_SPEC) { String timelineName = file.getName().substring(0, file.getName().length() - FileFilters.ANALYSIS_TIMELINE_XML.getExtension().length()); TimelineIdentifier timeline = new TimelineIdentifier(timelineName, file.getQualifiedPath(), group); result.add(timeline); } } return result; }
public void openFile(final FileObject file) { if (file.getName().getBaseName().endsWith(FileFilters.ANALYSIS_RESULT_SER.getExtension())) { openAnalysisResult(file, _parentModule); } else { final Injector injector = openAnalysisJob(file); if (injector == null) { // this may happen, in which case the error was signalled to the // user already return; } final AnalysisJobBuilderWindow window = injector.getInstance(AnalysisJobBuilderWindow.class); window.open(); if (_parentWindow != null && !_parentWindow.isDatastoreSet()) { _parentWindow.close(); } } }
@Override public ExecutionLog getLatestExecution(TenantIdentifier tenant, JobIdentifier job) { final TenantContext tenantContext = _tenantContextFactory.getContext(tenant); final RepositoryFolder resultFolder = tenantContext.getResultFolder(); final RepositoryFile latestFile = resultFolder.getLatestFile(job.getName(), FileFilters.ANALYSIS_EXECUTION_LOG_XML.getExtension()); if (latestFile == null) { // no execution available return null; } return readExecutionLogFile(latestFile, job, tenant, 1); }
@Override public ExecutionLog getExecution(TenantIdentifier tenant, ExecutionIdentifier executionIdentifier) throws DCSecurityException { if (executionIdentifier == null) { return null; } final String resultId = executionIdentifier.getResultId(); if (resultId == null) { return null; } final TenantContext tenantContext = _tenantContextFactory.getContext(tenant); final RepositoryFolder resultFolder = tenantContext.getResultFolder(); final RepositoryFile file = resultFolder.getFile(resultId + FileFilters.ANALYSIS_EXECUTION_LOG_XML .getExtension()); if (file == null) { logger.warn("No execution with result id: " + resultId); return null; } JobIdentifier jobIdentifier = JobIdentifier.fromExecutionIdentifier(executionIdentifier); return readExecutionLogFile(file, jobIdentifier, tenant, 3); }
private void getResultFileFromCluster(TenantContext tenantContext, ExecutionLogger executionLogger, String hadoopResultFileName, String jobName) { HdfsResource resultsResource = null; try { resultsResource = new HdfsResource(HadoopUtils.getFileSystem().getUri().resolve(hadoopResultFileName) .toString()); if (resultsResource != null && resultsResource.isExists()) { final RepositoryFolder repositoryResultFolder = tenantContext.getResultFolder(); final String fileName = HadoopJobExecutionUtils.getUrlReadyJobName(jobName) + FileFilters.ANALYSIS_RESULT_SER.getExtension(); final Resource resourceFile = repositoryResultFolder.createFile(fileName, null).toResource(); logger.info("Writing the result to" + resourceFile.getQualifiedPath()); FileHelper.copy(resultsResource, resourceFile); } else { final String message = "An error has occured while running the job. The result was not persisted on Hadoop. Please check Hadoop and/or DataCleaner logs"; final Exception error = new Exception(message); executionLogger.setStatusFailed(null, null, error); } } catch (Exception e) { executionLogger.setStatusFailed(null, null, e); } } }
@Override public List<ExecutionIdentifier> getAllExecutions(TenantIdentifier tenant, JobIdentifier job) throws IllegalStateException { final TenantContext tenantContext = _tenantContextFactory.getContext(tenant); final RepositoryFolder resultFolder = tenantContext.getResultFolder(); final List<RepositoryFile> files = resultFolder.getFiles(job.getName(), FileFilters.ANALYSIS_EXECUTION_LOG_XML .getExtension()); final List<ExecutionIdentifier> executionIdentifiers = CollectionUtils.map(files, new Func<RepositoryFile, ExecutionIdentifier>() { @Override public ExecutionIdentifier eval(final RepositoryFile file) { try { final ExecutionIdentifier result = file.readFile( new Func<InputStream, ExecutionIdentifier>() { @Override public ExecutionIdentifier eval(InputStream in) { return SaxExecutionIdentifierReader.read(in, file.getQualifiedPath()); } }); return result; } catch (Exception e) { logger.warn("The file " + file.getQualifiedPath() + " could not be read or parsed correctly " + e); return new ExecutionIdentifier("Execution failed for " + FilenameUtils.getBaseName(file .getQualifiedPath())); } } }); Collections.sort(executionIdentifiers); return executionIdentifiers; }
@Override public ResultContext getLatestResult(TenantIdentifier tenantIdentifier, JobIdentifier job) { final TenantContext context = _tenantContextFactory.getContext(tenantIdentifier.getId()); final RepositoryFolder resultsFolder = context.getResultFolder(); final String jobName = job.getName(); final RepositoryFile resultFile = resultsFolder.getLatestFile(jobName, FileFilters.ANALYSIS_RESULT_SER.getExtension()); if (resultFile == null) { return null; } return context.getResult(resultFile.getName()); }
private void serializeResult(final Serializable result) { final String resultFilename = _execution.getResultId() + FileFilters.ANALYSIS_RESULT_SER.getExtension();
final String resultExtension = FileFilters.ANALYSIS_RESULT_SER.getExtension(); final String executionLogExtension = FileFilters.ANALYSIS_EXECUTION_LOG_XML.getExtension();
@Override public TimelineIdentifier createTimelineDefinition(final TenantIdentifier tenant, final TimelineIdentifier timelineIdentifier, final TimelineDefinition timelineDefinition) { final String name = timelineIdentifier.getName(); final DashboardGroup group = timelineIdentifier.getGroup(); final RepositoryFolder timelinesFolder = _tenantContextFactory.getContext(tenant).getTimelineFolder(); final RepositoryFolder folder; if (group == null) { folder = timelinesFolder; } else { folder = timelinesFolder.getFolder(group.getName()); } final String fileName = name + FileFilters.ANALYSIS_TIMELINE_XML.getExtension(); final RepositoryFile file = folder.createFile(fileName, new WriteTimelineAction(timelineDefinition)); logger.info("Created timeline definition in file: {}", file); return new TimelineIdentifier(timelineIdentifier.getName(), file.getQualifiedPath(), group); }