@Override public NativeDocument process(NativeDocument document) throws Exception { document.addTagByKey(RepositoryStoreConstants.TAG_NAME_OBJECT_CLASS, BWMetaImporterConstants.SOURCE_FORMAT); return document; }
@Override public String apply(NativeDocument input) { return input.getId(); }
public boolean validate(NativeDocument document) { if (document == null) { return false; } String objectClass = document.getTagValue(RepositoryStoreConstants.TAG_NAME_OBJECT_CLASS); if (!BWMetaImporterConstants.SOURCE_FORMAT.equals(objectClass)) { return false; } Attachment<?> sourceAttachment = document.getAttachments().get(CommonExtractorContstants.SOURCE_ATTRIBUTE); return sourceAttachment != null; }
@Override public Iterable<DataResponse> feedFromCurrentElement() { NativeDocument document = state.getCurrentDocument(); if(document == null){ return Collections.emptyList(); } String objectClass = document.getTagValue(RepositoryStoreConstants.TAG_NAME_OBJECT_CLASS); if(!BWMetaImporterConstants.SOURCE_FORMAT.equals(objectClass)){ return Collections.emptyList(); } String levelToImport = getLevelToImport(document); Attachment<?> sourceAttachment = document.getAttachments().get(CommonExtractorContstants.SOURCE_ATTRIBUTE); if (sourceAttachment == null) { warn("No source attachment.", document.getId()); return Collections.emptyList(); } InputStream inputStream = getStream(sourceAttachment); List<YElement> elements = builder.build(new InputStreamResource(inputStream, "Record id: " + document.getId())); Stream<YElement> elementsStream = elements.stream(); if (StringUtils.isNotBlank(levelToImport)) { elementsStream = elementsStream.filter(e -> StringUtils.equals(YModelUtils.getType(e), levelToImport)); } return elementsStream.map(e -> toDataResponse(e, document)).collect(Collectors.toList()); }
private NativeDocument checkDataset(ReportDuplicateRequest request) { NativeDocument document = documentRepository.fetchDocument(request.getDuplicateDocumentId()); String dataset = document.getTagValue(RepositoryStoreConstants.TAG_NAME_DATASET); if (ApplicationConstants.DEFAULT_DATASET.equals(StringUtils.lowerCase(dataset))) { throw new InvalidEntryException(document.getId(), "Cannot request duplicate for resource added by user"); } return document; }
private String getPartFilename(NativeDocument document, Attachment<?> attachment) throws Exception { String name = attachment.getTagValue(RepositoryStoreConstants.TAG_NAME_CONTENT_NAME); if (name != null) { return name; } String xpath = attachment.getTagValue(ImporterConstants.PROPERTY_ATTACHMENT_PATH_IN_METADATA); if (xpath == null) { return null; } xpath = xpath.replace(LOCATION_XPATH_NAME, FILE_NAME_XPATH_NAME); String metadataLocation = attachment.getTagValue(ImporterConstants.PROPERTY_PARENT_ATTACHMENT); if (metadataLocation == null) { return null; } Attachment<?> metadataAttachment = document.getAttachments().get(metadataLocation); JXPathContext context = getBwMeta(metadataAttachment); try { String attachmentFileName = getFileName(context, xpath); return attachmentFileName; } catch (JXPathNotFoundException e) { LOGGER.warn("Couldn't retrieve attachment name from document: " + document.getId() + " and path: " + xpath); return null; } }
YElement element = item.getyElement(); NativeDocument document = documentFactory.createDocument(element.getId()); document.addTagByKey(TAG_NAME_IMPORT, identifierHolder.getProcessId()); document.addTagByKey(TAG_NAME_CREATION_TIMESTAMP, String.valueOf(System.currentTimeMillis())); document.addTagByKey(TAG_NAME_CONVERTED_FROM, element.getId()); if (definitionId != null) { document.addTagByKey(TAG_NAME_IMPORT_DATA_SOURCE_DEFINITIONS, definitionId); continue; BinaryAttachment binaryAttachment = document.addBinaryAttachment(partName, resource); if (file != null) { final String mimeType; binaryAttachment.addTagByKey(PROPERTY_PARENT_ATTACHMENT, sourcePartPath); YExportableAttachment attachment = document.addYExportableAttachment(sourcePartPath, element); attachment.addTagByKey(TAG_NAME_MIME, MiscellaneousConstants.BWMETA_MIME_TYPE); attachment.addTagByKey(TAG_NAME_TYPE, CommonExtractorContstants.SOURCE_ATTRIBUTE);
String sourcePartPath = input.buildStorePathPart(); NativeDocument document = documentFactory.createDocument(elementId); document.addTagByKey(TAG_NAME_IMPORT, identifierHolder.getProcessId()); document.addTagByKey(TAG_NAME_CREATION_TIMESTAMP, String.valueOf(System.currentTimeMillis())); document.addTagByKey(TAG_NAME_CONVERTED_FROM, elementId); if (definitionId != null) { document.addTagByKey(TAG_NAME_IMPORT_DATA_SOURCE_DEFINITIONS, definitionId); BinaryAttachment binaryAttachment = document.addBinaryAttachment(sourcePartPath, data); binaryAttachment.addTagByKey(TAG_NAME_MIME, MiscellaneousConstants.BWMETA_MIME_TYPE); binaryAttachment.addTagByKey(TAG_NAME_TYPE, "source");
private String getLevelToImport(NativeDocument document) { String hierarchyLevel = document.getTagValue(RepositoryStoreConstants.TAG_NAME_HIERARHY_LEVEL); if (StringUtils.isBlank(hierarchyLevel)) { return null; } List<String> hierarchyWithLevel = Splitter.on("=").splitToList(hierarchyLevel); if (hierarchyWithLevel.size() != 2) { return null; } return hierarchyWithLevel.get(1); }
@Override public Attachment<?> createAttachment(NativeDocument document, String parentPath, String path, int totalAttachmentsCount, Boolean downloadRemoteAttachments) throws InvalidAttachmentException { Assert.notNull(path, "Path is null"); Assert.notNull(parentPath, "Parent path is null"); File workingDirectory = new File(workingDirectoryPath); Assert.isTrue(workingDirectory.isDirectory(), "WorkingDirectory is not a directory"); Assert.notNull(workingDirectory, "WorkingDirectory is null"); AttachmentFetcher fetcher = repository.getSupportedFetcher(path); final AttachmentInformation information; if (fetcher != null) { information = fetcher.getAttachmentInformation(document.getId(), path, totalAttachmentsCount); } else { information = new AttachmentInformation(null, path); } if (information.getAttachment() != null) { document.addExistingAttachment(information.getAttachment(), false); return information.getAttachment(); } else if(information.getLocation() != null){ File attachmentFile = new File(workingDirectory + File.separator + information.getLocation()); if (attachmentFile.exists()) { return addAttachment(document, attachmentFile, parentPath); } else { throw new InvalidAttachmentException(path, ""); } } return null; }
private Attachment<?> getAttachmentPart(NativeDocument document, String fileName, int totalAttachmentsCount) throws Exception { List<Attachment<?>> contentAttachments = new ArrayList<>(); for (Entry<String, ? extends Attachment<?>> attachment : document.getAttachments().entrySet()) { if (isContentAttachment(attachment.getValue())) { contentAttachments.add(attachment.getValue()); String partFilename = getPartFilename(document, attachment.getValue()); if (StringUtils.equals(fileName, partFilename)) { return attachment.getValue(); } } } if (contentAttachments.isEmpty() || totalAttachmentsCount != 1) { return null; } return contentAttachments.get(0); }
private BinaryAttachment addAttachment(final NativeDocument inputDocument, File attachmentFile, String path) throws InvalidAttachmentException { try { String mimeType = MimeTypeHelper.resolveTypeForFile(attachmentFile); byte[] bytes = FileUtils.readFileToByteArray(attachmentFile); String attachmentPath = RepositoryStoreConstants.PATH_PREFIX_FULLTEXT_CONTENT + RepositoryStoreConstants.PATH_SEPARATOR + attachmentFile.getName(); String newPath = new IdGenerator().generateIdSuffix(attachmentPath, path); BinaryAttachment attachment = inputDocument.addBinaryAttachment(newPath, bytes); attachment.addTagByKey(TAG_NAME_MIME, mimeType); attachment.addTagByKey(TAG_NAME_CONTENT_NAME, attachmentFile.getName()); return attachment; } catch (IOException e) { throw new InvalidAttachmentException(attachmentFile.getAbsolutePath(), e.getMessage()); } } }
@Override public Document process(Document item) throws Exception { String ancestor = item.getTagValue(RepositoryStoreConstants.TAG_NAME_ANCESTOR); if (StringUtils.isBlank(ancestor)) { return item; } NativeDocument parent = cachedDataStore.fetchDocument(ancestor, RecordProjection.TAGS_ONLY); if(parent == null){ return item; } String structureId = parent.getTagValue(RepositoryStoreConstants.TAG_NAME_STRUCTURE); if (StringUtils.isBlank(structureId)) { return item; } if(identifierHolder.getProcessId().equals(structureId)){ item.addTagByKey(RepositoryStoreConstants.TAG_NAME_STRUCTURE, structureId); } return item; } }
@Override public void doWithFile(YContentFile file) { String type = file.getType(); if (FileTypes.FT_PLAIN_TEXT.equals(type)) { String location = file.getLocations().get(0); // System.out.println("adding..."+location); if (location != null) { Attachment<?> part = document.getAttachments().get(location); String text = ""; if (part instanceof BinaryAttachment) { byte[] content = ((BinaryAttachment) part).getContent(); text = new String(content); } textBuilder.append(text); } } }
private void processSingleAttachment(final NativeDocument inputDocument, final Attachment<?> attachment, List<String> invalidAttachments) { try { final String attachmentData = getAttachmentData(attachment); Document document = parseDocumentThreadSafetly(attachmentData); if (document != null) { Element root = document.getDocumentElement(); String namespace = root.getNamespaceURI(); if (StringUtils.isNotBlank(namespace)) { if (attachmentQueryMap.containsKey(namespace)) { String contentLocationXPaths = attachmentQueryMap.get(namespace); for (String contentLocationXPath : StringUtils.split(contentLocationXPaths)) { createAttachmentByLocation(inputDocument, attachment.getId(), document, namespace, contentLocationXPath, invalidAttachments); } } else { problemHandler.handleProblem(LogSeverity.WARN, inputDocument.getId(), ImporterConstants.PROBLEM_ERROR_READ_ATTACHMENTS, "Unknown namespace " + namespace + " . attachments will not be processed."); } } else { problemHandler.handleProblem(LogSeverity.WARN, inputDocument.getId(), ImporterConstants.PROBLEM_ERROR_READ_ATTACHMENTS, "Namespace was not defined in root element"); } } } catch (Exception e) { problemHandler.handleProblem(LogSeverity.WARN, inputDocument.getId(), ImporterConstants.PROBLEM_ERROR_READ_ATTACHMENTS, e); } }
@Override public NativeDocument process(NativeDocument item) throws Exception { if (item != null) { item.addTagByKey(tagKey, tagValue); } return item; }
boolean inSet = false; for (String set : sets) { if (nativeDocument.getTagValue(RepositoryStoreConstants.TAG_NAME_DATASET).equals(set)) { inSet = true; break;
@Override public NativeDocument process(final NativeDocument input) { if (MapUtils.isNotEmpty(attachmentQueryMap)) { for (Attachment<?> attachment : input.getAttachments().values()) { if (isPathAccepted(attachment.getId())) { List<String> invalidPaths = new ArrayList<String>(); processSingleAttachment(input, attachment, invalidPaths); if (!invalidPaths.isEmpty()) { attachment.addTagByKey(PROPERTY_INVALID_PATHS, StringUtils.join(invalidPaths, ";")); } } } } return input; }
private Resource toResource(NativeDocument document, Attachment<?> attachment) { InputStream stream = getStream(attachment); if (stream == null) { return null; } return new InputStreamResource(stream, "Record id: " + document.getId() + ", attachment id: " + attachment.getId()); }
@Override public NativeDocument process(NativeDocument nativeDocument) throws Exception { nativeDocument.addTagByKey(RepositoryStoreConstants.TAG_NAME_CREATION_TIMESTAMP, ""+System.currentTimeMillis()); if (processIdHolder != null) { nativeDocument.addTagByKey(RepositoryStoreConstants.TAG_NAME_IMPORT, processIdHolder.getProcessId()); } if (StringUtils.isNotBlank(definitionId)) { nativeDocument.addTagByKey(RepositoryStoreConstants.TAG_NAME_IMPORT_DATA_SOURCE_DEFINITIONS, definitionId); } return nativeDocument; } }