public AIP createAIP(AIPState state, String parentId, String type, Permissions permissions, String ingestSIPUUID, List<String> ingestSIPIds, String ingestJobId, boolean notify, String createdBy) throws RequestNotValidException, NotFoundException, GenericException, AlreadyExistsException, AuthorizationDeniedException { RodaCoreFactory.checkIfWriteIsAllowedAndIfFalseThrowException(nodeType); Directory directory = storage.createRandomDirectory(DefaultStoragePath.parse(RodaConstants.STORAGE_CONTAINER_AIP)); String id = directory.getStoragePath().getName(); Permissions inheritedPermissions = this.addParentPermissions(permissions, parentId); AIP aip = new AIP(id, parentId, type, state, inheritedPermissions, createdBy).setIngestSIPIds(ingestSIPIds) .setIngestJobId(ingestJobId).setIngestSIPUUID(ingestSIPUUID); createAIPMetadata(aip); if (notify) { notifyAipCreated(aip).failOnError(); } return aip; }
private DescriptiveMetadata updateDescriptiveMetadata(AIP aip, String representationId, String descriptiveMetadataId, String descriptiveMetadataType, String descriptiveMetadataVersion) { DescriptiveMetadata descriptiveMetadata; Optional<DescriptiveMetadata> odm = getDescriptiveMetadata(aip, representationId).stream() .filter(dm -> dm.getId().equals(descriptiveMetadataId)).findFirst(); if (odm.isPresent()) { descriptiveMetadata = odm.get(); descriptiveMetadata.setType(descriptiveMetadataType); descriptiveMetadata.setVersion(descriptiveMetadataVersion); } else { descriptiveMetadata = new DescriptiveMetadata(descriptiveMetadataId, aip.getId(), representationId, descriptiveMetadataType, descriptiveMetadataVersion); aip.addDescriptiveMetadata(descriptiveMetadata); } return descriptiveMetadata; }
public static AIP earkSIPToAIPUpdate(SIP sip, IndexedAIP indexedAIP, ModelService model, String username, Optional<String> searchScope, String ingestJobId, Report reportItem, Plugin<?> plugin) throws RequestNotValidException, NotFoundException, GenericException, AlreadyExistsException, AuthorizationDeniedException, ValidationException, LockingException { boolean notify = false; AIP aip; PluginHelper.acquireObjectLock(indexedAIP, plugin); // process IP information processIPInformation(model, sip, indexedAIP.getId(), notify, true); // process IPRepresentation information for (IPRepresentation representation : sip.getRepresentations()) { processIPRepresentationInformation(model, representation, indexedAIP.getId(), notify, true, username, reportItem); } aip = model.retrieveAIP(indexedAIP.getId()); aip.setGhost(false); if (searchScope.isPresent()) { aip.setParentId(searchScope.get()); } aip.addIngestUpdateJobId(ingestJobId); for (String id : sip.getIds()) { if (!aip.getIngestSIPIds().contains(id)) { aip.getIngestSIPIds().add(id); } } return model.updateAIP(aip, username); }
@Override public Map<String, Object> getPreCalculatedFields() { Map<String, Object> preCalculatedFields = new HashMap<>(); // indexing AIP inherited info preCalculatedFields.put(RodaConstants.INDEX_STATE, SolrUtils.formatEnum(aip.getState())); preCalculatedFields.put(RodaConstants.INGEST_SIP_IDS, aip.getIngestSIPIds()); preCalculatedFields.put(RodaConstants.INGEST_JOB_ID, aip.getIngestJobId()); preCalculatedFields.put(RodaConstants.INGEST_UPDATE_JOB_IDS, aip.getIngestUpdateJobIds()); preCalculatedFields.put(RodaConstants.FILE_ANCESTORS, ancestors); preCalculatedFields.putAll(SolrUtils.getPermissionsAsPreCalculatedFields(aip.getPermissions())); return preCalculatedFields; }
private List<DescriptiveMetadata> getDescriptiveMetadata(AIP aip, String representationId) { List<DescriptiveMetadata> descriptiveMetadataList = Collections.emptyList(); if (representationId == null) { // AIP descriptive metadata descriptiveMetadataList = aip.getDescriptiveMetadata(); } else { // Representation descriptive metadata Optional<Representation> oRep = aip.getRepresentations().stream() .filter(rep -> rep.getId().equals(representationId)).findFirst(); if (oRep.isPresent()) { descriptiveMetadataList = oRep.get().getDescriptiveMetadata(); } } return descriptiveMetadataList; }
doc.addField(RodaConstants.AIP_PARENT_ID, aip.getParentId()); doc.addField(RodaConstants.AIP_TYPE, aip.getType()); doc.addField(RodaConstants.AIP_CREATED_ON, SolrUtils.formatDate(aip.getCreatedOn())); doc.addField(RodaConstants.AIP_CREATED_BY, aip.getCreatedBy()); doc.addField(RodaConstants.AIP_UPDATED_ON, SolrUtils.formatDate(aip.getUpdatedOn())); doc.addField(RodaConstants.AIP_UPDATED_BY, aip.getUpdatedBy()); doc.addField(RodaConstants.INGEST_SIP_IDS, aip.getIngestSIPIds()); doc.addField(RodaConstants.INGEST_JOB_ID, aip.getIngestJobId()); doc.addField(RodaConstants.INGEST_UPDATE_JOB_IDS, aip.getIngestUpdateJobIds()); List<String> descriptiveMetadataIds = aip.getDescriptiveMetadata().stream().map(dm -> dm.getId()) .collect(Collectors.toList()); List<String> representationIds = aip.getRepresentations().stream().map(r -> r.getId()).collect(Collectors.toList()); doc.addField(RodaConstants.AIP_REPRESENTATION_ID, representationIds); doc.addField(RodaConstants.AIP_HAS_REPRESENTATIONS, !representationIds.isEmpty()); doc.addField(RodaConstants.AIP_GHOST, aip.getGhost() != null ? aip.getGhost() : false); SolrUtils.indexDescriptiveMetadataFields(RodaCoreFactory.getModelService(), aip.getId(), null, aip.getDescriptiveMetadata(), doc); Directory submissionDirectory = model.getSubmissionDirectory(aip.getId()); numberOfSubmissionFiles = storage.countResourcesUnderDirectory(submissionDirectory.getStoragePath(), true); } catch (NotFoundException e) { Directory documentationDirectory = model.getDocumentationDirectory(aip.getId());
public static List<List<String>> getDataInformation(List<String> fields, AIP aip, ModelService model, StorageService storage) { List<List<String>> dataInformation = new ArrayList<>(); for (Representation representation : aip.getRepresentations()) { boolean recursive = true; try (CloseableIterable<OptionalWithCause<File>> representationFiles = model.listFilesUnder(aip.getId(), representation.getId(), recursive)) { for (OptionalWithCause<File> subfile : representationFiles) { if (subfile.isPresent()) { dataInformation.add(retrieveFileInfo(fields, subfile.get(), aip, model, storage)); } else { LOGGER.error("Cannot retrieve file information", subfile.getCause()); } } } catch (NotFoundException | GenericException | RequestNotValidException | AuthorizationDeniedException | IOException e) { LOGGER.error("Error retrieving files of representation '{}' of AIP '{}': " + e.getMessage(), representation.getId(), aip.getId()); } } return dataInformation; }
public AIP createAIP(String parentId, String type, Permissions permissions, List<String> ingestSIPIds, String ingestJobId, boolean notify, String createdBy, boolean isGhost) throws RequestNotValidException, NotFoundException, GenericException, AlreadyExistsException, AuthorizationDeniedException { RodaCoreFactory.checkIfWriteIsAllowedAndIfFalseThrowException(nodeType); AIPState state = AIPState.ACTIVE; Directory directory = storage.createRandomDirectory(DefaultStoragePath.parse(RodaConstants.STORAGE_CONTAINER_AIP)); String id = directory.getStoragePath().getName(); Permissions inheritedPermissions = this.addParentPermissions(permissions, parentId); AIP aip = new AIP(id, parentId, type, state, inheritedPermissions, createdBy); aip.setGhost(isGhost); aip.setIngestSIPIds(ingestSIPIds); aip.setIngestJobId(ingestJobId); createAIPMetadata(aip); if (notify) { notifyAipCreated(aip).failOnError(); } return aip; }
@Override public String getUniqueId(AIP modelObject) { return modelObject.getId(); }
for (Representation representation : aip.getRepresentations()) { DefaultStoragePath representationStoragePath = DefaultStoragePath.parse(RodaConstants.STORAGE_CONTAINER_AIP, aip.getId(), RodaConstants.STORAGE_DIRECTORY_REPRESENTATIONS, representation.getId()); Path representationPath = storage.getDirectAccess(representationStoragePath).getPath(); representation.setCreatedBy(aip.getCreatedBy()); representation.setUpdatedOn(updateDate); representation.setUpdatedBy(aip.getUpdatedBy());
Report reportItem = PluginHelper.initPluginReportItem(this, aip.getId(), AIP.class); PluginHelper.updatePartialJobReport(this, model, reportItem, false, job); PluginState state = PluginState.SUCCESS; String aipId = aip.getId(); String jobId = aip.getIngestJobId(); String ingestSIPUUID = aip.getIngestSIPUUID(); if (accept) { aip.setState(AIPState.ACTIVE); model.updateAIPState(aip, job.getUsername()); jobPluginInfo.incrementObjectsProcessed(state); StringBuilder outcomeText = new StringBuilder().append("The AIP '").append(aip.getId()).append("' was "); if (accept) { outcomeText.append("accepted into the repository."); model.createUpdateAIPEvent(aip.getId(), null, null, null, PreservationEventType.APPRAISAL, "The process of updating an non active object of the repository", state, outcomeText.toString(), null, job.getUsername(), true);
private void processTransferredResource(IndexService index, ModelService model, Report report, Job job, TransferredResource transferredResource) { Report reportItem = PluginHelper.initPluginReportItem(this, transferredResource); Path bagitPath = Paths.get(transferredResource.getFullPath()); try { LOGGER.debug("Converting {} to AIP", bagitPath); SIP bagit = BagitSIP.parse(bagitPath); Optional<String> computedParentId = PluginHelper.getComputedParent(model, index, bagit.getAncestors(), computedSearchScope, forceSearchScope, job.getId()); AIP aipCreated = BagitToAIPPluginUtils.bagitToAip(bagit, model, METADATA_FILE, Arrays.asList(transferredResource.getName()), reportItem.getJobId(), computedParentId, job.getUsername(), PermissionUtils.getIngestPermissions(job.getUsername()), transferredResource.getUUID()); PluginHelper.createSubmission(model, createSubmission, bagitPath, aipCreated.getId()); createUnpackingEventSuccess(model, index, transferredResource, aipCreated, UNPACK_DESCRIPTION); reportItem.setSourceAndOutcomeObjectId(reportItem.getSourceObjectId(), aipCreated.getId()) .setPluginState(PluginState.SUCCESS); if (aipCreated.getParentId() == null && computedParentId.isPresent()) { reportItem.setPluginDetails(String.format("Parent with id '%s' not found", computedParentId.get())); } createWellformedEventSuccess(model, index, transferredResource, aipCreated); LOGGER.debug("Done with converting {} to AIP {}", bagitPath, aipCreated.getId()); } catch (RODAException | RuntimeException | ParseException e) { reportItem.setPluginState(PluginState.FAILURE).setPluginDetails(e.getMessage()); LOGGER.error("Error converting " + bagitPath + " to AIP", e); } report.addReport(reportItem); PluginHelper.createJobReport(this, model, reportItem); }
private void updateAIPsToBeAppraised(ModelService model, List<AIP> aips, IngestJobPluginInfo jobPluginInfo) { for (AIP aip : aips) { aip.setState(AIPState.UNDER_APPRAISAL); try { aip = model.updateAIPState(aip, PluginHelper.getJobUsername(this, model)); getParameterValues().put(RodaConstants.PLUGIN_PARAMS_OUTCOMEOBJECTID_TO_SOURCEOBJECTID_MAP, JsonUtils.getJsonFromObject(jobPluginInfo.getAipIdToTransferredResourceIds())); // update main report outcomeObjectState PluginHelper.updateJobReportState(this, model, aip.getIngestSIPUUID(), aip.getId(), AIPState.UNDER_APPRAISAL); // update counters of manual intervention jobPluginInfo.incrementOutcomeObjectsWithManualIntervention(); } catch (GenericException | NotFoundException | RequestNotValidException | AuthorizationDeniedException e) { LOGGER.error("Error while updating AIP state to '{}'. Reason: {}", AIPState.UNDER_APPRAISAL, e.getMessage()); } } }
public static AIP earkSIPToAIP(SIP sip, String username, Permissions fullPermissions, ModelService model, List<String> ingestSIPIds, String ingestJobId, Optional<String> parentId, String ingestSIPUUID, Plugin<?> plugin) throws RequestNotValidException, NotFoundException, GenericException, AlreadyExistsException, AuthorizationDeniedException, ValidationException, IOException, LockingException { AIPState state = AIPState.INGEST_PROCESSING; Permissions permissions = new Permissions(); boolean notify = false; String aipType = IngestHelper.getType(sip); AIP aip = model.createAIP(state, parentId.orElse(null), aipType, permissions, ingestSIPUUID, ingestSIPIds, ingestJobId, notify, username); PluginHelper.acquireObjectLock(aip, plugin); // process IP information processIPInformation(model, sip, aip.getId(), notify, false); // process IPRepresentation information for (IPRepresentation representation : sip.getRepresentations()) { processIPRepresentationInformation(model, representation, aip.getId(), notify, false, username, null); } model.notifyAipCreated(aip.getId()); AIP createdAIP = model.retrieveAIP(aip.getId()); // Set Permissions Permissions readPermissions = PermissionUtils.grantReadPermissionToUserGroup(model, createdAIP, aip.getPermissions()); Permissions finalPermissions = PermissionUtils.grantAllPermissions(username, readPermissions, fullPermissions); createdAIP.setPermissions(finalPermissions); model.updateAIP(createdAIP, username); return model.retrieveAIP(aip.getId()); }
private void processAIPs(ModelService model, JobPluginInfo jobPluginInfo, Job cachedJob, List<AIP> list) { for (AIP aip : list) { try { LOGGER.debug("Processing AIP {}", aip.getId()); String parentId = aip.getParentId(); if (parentId != null) { try { model.retrieveAIP(parentId); } catch (RODAException e) { // orphan was found aip.setParentId(newParentId); model.updateAIP(aip, cachedJob.getUsername()); // reporting Report reportItem = PluginHelper.initPluginReportItem(this, aip.getId(), AIP.class, AIPState.ACTIVE) .setPluginState(PluginState.SUCCESS) .setPluginDetails(String.format(MOVED_ORPHAN_AIP_FROM_TO, parentId, newParentId)); PluginHelper.updatePartialJobReport(this, model, reportItem, true, cachedJob); } } jobPluginInfo.incrementObjectsProcessedWithSuccess(); } catch (RODAException e) { LOGGER.error("Error processing AIP {} (RemoveOrphansAction)", aip.getId(), e); Report reportItem = PluginHelper.initPluginReportItem(this, aip.getId(), AIP.class, AIPState.ACTIVE) .setPluginState(PluginState.FAILURE).addPluginDetails(e.getMessage()); PluginHelper.updatePartialJobReport(this, model, reportItem, true, cachedJob); jobPluginInfo.incrementObjectsProcessedWithFailure(); } } }
@Override public ReturnWithExceptions<Void, ModelObserver> representationCreated(Representation representation) { ReturnWithExceptions<Void, ModelObserver> ret = new ReturnWithExceptions<>(this); try { AIP aip = model.retrieveAIP(representation.getAipId()); List<String> ancestors = SolrUtils.getAncestors(aip.getParentId(), model); indexRepresentation(aip, representation, ancestors).addTo(ret); if (ret.isEmpty()) { indexPreservationsEvents(aip.getId(), representation.getId()).addTo(ret); if (aip.getRepresentations().size() == 1) { SolrUtils.update(index, IndexedAIP.class, aip.getId(), Collections.singletonMap(RodaConstants.AIP_HAS_REPRESENTATIONS, true), (ModelObserver) this).addTo(ret); } } } catch (RequestNotValidException | NotFoundException | GenericException | AuthorizationDeniedException e) { LOGGER.error("Cannot index representation: {}", representation, e); ret.add(e); } return ret; }
private void processAIP(IndexService index, ModelService model, Report report, Job job, AIP aip) { Report reportItem = PluginHelper.initPluginReportItem(this, aip.getId(), AIP.class, AIPState.INGEST_PROCESSING); PluginHelper.updatePartialJobReport(this, model, reportItem, false, job); String outcomeDetail = ""; try { LOGGER.debug("Auto accepting AIP {}", aip.getId()); if (aip.getState() != AIPState.ACTIVE) { aip.setState(AIPState.ACTIVE); model.updateAIPState(aip, job.getUsername()); } reportItem.setPluginState(PluginState.SUCCESS).setOutcomeObjectState(AIPState.ACTIVE); LOGGER.debug("Done with auto accepting AIP {}", aip.getId()); } catch (RODAException e) { LOGGER.error("Error updating AIP (metadata attribute state=ACTIVE)", e); outcomeDetail = "Error updating AIP (metadata attribute state=ACTIVE): " + e.getMessage(); reportItem.setPluginState(PluginState.FAILURE).setPluginDetails(outcomeDetail) .setOutcomeObjectState(AIPState.UNDER_APPRAISAL); } try { boolean notify = true; PluginHelper.createPluginEvent(this, aip.getId(), model, index, reportItem.getPluginState(), outcomeDetail, notify); } catch (ValidationException | RequestNotValidException | NotFoundException | GenericException | AuthorizationDeniedException | AlreadyExistsException e) { LOGGER.error("Error creating event: {}", e.getMessage(), e); } report.addReport(reportItem); PluginHelper.updatePartialJobReport(this, model, reportItem, true, job); }
private ReturnWithExceptions<Void, ModelObserver> preservationEventsPermissionsUpdated(final AIP aip) { ReturnWithExceptions<Void, ModelObserver> ret = new ReturnWithExceptions<>(this); try (CloseableIterable<OptionalWithCause<PreservationMetadata>> preservationMetadata = model .listPreservationMetadata(aip.getId(), true)) { for (OptionalWithCause<PreservationMetadata> opm : preservationMetadata) { if (opm.isPresent()) { PreservationMetadata pm = opm.get(); if (PreservationMetadataType.EVENT.equals(pm.getType())) { preservationEventPermissionsUpdated(pm, aip.getPermissions(), aip.getState()).addTo(ret); } } else { LOGGER.error("Cannot index premis event", opm.getCause()); ret.add(opm.getCause()); } } } catch (RequestNotValidException | NotFoundException | GenericException | AuthorizationDeniedException | IOException e) { LOGGER.error("Cannot index preservation events", e); ret.add(e); } return ret; }
if (modelAIP.getType() != null) { node = node.put("type", modelAIP.getType()); List<DescriptiveMetadata> descriptiveMetadata = modelAIP.getDescriptiveMetadata(); if (descriptiveMetadata != null && !descriptiveMetadata.isEmpty()) { ArrayNode metadata = mapper.createArrayNode(); dmNode = dmNode.put("metadataVersion", dm.getVersion()); Binary b = model.retrieveDescriptiveMetadataBinary(modelAIP.getId(), dm.getId()); InputStream is = b.getContent().createInputStream(); dmNode = dmNode.put("content", new String(Base64.encodeBase64(IOUtils.toByteArray(is))));
ModelService model = RodaCoreFactory.getModelService(); StoragePath aipJsonPath = DefaultStoragePath.parse(ModelUtils.getAIPStoragePath(aip.getId()), RodaConstants.STORAGE_AIP_METADATA_FILENAME); addToZip(zipEntries, storage.getBinary(aipJsonPath)); for (DescriptiveMetadata dm : aip.getDescriptiveMetadata()) { Binary dmBinary = model.retrieveDescriptiveMetadataBinary(aip.getId(), dm.getId()); addToZip(zipEntries, dmBinary); .listPreservationMetadata(aip.getId(), true)) { for (OptionalWithCause<org.roda.core.data.v2.ip.metadata.PreservationMetadata> preservation : preservations) { if (preservation.isPresent()) { for (Representation rep : aip.getRepresentations()) { try (CloseableIterable<OptionalWithCause<org.roda.core.data.v2.ip.File>> allFiles = model .listFilesUnder(aip.getId(), rep.getId(), true)) { for (OptionalWithCause<org.roda.core.data.v2.ip.File> file : allFiles) { if (file.isPresent()) { .listOtherMetadata(aip.getId(), rep.getId(), null, null, null)) { for (OptionalWithCause<org.roda.core.data.v2.ip.metadata.OtherMetadata> otherMetadata : allOtherMetadata) { if (otherMetadata.isPresent()) { OtherMetadata o = otherMetadata.get(); StoragePath otherMetadataStoragePath = ModelUtils.getOtherMetadataStoragePath(aip.getId(), rep.getId(), o.getFileDirectoryPath(), o.getFileId(), o.getFileSuffix(), o.getType()); addToZip(zipEntries, storage.getBinary(otherMetadataStoragePath));