How to use

Document

in

com.github.cafdataprocessing.corepolicy.common

private boolean shouldSkipPolicyBecauseAlreadyRunWithDiagnostics(com.github.cafdataprocessing.corepolicy.common.Document doc,
                                 Long policyId) {
  if (doc.getMetadata().containsKey(PolicyWorkerConstants.POLICYWORKER_FAILURE_POLICY + policyId)) {
    return true;
  }
  return false;
}

@Override
public void serialize(Document value, JsonGenerator jgen, SerializerProvider provider) throws IOException, JsonProcessingException {
  jgen.writeStartObject();
  if(value.getMetadata()!=null){
    for(String key:value.getMetadata().keySet()){
      if(key.equalsIgnoreCase(DocumentFields.Reference)){
        jgen.writeFieldName(DocumentFields.Reference);
        jgen.writeString(value.getReference());
        Collection<String> values = value.getMetadata().get(key);
        if(values.size() == 1) {
          jgen.writeFieldName(key);
  if(value.getDocuments()!=null){
    jgen.writeArrayFieldStart("document");
    for(Document childDocument:value.getDocuments()){
      serialize(childDocument, jgen, provider);

public static String getReference(Document document){
  return getReference(document.getMetadata(), !document.getDocuments().isEmpty());
}

com.github.cafdataprocessing.worker.policy.shared.Document newDocument =
    new com.github.cafdataprocessing.worker.policy.shared.Document();
newDocument.setReference(document.getReference());
if (document.getMetadata() != null) {
  for (Map.Entry<String, String> entry : document.getMetadata().entries()) {
    newDocument.getMetadata().put(entry.getKey(), entry.getValue());
if(document.getStreams() != null) {
  for (Map.Entry<String, InputStream> streamEntry : document.getStreams().entries()){
    if(streamEntry.getValue() == null || !(streamEntry.getValue() instanceof DataStoreAwareInputStream)){
      continue;
if (document.getDocuments() != null) {
  newDocument.setDocuments(document.getDocuments().stream().map(this::convert).collect(Collectors.toList()));

public DocumentUnderEvaluationImpl(Document document, ConditionEngineMetadata conditionEngineMetadata, ApiProperties apiProperties){
  this(conditionEngineMetadata, apiProperties);
  // we hold onto a list of  MetadataValues internally, so convert at this point so we do it only once.
  for( Map.Entry<String, String> entry : document.getMetadata().entries())
  {
    this.metadata.put( entry.getKey(), new MetadataValue(apiProperties, entry.getValue()));
  }
  // we hold onto a list of  MetadataValues internally, so convert at this point so we do it only once.
  for( Map.Entry<String, InputStream> entry : document.getStreams().entries())
  {
    this.streams.put( entry.getKey(), new MetadataValue(apiProperties, entry.getValue()));
  }
  this.addMetadataString(DocumentFields.ChildDocumentCount, String.valueOf(document.getDocuments().size()));
  final int depth = getDepth();
  this.documents = document.getDocuments().stream().
      map(d -> {
        DocumentUnderEvaluation documentUnderEvaluation = new DocumentUnderEvaluationImpl(d, conditionEngineMetadata, apiProperties);
        documentUnderEvaluation.addMetadataString( DocumentFields.ChildDocumentDepth, String.valueOf(depth+1));
        return documentUnderEvaluation;
      }).collect(Collectors.toList());
  setupPreevaluatedInformation(document);
}

for (Document document : documents) {
  try {
    document.close();
  } catch (Exception e) {
    logger.warn("Could not close document", e);

public static void reapplyTemporaryWorkingData(com.github.cafdataprocessing.corepolicy.common.Document document,
                        Multimap<String, String> temporaryData )
{
  Multimap<String, String> docData = document.getMetadata();
  for( String propName : temporaryData.keySet() )
  {
    docData.putAll( propName, temporaryData.get(propName));
  }
}

  private String getStorageReference(Document document)
  {
    Collection<String> storageReferences = document.getMetadata().get("storageReference");
    if (storageReferences.size() != 1) {
      throw new RuntimeException("No storageReference set");
    }
    return storageReferences.stream().findAny().get();
  }
}

public final static List<Long> getCollectionSequencesAlreadyStarted( com.github.cafdataprocessing.corepolicy.common.Document document ){
  Multimap<String, String> metadata = document.getMetadata();
  // get ids from property = "POLICYWORKER_COLLECTIONSEQUENCE" - mv field of sequenceids.
  return getIdsFromMetadataPropertyStringMap(metadata, ApiStrings.POLICYWORKER_COLLECTION_SEQUENCE);
}

private static Collection<Long> getPoliciesYetToBeExecuted(com.github.cafdataprocessing.corepolicy.common.Document document,
                              Long collectionSequenceId, Collection<Long> resolvedPolicies) {
  Collection<Long> policiesExecutedAlready =
      DocumentFields.getPoliciesAlreadyExecutedForStringMap(collectionSequenceId, document.getMetadata());
  Collection<Long> policiesYetToBeExecuted = new ArrayList<>();
  policiesYetToBeExecuted.addAll(resolvedPolicies);
  policiesYetToBeExecuted.removeAll(policiesExecutedAlready);
  return policiesYetToBeExecuted;
}

@Override
public Long extract(Document document) throws Exception {
  Optional<String> collectionSequenceIdFieldValue = document.getMetadata().get(DocumentFields.CollectionSequence).stream().findFirst();
  if(!collectionSequenceIdFieldValue.isPresent()){
    throw new Exception("No " + DocumentFields.CollectionSequence + " field");
  }
  return getCollectionSequenceId(collectionSequenceIdFieldValue.get());
}

public final static List<Long> getCollectionSequencesAlreadyCompleted( com.github.cafdataprocessing.corepolicy.common.Document document ){
  Multimap<String, String> metadata = document.getMetadata();
  // get ids from property = "POLICYWORKER_COLLECTIONSEQUENCE_COMPLETED" - mv field of sequenceids.
  return getIdsFromMetadataPropertyStringMap(metadata, ApiStrings.POLICYWORKER_COLLECTION_SEQUENCES_COMPLETED);
}

public static void setReference(Document document, String reference){
  setReference(document.getMetadata(), reference);
}

/***
 * Add the collection sequence id when we have completed executing all resolved policies for its classification.
 * @param document
 * @param collectionSequenceId
 */
public final static void addCollectionSequenceCompletedInfo( com.github.cafdataprocessing.corepolicy.common.Document document, Long collectionSequenceId )
{
  Multimap<String, String> metadata = document.getMetadata();
  // get all completed collection sequences - "POLICYWORKER_COLLECTION_SEQUENCES_COMPLETED" - mv field
  List<Long> csIds = getIdsFromMetadataPropertyStringMap(metadata, ApiStrings.POLICYWORKER_COLLECTION_SEQUENCES_COMPLETED);
  // if the specified sequence already exists in this list, just return, we have nothing to do.
  if ( csIds.contains( collectionSequenceId ))
    return;
  // add on the collection sequence id onto the mv list.
  metadata.put(ApiStrings.POLICYWORKER_COLLECTION_SEQUENCES_COMPLETED, collectionSequenceId.toString() );
}

/**
 * Removes fields that are added to track working progress through Collection Sequence execution.
 * @param document The Document to remove temporary working data from.
 * @return The temporary data that was removed.
 */
public static Multimap<String, String> removeTemporaryWorkingData(com.github.cafdataprocessing.corepolicy.common.Document document )
{
  // using same map type as the used inside our document object.
  Multimap<String, String> temporaryData = ArrayListMultimap.create();
  Multimap<String, String> docData = document.getMetadata();
  getListOfKnownTemporaryData(docData).stream().filter(propName -> docData.containsKey(propName)).forEach(propName -> {
    temporaryData.putAll(propName, docData.get(propName));
    docData.removeAll(propName);
  });
  return temporaryData;
}

/**
 * This method will REMOVE any existing temp (e.g. validation hash) metadata fields, and add new ones that represent
 * this metadata.
 * @param document The document who's metadata should be updated.
 */
public void applyTemporaryMetadataToDocument( ConditionEngineResult conditionEngineResult, Document document ){
  try {
    //Lets remove all the field values
    document.getMetadata().removeAll(DocumentFields.EvaluationInformationBlob);
    document.getMetadata().removeAll(DocumentFields.MetadataHash);
    //now lets get new values and add them to the document
    String sourceJson = mapper.writeValueAsString(conditionEngineResult);
    String compressedBlob = ZipUtils.compressStringAndEncode(sourceJson);
    String securityHash = generateSecurityHash(conditionEngineResult);
    document.getMetadata().put(DocumentFields.EvaluationInformationBlob, compressedBlob);
    document.getMetadata().put(DocumentFields.MetadataHash, securityHash);
  } catch (IOException e) {
    CpeException cpeException = new BackEndRequestFailedCpeException(BackEndRequestFailedErrors.UnableToAddTemporaryMetadata,e );
    logger.error("Error adding temporary evaluation metadata to a document.", cpeException);
    throw cpeException;
  }
}

  protected void applyFieldActions(Document document, Collection<FieldAction> fieldActions) {
    for (FieldAction fieldAction : fieldActions) {

      FieldAction.Action action = fieldAction.getAction();

      // N.B. Weird javac behaviour whereby if a switch is used below 2 ProcessDocument class files are
      // generated, once called ProcessDocument$1.class and once ProcessDocument.class which stops us from
      // correctly running javah on this class.
      if ( action == FieldAction.Action.SET_FIELD_VALUE ){
        document.getMetadata().get(fieldAction.getFieldName()).clear();
        document.getMetadata().put(fieldAction.getFieldName(), fieldAction.getFieldValue());
      }
      else if ( action == FieldAction.Action.ADD_FIELD_VALUE ) {
        document.getMetadata().put(fieldAction.getFieldName(), fieldAction.getFieldValue());
      }
    }
  }
}

  private void applyFieldActions(Document document, Collection<FieldAction> fieldActions) {
    for (FieldAction fieldAction : fieldActions) {

      FieldAction.Action action = fieldAction.getAction();

      // N.B. Weird javac behaviour whereby if a switch is used below 2 ProcessDocument class files are
      // generated, once called ProcessDocument$1.class and once ProcessDocument.class which stops us from
      // correctly running javah on this class.
      if ( action == FieldAction.Action.SET_FIELD_VALUE ){
        document.getMetadata().get(fieldAction.getFieldName()).clear();
        document.getMetadata().put(fieldAction.getFieldName(), fieldAction.getFieldValue());
      }
      else if ( action == FieldAction.Action.ADD_FIELD_VALUE ) {
        document.getMetadata().put(fieldAction.getFieldName(), fieldAction.getFieldValue());
      }
    }
  }
}

private void setupPreevaluatedInformation(Document document) {
  // Check if we have any supplied metadata which has been evaluated before.
  ConditionEngineResult result = conditionEngineMetadata.createResult(document.getMetadata());
  applyEvaluationInfo(result, this);
}

private Multimap<String, ReferencedData> getRequestedFieldsData(Collection<String> fieldsRequested, Document document) {
  //gather source data fields for Extract task
  Multimap<String, ReferencedData> sourceData = ArrayListMultimap.create();
  //add metadata fields to source data (filter to list provided on policy definition if required)
  for (Map.Entry<String, String> metadata : document.getMetadata().entries()) {
    String fieldName = metadata.getKey();
    if (fieldsRequested.contains(fieldName)) {
      sourceData.put(fieldName, ReferencedData.getWrappedData(metadata.getValue().getBytes()));
    }
  }
  WorkerResponseHolder workerResponseHolder = applicationContext.getBean(WorkerResponseHolder.class);
  com.github.cafdataprocessing.worker.policy.shared.Document taskDataDocumentToClassify = workerResponseHolder.getTaskData().getDocument();
  //add metadata reference fields to source data (filter to list provided on policy definition if required)
  for (Map.Entry<String, ReferencedData> metadataReference : taskDataDocumentToClassify.getMetadataReferences().entries()) {
    String fieldName = metadataReference.getKey();
    if (fieldsRequested.contains(fieldName)) {
      sourceData.put(metadataReference.getKey(), metadataReference.getValue());
    }
  }
  return sourceData;
}