provenance_schema.yaml

############################################# Rules #############################################
# Entity properties:
# - type: data type of the property, one of the following: string|integer|boolean|list|json_string
# - generated: whether the property is auto generated (either with a  `before_create_trigger` or not) or user provided, default to false
# - required_on_create: whether the property is required from user reqeust JSON for entity creation via POST
# - immutable: true indicates the property can NOT be updated after the entity is created, default to false
# - transient: whether the property to persist in database or not, default to false
# - exposed: whether the property gets returned to the user or not, default to true
# - indexed: whether the property gets indexed into Elasticsearch/Opensearch or not, default to true. Applicable to the specialized `GET /documents/<id>` endpoint
# - trigger types: before_create_trigger|after_create_trigger|before_update_trigger|after_update_trigger|on_read_trigger|on_index_trigger, one property can have none (default) or more than one triggers
# - updated_peripherally: a temporary measure to correctly handle any attributes which are potentially updated by multiple triggers

# Entity level validators:
# - types: before_entity_create_validator - validation method needed for creating an entity.
#          before_entity_update_validator - validation method needed for updating an entity.

# Property level validators: 
# - types: before_property_create_validators|before_property_update_validators, a list of validation methods

# Entity creation via http POST request:
# - Use `generated: true` to mark a property as to be auto generated by the program instead of from user input JSON
# - If a property is marked as `generated: true`, either no trigger method needed (E.g. Donor.image_files, Dataset.title) or a `before_create_trigger` can be used to generate the value
# - If a property has `before_create_trigger`, it can't be specified in client request JSON
# - If a property is marked as `generated: true`, it can't be `required_on_create: true` at the same time
# - Use `required_on_create: true` to mark a property as required from user input
# - If a property is marked as `required_on_create: true`, it can't be `generated: true` and must not have trigger at the same time
# - If a property is marked as `generated: true`, user can't specify this property in request JSON since it'll be auto generated with or without using other fields

# Entity update via http PUT request:
# - By default, all entity properties as mutable (can be updated directly by users). Use `immutable: true` to mark a property as immutable
# - Use `auto_update: true` to mark a property as to be ALWAYS auto updated when the entity gets updated, e.g., `last_modified_timestamp`
# - If a property is marked as `auto_update: true`, there must be a `before_update_trigger` method defined for that property

# Entity retrival via http GET request:
# - By default, all entity properties will be returned. Use `exposed: false` to filter out the property from response
# - If a property has `exposed: false`, it doesn't need any `on_read_trigger` method
# - If a property has `on_read_trigger`, it must be `transient: true`, meaning it's not stored in neo4j and only available during response
# - If a property is `transient: true`, it can have `on_read_trigger` or not have one (when `exposed: false`)
# - If a property has `on_index_trigger`, it must be marked as `indexed: true`

############################################# Schema #############################################
# Shared properties across ACTIVITIES and ENTITIES
shared_properties: &shared_properties
  ###### Creation info properties ######
  created_timestamp:
    type: integer
    generated: true # Disallow user input from request json when being created
    immutable: true # Disallow update of this property when the entity is being updated
    indexed: true
    description: "The timestamp of when the node was created.  The format is an integer representing milliseconds since midnight Jan 1, 1970"
    before_create_trigger: set_timestamp
  created_by_user_displayname:
    type: string
    generated: true
    immutable: true
    indexed: true
    description: "The name of the person or process authenticated when creating the object"
    before_create_trigger: set_user_displayname
  created_by_user_email:
    type: string
    generated: true
    immutable: true
    indexed: true
    description: "The email address of the person or process authenticated when creating the object."
    before_create_trigger: set_user_email
  created_by_user_sub:
    type: string
    generated: true
    immutable: true
    indexed: false
    description: "The subject id as provided by the authorization mechanism for the person or process authenticated when creating the object."
    before_create_trigger: set_user_sub
  ###### ID properties ######
  uuid:
    type: string
    generated: true
    immutable: true
    indexed: true
    description: "The HuBMAP unique identifier, intended for internal software use only.  This is a 32 digit hexadecimal uuid e.g. 461bbfdc353a2673e381f632510b0f17"
    before_create_trigger: set_uuid
  hubmap_id:
    type: string
    generated: true
    immutable: true
    indexed: true
    description: "A HuBMAP Consortium wide unique identifier randomly generated in the format HBM###.ABCD.### for every entity."
    before_create_trigger: set_hubmap_id


####################################################################################################
## Activities
####################################################################################################

ACTIVITIES:
  Activity:
    properties:
      <<: *shared_properties
      creation_action:
        type: string
        indexed: false
        description: "The activity that was performed."
        before_create_trigger: set_activity_creation_action


####################################################################################################
## Entities
####################################################################################################

doi_properties: &doi_properties
  ###### DOI properties ######
  registered_doi:
    # This property gets set via a PUT by ingest-api after the DOI is created via DataCite
    # So we can't define it as `immutable: true`
    type: string
    indexed: true
    description: "The doi of the registered entity. e.g. 10.35079/hbm289.pcbm.487"
    before_property_create_validators:
     - verify_DOI_pair
     - halt_update_if_DOI_exists
    before_property_update_validators:
     - verify_DOI_pair
     - halt_update_if_DOI_exists
  doi_url:
    # This property gets set via a PUT by ingest-api after the DOI is created via DataCite
    # So we can't define it as `immutable: true`
    type: string
    indexed: true
    description: "The url from the doi registry for this entity. e.g. https://doi.org/10.35079/hbm289.pcbm.487"
    before_property_create_validators:
     - verify_DOI_pair
     - halt_update_if_DOI_exists
    before_property_update_validators:
     - verify_DOI_pair
     - halt_update_if_DOI_exists
  contributors:
    type: list
    indexed: true
    description: "A list of the people who created the entity with full name, email, ORCID iD, institution, etc.. This is analogous to the author list on a publication."
  contacts:
    type: list
    indexed: true
    description: "A list of the people who are the main contacts to get information about the entity."

shared_entity_properties: &shared_entity_properties 
  ###### Last Modified Properties #######
  last_modified_timestamp:
    type: integer
    generated: true
    indexed: true
    auto_update: true # Will always update automatically if the entity gets updated, different from `immutable: false` (can be updated by users or not)
    description: "The timestamp of when the object was last modified.  The format is an integer representing milliseconds since midnight, Jan 1, 1970"
    # Set on both create and update
    # These two properties will have the same timestamp due to the use of 
    # neo4j function TIMESTAMP() instead of generating the timestamp in Python
    before_create_trigger: set_timestamp
    before_update_trigger: set_timestamp
  last_modified_user_sub:
    type: string
    generated: true
    indexed: false
    auto_update: true
    description: "The subject id of the user who last modified the entity as provided by the authorization mechanism for the person or process authenticated when the object was modified."
    # Set on both create and update
    before_create_trigger: set_user_sub
    before_update_trigger: set_user_sub
  last_modified_user_email:
    type: string
    generated: true
    indexed: false
    auto_update: true
    description: "The email address of the person or process authenticated when the object was last modified."
    # Set on both create and update
    before_create_trigger: set_user_email
    before_update_trigger: set_user_email
  last_modified_user_displayname:
    type: string
    generated: true
    indexed: false
    auto_update: true
    description: "The name of the person or process authenticated when the object was last modified."
    # Set on both create and update
    before_create_trigger: set_user_displayname
    before_update_trigger: set_user_displayname
  ###### All other shared properties ######
  entity_type:
    type: string
    generated: true
    immutable: true
    indexed: true
    description: "One of the normalized entity types: Dataset, Collection, Sample, Donor"
    before_create_trigger: set_entity_type

ENTITIES:
  ############################################# Collection #############################################
  Collection:
    before_entity_update_validator:
      # Halt modification of entities which are "locked", such as a Dataset with status == 'Published'
      - validate_entity_not_locked_before_update
    excluded_properties_from_public_response:
      - datasets:
        - lab_dataset_id
        - metadata:
          - lab_id
    # Collection can not be derivation source but not target
    derivation:
      source: false
      target: false
    # Collection doesn't actually need data_access_level property
    properties: &shared_collection_properties
      <<: *shared_properties
      <<: *shared_entity_properties
      # Because Collection-specific validation is needed for some
      # DOI fields, we don't use the shared <<: *doi_properties here
      ###### DOI properties with entity-specific validations + copies of validations in doi_properties ######
      registered_doi:
        type: string
        indexed: true
        description: "See description in doi_properties section above"
        before_property_create_validators:
          - verify_DOI_pair
          - halt_DOI_if_collection_missing_elements
          - halt_DOI_if_unpublished_dataset
        before_property_update_validators:
          - verify_DOI_pair
          - halt_update_if_DOI_exists
          - halt_DOI_if_collection_missing_elements
          - halt_DOI_if_unpublished_dataset
      doi_url:
        type: string
        indexed: true
        description: "See description in doi_properties section above"
        before_property_create_validators:
          - verify_DOI_pair
          - halt_DOI_if_collection_missing_elements
          - halt_DOI_if_unpublished_dataset
        before_property_update_validators:
          - verify_DOI_pair
          - halt_update_if_DOI_exists
          - halt_DOI_if_collection_missing_elements
          - halt_DOI_if_unpublished_dataset
      # Contributors and contacts are required for a Collection to be public
      contributors:
        type: list
        indexed: true
        description: "See description in doi_properties section above"
      contacts:
        type: list
        indexed: true
        description: "See description in doi_properties section above"

      ###### entity-specific validations ######
      title:
        type: string
        indexed: true
        description: "The title of the Collection."
        required_on_create: true
      description:
        type: string
        indexed: true
        description: "Free text description of the collection"
        required_on_create: true
      ###### Transient properties ######
      dataset_uuids:
        type: list
        indexed: false
        before_property_create_validators:
          - validate_no_duplicates_in_list
          - collection_entities_are_existing_datasets
        before_property_update_validators:
          - validate_no_duplicates_in_list
          - collection_entities_are_existing_datasets
        transient: true
        exposed: false
        description: "The uuids of Datasets in this Collection.  Used to pass source entity ids in on a POST call to create the linkages."
        # Note: link_collection_to_direct_datasets() will always delete all the old linkages first
        after_create_trigger: link_collection_to_datasets
        after_update_trigger: link_collection_to_datasets
      datasets:
        type: list
        transient: true
        generated: true
        indexed: true
        description: "The datasets that are contained in the collection."
        # A few time-consuming properties (with read triggers) of each dataset are excluded
        on_read_trigger: get_collection_datasets
        on_index_trigger: get_collection_datasets

      # Added group_uuid and group_name for Collection - 4/30/2024 by Zhou
      # A user who is a member of multiple groups HAS to send in the group_uuid 
      group_uuid:
        type: string
        immutable: true
        indexed: true
        description: "The uuid of globus group which the user who created this entity is a member of.  This is required on Create/POST if the user creating the Donor is a member of more than one write group.  This property cannot be set via PUT (only on Create/POST)."
        before_create_trigger: set_group_uuid #method that, if group_uuid is not already set looks for membership in a single "data provider" group and sets to that. Otherwise if not set and no single "provider group" membership throws error
      # Must set in neo4j
      group_name:
        # It's not being mapped in the current version, what to do for the existing entities?
        type: string
        generated: true
        immutable: true
        indexed: true
        description: "The displayname of globus group which the user who created this entity is a member of"
        before_create_trigger: set_group_name #same as group_uuid, except set group_name


  ############################################# Dataset #############################################
  Dataset:
    # Only allowed applications can create new Dataset via POST
    before_entity_create_validator: validate_application_header_before_entity_create
    before_entity_update_validator:
      # Halt modification of entities which are "locked", such as a Dataset with status == 'Published'
      - validate_entity_not_locked_before_update
    # Dataset can be either derivation source or target
    excluded_properties_from_public_response:
      - lab_dataset_id
      - metadata:
        - lab_id
      - direct_ancestors:
        # Sample ancestors of a Dataset must have these fields removed
        - lab_tissue_sample_id
        - submission_id
        # Dataset ancestors of a Dataset must have these fields removed
        - lab_dataset_id
        # Both Sample and Dataset ancestors of a Dataset must have these fields removed
        - metadata:
          - lab_id
    derivation:
      source: true
      target: true
    properties: &shared_dataset_properties
      <<: *shared_properties
      <<: *shared_entity_properties
      <<: *doi_properties
      antibodies:
        type: list
        indexed: false
        description: "A list of antibodies used in the assay that created the dataset"
      creation_action:
        type: string
        transient: true
        immutable: true
        indexed: true
        on_read_trigger: get_creation_action_activity
        on_index_trigger: get_creation_action_activity
        description: "The activity that was performed."
        before_property_create_validators:
          - validate_creation_action
      description:
        type: string
        indexed: true
        description: "Free text description of the dataset"
      dataset_info:
        type: string
        indexed: true
        description: "Additional information about the dataset, which can be used to find this dataset, including lab specific (non-PHI) identifiers."
      # The Dataset.data_access_level is based on Dataset.status and Dataset.contains_human_genetic_sequences
      data_access_level:
        type: string
        generated: true
        indexed: true
        description: "One of the values: public, consortium, protected. Only Dataset may have protected value"
        before_create_trigger: set_data_access_level
      # When contains_human_genetic_sequences is true, even if status is 'Published', the data_access_level is still 'protected'
      contains_human_genetic_sequences:
        type: boolean
        indexed: true
        required_on_create: true # Only required for create via POST, not update via PUT
        description: "True if the data contains any human genetic sequence information."
      error_message:
        type: string
        indexed: true
        description: "An open text field that holds the last error message that arose from pipeline validation or analysis."
      status:
        type: string
        before_property_update_validators: 
          - validate_application_header_before_property_update
          - validate_dataset_status_value
          - validate_status_changed
          - validate_dataset_not_component
        generated: true
        indexed: true
        description: "One of: New|Processing|Published|QA|Error|Hold|Invalid|Submitted|Incomplete"
        before_create_trigger: set_dataset_status_new
        after_create_trigger: set_status_history
        after_update_trigger: update_status
      title:
        type: string
        generated: true # Disallow entry from users via POST
        immutable: true # Disallow update via PUT
        indexed: true
        description: "The auto generated dataset title."
        on_read_trigger: get_dataset_title
        on_index_trigger: get_dataset_title
      lab_dataset_id:
        type: string
        indexed: true
        description: "A name or identifier used by the lab who is uploading the data to cross reference the data locally"
      data_types:
        before_property_create_validators:
          - validate_no_duplicates_in_list
        before_property_update_validators:
          - validate_no_duplicates_in_list
        type: list
        indexed: true
        required_on_create: false
        description: "The data or assay types contained in this dataset as a json array of strings."
      dataset_type:
        before_property_create_validators:
          - validate_recognized_dataset_type
        before_property_update_validators:
          - validate_recognized_dataset_type
        type: string
        indexed: true
        required_on_create: true # Required for create via POST, not update via PUT
        description: "The assay types of this Dataset. Valid values are from UBKG are queried by schema_manager.get_valueset_dataset_type() using the Ontology API."
      collections:
        type: list
        transient: true
        generated: true
        indexed: false
        description: "A list of collections that this dataset belongs to. Will be returned in response"
        on_read_trigger: get_dataset_collections
        # No on_index_trigger to include collections in the OpenSearch document for a Dataset
      upload:
        type: json_string # dict
        transient: true
        generated: true
        indexed: false
        description: "The Upload that this dataset is associated with. Will be returned in response"
        on_read_trigger: get_dataset_upload
        # No on_index_trigger to include upload in the OpenSearch document for a Dataset
      direct_ancestor_uuids:
        required_on_create: true # Only required for create via POST, not update via PUT
        type: list
        before_property_create_validators:
          - validate_no_duplicates_in_list
        before_property_update_validators:
          - validate_no_duplicates_in_list
          - validate_not_invalid_creation_action
          - validate_id_not_in_direct_ancestor
        transient: true
        exposed: false
        indexed: false
        description: "The uuids of source entities from which this new entity is derived.  Used to pass source entity ids in on POST or PUT calls used to create the linkages."
        # Note: link_dataset_to_direct_ancestors() will always delete all the old linkages first
        after_create_trigger: link_dataset_to_direct_ancestors
        after_update_trigger: link_dataset_to_direct_ancestors
      direct_ancestors:
        type: list
        description: "A list of direct parent ancestors (one level above) that the Dataset was derived from."
        generated: true
        transient: true
        indexed: false
        on_read_trigger: get_dataset_direct_ancestors
        # No on_index_trigger to include direct_ancestors in the OpenSearch document for a Dataset
      published_timestamp:
        type: integer
        immutable: true
        generated: true
        indexed: true
        description: "The timestamp of when the dataset was published.  The format is an integer representing milliseconds since midnight, Jan 1, 1970.  Cannot be set directly must be set with the /datasets/<id>/publish method."
      published_user_displayname:
        type: string
        generated: true
        immutable: true
        indexed: false
        description: "The name of the authenticated user or process that published the data.  Cannot be set directly must be set with the /datasets/<id>/publish method."
      published_user_sub:
        type: string
        generated: true
        immutable: true
        indexed: false
        description: "The subject id as provided by the authorization mechanism for the person or process authenticated when the dataset was publised.  Cannot be set directly must be set with the /datasets/<id>/publish method."
      published_user_email:
        type: string
        generated: true
        immutable: true
        indexed: false
        description: "The email address provided by the authorization mechanism for the person or process authenticated when published.  Cannot be set directly must be set with the /datasets/<id>/publish method."
      pipeline_message:
        #todo: where is this attribute sourced from?  Is it stored in the database? <- Not in neo4j
        type: string
        indexed: false
      ingest_metadata:
        type: json_string # dict
        indexed: true
        description: "Information associated with running the ingest and processing pipelines."
      metadata:
        type: json_string
        indexed: true
        description: "Metadata associated with the ingested experimental data."
      files:
        type: list
        indexed: true
        description: "A list of files associated with the dataset." 
      calculated_metadata:
        type: json_string
        indexed: true
        description: "Calculated metadata outputted from the processing pipeline." 
      local_directory_rel_path:
        # Example: protected/<TMC>/<uuid>
        type: string
        generated: true
        transient: true
        # This attribute should not go in the index ever in the way it comes from Neo4j, but also is not
        # needed now.  So simply excluded it from the index rather than trigger fixing it up & then skipping it.
        indexed: false
        description: "The path on the local file system, relative to the base data directory, where the data is stored."
        on_read_trigger: get_local_directory_rel_path
        # No on_index_trigger to include local_directory_rel_path in the OpenSearch document for a Dataset
      run_id:
        type: string
        indexed: false
      ingest_id:
        type: string
        indexed: false
      # A user who is a member of multiple groups HAS to send in the group_uuid
      group_uuid:
        type: string
        immutable: true
        indexed: true
        description: "The uuid of globus group which the user who created this entity is a member of.  This is required on Create/POST if the user creating the Donor is a member of more than one write group.  This property cannot be set via PUT (only on Create/POST)."
        before_create_trigger: set_group_uuid #method that, if group_uuid is not already set looks for membership in a single "data provider" group and sets to that. Otherwise if not set and no single "provider group" membership throws error
      # Must set in neo4j
      group_name:
        # It's not being mapped in the current version, what to do for the existing entities?
        type: string
        generated: true
        immutable: true
        indexed: true
        description: "The displayname of globus group which the user who created this entity is a member of"
        before_create_trigger: set_group_name #same as group_uuid, except set group_name
      previous_revision_uuid:
        type: string
        transient: true
        immutable: true
        indexed: true
        description: "The uuid of previous revision dataset"
        before_property_create_validators:
          - validate_if_revision_is_unique
        after_create_trigger: link_to_previous_revision
        on_read_trigger: get_previous_revision_uuid
        on_index_trigger: get_previous_revision_uuid
      next_revision_uuid:
        type: string
        generated: true
        transient: true
        immutable: true
        indexed: true
        description: "The uuid of next revision dataset"
        on_read_trigger: get_next_revision_uuid
        on_index_trigger: get_next_revision_uuid
      superseded_associated_processed_component_uuids:
        type: list
        # This property gets set via a PUT by entity-api /entities/{{dataset_uuid}} to point to the Multi-Assay Dataset
        # superseding the entity with this attribute, so we can't define it as `immutable: true`.
        # Modifications to an existing attribute are rejected using a validation trigger.
        immutable: false
        indexed: true
        description: "List of uuids of existing Datasets used to construct this Multi-Assay Dataset, when present"
        before_property_create_validators:
          - verify_multi_assay_dataset_components
        before_property_update_validators:
          - verify_multi_assay_dataset_components
      new_associated_multi_assay_uuid:
        type: string
        # This property gets set via a PUT by entity-api /entities/{{dataset_uuid}} to point to the Multi-Assay Dataset
        # superseding the entity with this attribute, so we can't define it as `immutable: true`.
        # Modifications to an existing attribute are rejected using a validation trigger.
        immutable: false
        indexed: true
        description: "The uuid of the Multi-Assay Dataset constructed using this Dataset, when present"
        before_property_create_validators:
          - verify_multi_assay_dataset_components
        before_property_update_validators:
          - verify_multi_assay_dataset_components
      # No like image and metadata files handling for Donor/Sample
      # Dataset has only one thumbnail file
      thumbnail_file:
        generated: true
        indexed: true
        type: json_string
        description: "The dataset thumbnail file detail. Stored in db as a stringfied json, e.g., {'filename': 'thumbnail.jpg', 'file_uuid': 'dadasdasdadda'}"
        # The updated_peripherally tag is a temporary measure to correctly handle any attributes
        # which are potentially updated by multiple triggers
        updated_peripherally: true
      thumbnail_file_to_add:
        type: json_string 
        transient: true
        exposed: false
        indexed: false
        description: 'Just a temporary file id. Provide as a json object with an temp_file_id like {"temp_file_id":"dzevgd6xjs4d5grmcp4n"}'
        before_create_trigger: commit_thumbnail_file
        # This before_update_trigger with the same commit process can be used by ingest-api to update the dataset via PUT call
        before_update_trigger: commit_thumbnail_file
        # The updated_peripherally tag is a temporary measure to correctly handle any attributes
        # which are potentially updated by multiple triggers
        updated_peripherally: true
      thumbnail_file_to_remove:
        # This is only valid on update via a PUT request
        type: string 
        transient: true
        exposed: false
        indexed: false
        description: 'The thumbnail image file previously uploaded to delete. Provide as a string of the file_uuid like: "232934234234234234234270c0ea6c51d604a850558ef2247d0b4"'
        before_update_trigger: delete_thumbnail_file
        # The updated_peripherally tag is a temporary measure to correctly handle any attributes
        # which are potentially updated by multiple triggers
        updated_peripherally: true
      retraction_reason:
        type: string
        indexed: true
        before_property_update_validators:
          - validate_if_retraction_permitted
          - validate_sub_status_provided
        description: 'Information recorded about why a the dataset was retracted.'
      sub_status:
        type: string
        indexed: true
        before_property_update_validators:
          - validate_if_retraction_permitted
          - validate_retraction_reason_provided
          - validate_retracted_dataset_sub_status_value
        description: 'A sub-status provided to further define the status. The only current allowable value is "Retracted"'
      provider_info:
        type: string
        indexed: true
        description: 'Information recorded about the data provider before an analysis pipeline is run on the data.'
      dbgap_sra_experiment_url:
        type: string
        indexed: true
        description: 'A URL linking the dataset to the associated uploaded data at dbGaP.'
      dbgap_study_url:
        type: string
        indexed: true
        description: 'A URL linking the dataset to the particular study on dbGap it belongs to'
      status_history:
        type: list
        description: "A list of all status change events. Each entry in the list is a dictionary containing the change_timestamp, changed_by_email, previous_status, new_status"
        generated: true
        indexed: false
        immutable: true
      assigned_to_group_name:
        type: string
        indexed: false
        description: The group who is responsible for the next step in the ingest process
        before_property_create_validators:
          - validate_in_admin_group
          - validate_group_name
        before_property_update_validators:
          - validate_in_admin_group
          - validate_group_name
      ingest_task:
        type: string
        indexed: false
        description: A description of the next task in the ingest process
        before_property_create_validators:
          - validate_in_admin_group
        before_property_update_validators:
          - validate_in_admin_group

  ############################################# Publication #############################################
  Publication:
    # This superclass property is optional
    superclass: Dataset
    # Only allowed applications can create new Publication via POST
    before_entity_create_validator: validate_application_header_before_entity_create
    before_entity_update_validator:
      # Halt modification of entities which are "locked", such as a Dataset with status == 'Published'
      - validate_entity_not_locked_before_update
    # Publications can be either derivation source or target
    derivation:
      source: true
      target: true
    properties:
      <<: *shared_dataset_properties
      title:
        type: string
        indexed: true
        description: "The title of the publication."
        required_on_create: true # Only required for create via POST, not update via PUT
      creation_action:
        type: string
        transient: true
        generated: true
        immutable: true
        indexed: true
        on_read_trigger: get_creation_action_activity
        on_index_trigger: get_creation_action_activity
        description: "The activity that was performed."
      dataset_type:
        before_create_trigger: set_publication_dataset_type
        type: string
        generated: true
        immutable: true
        indexed: true
        description: "The assay types of this Dataset. Valid values are from UBKG are queried by schema_manager.get_valueset_dataset_type() using the Ontology API."
      publication_date:
        type: string
        indexed: true
        description: 'The date of publication'
        required_on_create: true # Only required for create via POST, not update via PUT
        before_property_create_validators:
          - validate_publication_date
        before_create_trigger: set_publication_date
        before_property_update_validators:
          - validate_publication_date
        before_update_trigger: set_publication_date
      publication_doi:
        type: string
        indexed: true
        description: 'The doi of the publication. (##.####/[alpha-numeric-string])'
      publication_url:
        type: string
        indexed: true
        description: 'The URL at the publishers server for print/pre-print (http(s)://[alpha-numeric-string].[alpha-numeric-string].[...])'
        required_on_create: true # Only required for create via POST, not update via PUT
      publication_venue:
        type: string
        indexed: true
        description: 'The venue of the publication, journal, conference, preprint server, etc...'
        required_on_create: true # Only required for create via POST, not update via PUT
      volume:
        type: integer
        indexed: true
        description: 'The volume number of a journal that it was published in.'
      issue:
        type: integer
        indexed: true
        description: 'The issue number of the journal that it was published in.'
      pages_or_article_num:
        type: string
        indexed: true
        description: 'The pages or the article number in the publication journal e.g., "23", "23-49", "e1003424".'
      publication_status:
        type: boolean
        indexed: true
        required_on_create: true
        description: 'A boolean representing if the publication has been published yet or not.  (Published in the target/venue journal/proceeding/etc.. NOT published in the sense of Dataset publication)'
      omap_doi:
        type: string
        indexed: true
        description: 'A DOI pointing to an Organ Mapping Antibody Panel relevant to this publication'
      associated_collection:
        type: json_string # dict
        generated: true
        transient: true
        indexed: true
        on_read_trigger: get_publication_associated_collection
        on_index_trigger: get_publication_associated_collection
        description: "The collection from which this publication uses data"
      associated_collection_uuid:
        type: string
        transient: true
        exposed: false
        indexed: false
        description: "The uuid of the associated collection for a given publication"
        after_create_trigger: link_publication_to_associated_collection
        after_update_trigger: link_publication_to_associated_collection
      assigned_to_group_name: null # This assigned_to_group_name is Dataset specific, Publication doesn't have it
      ingest_task: null # This ingest_task is Dataset specific, Publication doesn't have it
      new_associated_multi_assay_uuid: null # Dataset-only attribute of Multi-Assay Dataset relationships
      superseded_associated_processed_component_uuids: null # Dataset-only attribute of Multi-Assay Dataset relationships

  ############################################# Donor #############################################
  Donor:
    # Donor can be derivation source but not target
    derivation:
      source: false
      target: true
    excluded_properties_from_public_response:
      - lab_donor_id
      - submission_id
      - label
    before_entity_update_validator:
      # Halt modification of entities which are "locked", such as a Dataset with status == 'Published'
      - validate_entity_not_locked_before_update
    properties:
      <<: *shared_properties
      <<: *shared_entity_properties
      <<: *doi_properties
      description:
        type: string
        indexed: true
        description: "Free text description of the donor"
      # A Donor is public only if any dataset below it in the provenance hierarchy is published
      # (i.e. Dataset.status == "Published")
      data_access_level:
        type: string
        generated: true
        indexed: true
        description: "One of the values: public, consortium"
        before_create_trigger: set_data_access_level
      image_files:
        # This property has no before_create_trigger method, but will be created via `image_files_to_add` and updated via `image_files_to_delete`
        type: list 
        generated: true # Disallow direct user input from create via POST, but can be used for update via PUT
        indexed: false
        description: "List of uploaded image files and descriptions of the files. Stored in db as a stringfied json array."
        # Only file descriptions are allowed to be updated directly using this property
        # Adding or deleting files must via the other two properties
        before_update_trigger: update_file_descriptions
        # The updated_peripherally tag is a temporary measure to correctly handle any attributes
        # which are potentially updated by multiple triggers
        updated_peripherally: true
      image_files_to_add:
        type: list 
        transient: true
        exposed: false
        indexed: false
        description: 'List of temporary file ids with an optional description. Provide as a json array with an temp_file_id and description attribute for each element like {"files": [{"temp_file_id":"dzevgd6xjs4d5grmcp4n","description":"This is image file one"},{"temp_file_id":"yrahjadfhadf","description":"This is image file two"}]}'
        before_create_trigger: commit_image_files
        before_update_trigger: commit_image_files
        # The updated_peripherally tag is a temporary measure to correctly handle any attributes
        # which are potentially updated by multiple triggers
        updated_peripherally: true
      image_files_to_remove:
        # This is only valid on update via a PUT request
        type: list 
        transient: true
        exposed: false
        indexed: false
        description: 'List of image files previously uploaded to delete. Provide as a json array of the file_uuids of the file like: ["232934234234234234234270c0ea6c51d604a850558ef2247d0b4", "230948203482234234234a57bfe9c056d08a0f8e6cd612baa3bfa"]'
        before_update_trigger: delete_image_files
        # The updated_peripherally tag is a temporary measure to correctly handle any attributes
        # which are potentially updated by multiple triggers
        updated_peripherally: true
      metadata:
        type: json_string # dict
        indexed: true
        description: "Donor metadata coded as described here: <get donor metadata doc from Chuck>"
      protocol_url:
        type: string
        indexed: true
        required_on_create: true # Only required for create via POST, not update via PUT
        description: "The protocols.io doi url pointing the protocol describing the donor selection, inclusion/exclusion criteria"
      lab_donor_id:
        type: string
        indexed: true
        description: "A lab specific identifier for the donor."
      # No submission_id for Dataset, Collection, and Upload
      # Only Donor and Sample have this property
      submission_id:
        type: string
        immutable: true
        generated: true
        indexed: true
        description: "The hubmap internal id with embedded semantic information e.g.: VAN0003. This id is generated at creation time which tracks the lab, donor, organ and sample hierarchy per the following: https://docs.google.com/document/d/1DjHgmqWF1VA5-3mfzLFNfabbzmc8KLSG9xWx1DDLlzo/edit?usp=sharing"
        before_create_trigger: set_submission_id
      # A user who is a member of multiple groups HAS to send in the group_uuid 
      group_uuid:
        type: string
        immutable: true
        indexed: true
        description: "The uuid of globus group which the user who created this entity is a member of.  This is required on Create/POST if the user creating the Donor is a member of more than one write group.  This property cannot be set via PUT (only on Create/POST)."
        before_create_trigger: set_group_uuid #method that, if group_uuid is not already set looks for membership in a single "data provider" group and sets to that. Otherwise if not set and no single "provider group" membership throws error.  This field is also used to link (Neo4j relationship) to the correct Lab node on creation.
        # Note: link_donor_to_lab() will always delete all the old linkages first
        after_create_trigger: link_donor_to_lab
      # Must set in neo4j
      group_name:
        type: string
        generated: true
        immutable: true
        indexed: true
        description: "The displayname of globus group which the user who created this entity is a member of"
        before_create_trigger: set_group_name
        #todo: map to provenance_group_name in ES - ? In the current version it's not mapped
      portal_metadata_upload_files:
        type: json_string
        indexed: true
        description: "list of relative paths to metadata files"
        #todo: migrate to new attribute (set, maybe) representing a single uploaded sample_metadata.tsv file using the now file upload methods.  Leave this in place for now, but add new attribute(s) to support new methon.  Need to do a design session.
      next_identifier:
        type: string
        immutable: true
        exposed: false
        indexed: false
        description: "Internal count of child samples, used for generating the submission id of the children. This field is not exposed via the api."
        #todo: add new yaml attribute, exposed that defaults to true. When false the attribute is not returned in ws response
      label:
        type: string
        indexed: true
        description: "Lab provided, de-identified name for the donor"
      open_consent:
        type: boolean
        indexed: false
        description: "True if the donor was consented to allow public, open access to any data, identified or not, derived from the donor's tissue.  Otherwise false."


  ############################################# Sample #############################################
  Sample:
    # Sample can be either derivation source or target
    derivation:
      source: true
      target: true
    excluded_properties_from_public_response:
      - lab_tissue_sample_id
      - submission_id
      - metadata:
        - lab_id
      - direct_ancestor:
        # Donor ancestors of a Sample must have these fields removed
        - lab_donor_id
        - label
        # Sample ancestors of a Sample must have these fields removed
        - lab_tissue_sample_id
        - metadata:
          - lab_id
        # Both Sample and Donor ancestors of a Sample must have these fields removed
        - submission_id
    before_entity_update_validator:
      # Halt modification of entities which are "locked", such as a Dataset with status == 'Published'
      - validate_entity_not_locked_before_update
    properties:
      <<: *shared_properties
      <<: *shared_entity_properties
      <<: *doi_properties
      description:
        type: string
        indexed: true
        description: "Free text description of the sample"
      # A Sample is public only if any dataset below it in the provenance hierarchy is published
      # (i.e. Dataset.status == "Published")
      data_access_level:
        type: string
        generated: true
        indexed: true
        description: "One of the values: public, consortium, protected. Only Dataset may have protected value"
        before_create_trigger: set_data_access_level
      sample_category:
        type: string
        indexed: true
        immutable: true
        required_on_create: true # Only require for cerate via POST, not update via PUT
        description: "A code representing the type of specimen. Must be an organ, block, section, or suspension"
        before_property_create_validators:
          - validate_sample_category
        before_property_update_validators:
          - validate_sample_category
      portal_metadata_upload_files:
        type: json_string
        indexed: true
        description: "A list of relative paths to metadata files"
        #todo: migrate to new attribute (set, maybe) representing a single uploaded sample_metadata.tsv file using the now file upload methods.  Leave this in place for now, but add new attribute(s) to support new methon.  Need to do a design session.
      protocol_url:
        type: string
        indexed: true
        required_on_create: true
        description: "The protocols.io doi url pointing the protocol under wich the sample was obtained and/or prepared."
      image_file_metadata:
        #todo: migrate to new attribute set as above portal_metadata_upload files
        type: json_string
        indexed: true
        description: "A list of uploaded image files and descriptions of the files."
      # A user who is a member of multiple groups HAS to send in the group_uuid 
      group_uuid:
        type: string
        immutable: true
        indexed: true
        description: "The uuid of globus group which the user who created this entity is a member of.  This is required on Create/POST if the user creating the Donor is a member of more than one write group.  This property cannot be set via PUT (only on Create/POST)."
        before_create_trigger: set_group_uuid #method that, if group_uuid is not already set looks for membership in a single "data provider" group and sets to that. Otherwise if not set and no single "provider group" membership throws error.  This field is also used to link (Neo4j relationship) to the correct Lab node on creation.
      # Must set in neo4j
      group_name:
        type: string
        generated: true
        immutable: true
        indexed: true
        description: "The displayname of globus group which the user who created this entity is a member of"
        before_create_trigger: set_group_name
      organ:
        type: string
        indexed: true
        immutable: true
        description: "Organ code specifier, only set if sample_category == organ"
      organ_other:
        type: string
        indexed: true
        description: The organ type provided by the user if "other" organ type is selected
      direct_ancestor_uuid:
        type: string
        required_on_create: true # Creating the hubmap ids via uuid-api requires `parent_ids`
        transient: true
        exposed: false
        indexed: false
        description: "The uuid of source entity from which this new entity is derived from. Used on creation or edit to create an action and relationship to the ancestor.  The direct ancestor must be a Donor or Sample.  If the direct ancestor is a Donor, the sample must be of type organ."
        # Note: link_sample_to_direct_ancestor() will always delete all the old linkages first
        after_create_trigger: link_sample_to_direct_ancestor
        after_update_trigger: link_sample_to_direct_ancestor
        before_property_update_validators:
          - validate_id_not_in_direct_ancestor
      direct_ancestor:
        type: json_string # dict
        generated: true
        immutable: true
        transient: true
        indexed: false
        on_read_trigger: get_sample_direct_ancestor
        # No on_index_trigger to include direct_ancestor in the OpenSearch document for a Sample
        description: "The entitiy directly above this sample in the provenance graph (direct parent)."
      # No submission_id for Dataset, Collection, and Upload
      # Only Donor and Sample have this property
      submission_id:
        type: string
        generated: true
        immutable: true
        indexed: true
        description: "The hubmap internal id with embedded semantic information e.g.: VAN0003-LK-1-10.  This id is generated at creation time which tracks the lab, donor, organ and sample hierarchy per the following: https://docs.google.com/document/d/1DjHgmqWF1VA5-3mfzLFNfabbzmc8KLSG9xWx1DDLlzo/edit?usp=sharing"
        before_create_trigger: set_submission_id
      lab_tissue_sample_id:
        type: string
        indexed: true
        description: "Lab specific id for the sample."
      next_identifier:
        type: string
        immutable: true
        exposed: false # For internal use only
        indexed: false
        description: "Internal count of child samples, used for generating the submission id of the children. This field is not exposed via the api."
        #todo: add new yaml attribute, exposed that defaults to true.  When false the attribute is not returned in ws response
      metadata:
        type: json_string
        indexed: true
        description: "The sample specific metadata derived from the uploaded sample_metadata.tsv file"
      rui_location:
        type: json_string
        indexed: true
        description: "The sample location and orientation in the ancestor organ as specified in the RUI tool."
      rui_exemption:
        type: boolean
        indexed: true
        description: "Identifies Sample Blocks that are exempt from spatial registration via RUI."
      visit:
        type: string
        indexed: true
        description: "The visit id for the donor/patient when the sample was obtained."
      image_files:
        # This property has no before_create_trigger method, but will be created via `image_files_to_add` and updated via `image_files_to_remove`
        type: list 
        generated: true # Disallow direct user input from create via POST, but can be used for update via PUT
        indexed: false
        description: "List of uploaded image files and descriptions of the files. Stored in db as a stringfied json array."
        # Only file descriptions are allowed to be updated directly using this property
        # Adding or deleting files must via the other two properties
        before_update_trigger: update_file_descriptions
        # The updated_peripherally tag is a temporary measure to correctly handle any attributes
        # which are potentially updated by multiple triggers
        updated_peripherally: true
      image_files_to_add:
        type: list 
        transient: true
        exposed: false
        indexed: false
        description: 'List of temporary file ids with an optional description. Provide as a json array with an temp_file_id and description attribute for each element like {"files": [{"temp_file_id":"dzevgd6xjs4d5grmcp4n","description":"This is image file one"},{"temp_file_id":"yrahjadfhadf","description":"This is image file two"}]}'
        before_create_trigger: commit_image_files
        before_update_trigger: commit_image_files
        # The updated_peripherally tag is a temporary measure to correctly handle any attributes
        # which are potentially updated by multiple triggers
        updated_peripherally: true
      image_files_to_remove:
        # This is only valid on update via a PUT request
        type: list 
        transient: true
        exposed: false
        indexed: false
        description: 'List of image files previously uploaded to delete. Provide as a json array of the file_uuids of the file like: ["232934234234234234234270c0ea6c51d604a850558ef2247d0b4", "230948203482234234234a57bfe9c056d08a0f8e6cd612baa3bfa"]'
        before_update_trigger: delete_image_files
        # The updated_peripherally tag is a temporary measure to correctly handle any attributes
        # which are potentially updated by multiple triggers
        updated_peripherally: true
      metadata_files:
        # This property has no before_create_trigger method, but will be created via `metadata_files_to_add` and updated via `metadata_files_to_remove`
        type: list 
        generated: true # Disallow direct user input from create via POST, but can be used for update via PUT
        indexed: false
        description: "List of uploaded image files and descriptions of the files. Stored in db as a stringfied json array."
        # Only file descriptions are allowed to be updated directly using this property
        # Adding or deleting files must via the other two properties
        before_update_trigger: update_file_descriptions
        # The updated_peripherally tag is a temporary measure to correctly handle any attributes
        # which are potentially updated by multiple triggers
        updated_peripherally: true
      metadata_files_to_add:
        type: list 
        transient: true
        exposed: false
        indexed: false
        description: 'List of temporary file ids with an optional description. Provide as a json array with an temp_file_id and description attribute for each element like {"files": [{"temp_file_id":"dzevgd6xjs4d5grmcp4n","description":"This is image file one"},{"temp_file_id":"yrahjadfhadf","description":"This is image file two"}]}'
        before_create_trigger: commit_metadata_files
        before_update_trigger: commit_metadata_files
        # The updated_peripherally tag is a temporary measure to correctly handle any attributes
        # which are potentially updated by multiple triggers
        updated_peripherally: true
      metadata_files_to_remove:
        # This is only valid on update via a PUT request
        type: list 
        transient: true
        exposed: false
        indexed: false
        description: 'List of image files previously uploaded to delete. Provide as a json array of the file_uuids of the file like: ["232934234234234234234270c0ea6c51d604a850558ef2247d0b4", "230948203482234234234a57bfe9c056d08a0f8e6cd612baa3bfa"]'
        before_update_trigger: delete_metadata_files
        # The updated_peripherally tag is a temporary measure to correctly handle any attributes
        # which are potentially updated by multiple triggers
        updated_peripherally: true
      thumbnail_file:
        generated: true
        indexed: true
        type: json_string
        description: "The thumbnail file detail. Stored in db as a stringfied json, e.g., {'filename': 'thumbnail.jpg', 'file_uuid': 'dadasdasdadda'}"
        # The updated_peripherally tag is a temporary measure to correctly handle any attributes
        # which are potentially updated by multiple triggers
        updated_peripherally: true
      thumbnail_file_to_add:
        type: json_string 
        transient: true
        exposed: false
        indexed: false
        description: 'Just a temporary file id. Provide as a json object with an temp_file_id like {"temp_file_id":"dzevgd6xjs4d5grmcp4n"}'
        before_create_trigger: commit_thumbnail_file
        # This before_update_trigger with the same commit process can be used by ingest-api to update the dataset via PUT call
        before_update_trigger: commit_thumbnail_file
        # The updated_peripherally tag is a temporary measure to correctly handle any attributes
        # which are potentially updated by multiple triggers
        updated_peripherally: true
      thumbnail_file_to_remove:
        # This is only valid on update via a PUT request
        type: string 
        transient: true
        exposed: false
        indexed: false
        description: 'The thumbnail image file previously uploaded to delete. Provide as a string of the file_uuid like: "232934234234234234234270c0ea6c51d604a850558ef2247d0b4"'
        before_update_trigger: delete_thumbnail_file
        # The updated_peripherally tag is a temporary measure to correctly handle any attributes
        # which are potentially updated by multiple triggers
        updated_peripherally: true


  ############################################# Upload #############################################
  # Eventually we might allow folks to bulk register Samples or Donors through the same mechanism.
  Upload:
    # Only allowed applications can create new Upload via POST
    before_entity_create_validator: validate_application_header_before_entity_create
    # No before_entity_update_validator needed for Upload because the entity is
    # always considered "non-public", and therefore not blocked from update/PUT.
    #
    # Upload requires a Lab entity as an ancestor, and has no allowed descendants
    derivation:
      source: false
      target: false # Set to false since the schema doesn't handle Lab currently
    properties:
      <<: *shared_properties
      <<: *shared_entity_properties
      description:
        type: string
        indexed: true
        description: "Free text description of the data being submitted."
      title:
        type: string
        indexed: true
        required_on_create: true
        description: "Title of the datasets, a sentence or less"
      status:
        before_property_update_validators: 
          - validate_application_header_before_property_update
          - validate_upload_status_value
          - validate_status_changed
        type: string
        generated: true
        indexed: true
        description: "One of: New|Valid|Invalid|Error|Reorganized|Processing|Submitted|Incomplete"
        # Trigger method will set the status to "New" on create
        before_create_trigger: set_upload_status_new
        after_create_trigger: set_status_history
        after_update_trigger: set_status_history
      validation_message:
        type: string
        indexed: false
        description: A message from the validation tools describing what is invalid with the upload.
      group_uuid:
        type: string
        immutable: true
        indexed: true
        description: "The uuid of globus group which the user who created this entity is a member of.  This is required on Create/POST if the user creating the Donor is a member of more than one write group.  This property cannot be set via PUT (only on Create/POST)."
        before_create_trigger: set_group_uuid #method that, if group_uuid is not already set looks for membership in a single "data provider" group and sets to that. Otherwise if not set and no single "provider group" membership throws error.  This field is also used to link (Neo4j relationship) to the correct Lab node on creation.
        # Note: link_upload_to_lab() will always delete all the old linkages first
        after_create_trigger: link_upload_to_lab
      # Must set in neo4j
      group_name:
        type: string
        generated: true
        immutable: true
        indexed: true
        description: "The displayname of globus group which the user who created this entity is a member of"
        before_create_trigger: set_group_name
      #transient attribute used to pass in a list of dataset ids to be linked to the Submission node
      #datasets will be added only via PUT/update, NEVER on POST/create
      dataset_uuids_to_link:
        type: list 
        before_property_update_validators:
          - validate_no_duplicates_in_list
        generated: true # Disallow user input from request json when being created
        indexed: false
        transient: true
        exposed: false
        description: 'List of datasets to add to the Upload. Provide as a json array of the dataset uuids like: ["232934234234234234234270c0ea6c51d604a850558ef2247d0b4", "230948203482234234234a57bfe9c056d08a0f8e6cd612baa3bfa"]'
        # Use after_update_trigger instead of before_update_trigger since we are not updating this property
        after_update_trigger: link_datasets_to_upload 
      #transient attribute used to pass in a list of dataset ids of datasets to be removed from a DataSubmisison
      dataset_uuids_to_unlink:
        type: list 
        before_property_update_validators:
          - validate_no_duplicates_in_list
        generated: true # Disallow user input from request json when being created
        indexed: false
        transient: true
        exposed: false
        description: 'List of datasets to remove from a Upload. Provide as a json array of the dataset uuids like: ["232934234234234234234270c0ea6c51d604a850558ef2247d0b4", "230948203482234234234a57bfe9c056d08a0f8e6cd612baa3bfa"]'
        # Use after_update_trigger instead of before_update_trigger since we are not updating this property
        after_update_trigger: unlink_datasets_from_upload 
      datasets:
        type: list
        generated: true # Disallow user input from request json when being created
        indexed: true
        transient: true
        description: "The datasets that are contained in this Upload."
        # A few time-consuming properties (with read triggers) of each dataset are excluded
        on_read_trigger: get_upload_datasets
        on_index_trigger: get_upload_datasets
      status_history:
        type: list
        description: "A list of all status change events. Each entry in the list is a dictionary containing the change_timestamp, changed_by_email, previous_status, new_status"
        generated: true
        indexed: false
        immutable: true
      assigned_to_group_name:
        type: string
        description: The group who is responsible for the next step in the ingest process
        indexed: false
        before_property_create_validators:
          - validate_in_admin_group
          - validate_group_name
        before_property_update_validators:
          - validate_in_admin_group
          - validate_group_name
      ingest_task:
        type: string
        description: A description of the next task in the ingest process
        indexed: false
        before_property_create_validators:
          - validate_in_admin_group
        before_property_update_validators:
          - validate_in_admin_group
      intended_dataset_type:
        type: string
        required_on_create: true
        indexed: true
        description: The dataset type of the intended datasets that will be uploaded as part of the Upload.
        before_property_create_validators:
          - validate_intended_dataset_type
        before_property_update_validators:
          - validate_intended_dataset_type
      intended_organ: 
        required_on_create: true
        type: string
        indexed: true
        description: The organ code representing the organ type that the data contained in the upload will be registered/associated with.
      anticipated_complete_upload_month:
        type: string
        indexed: true
        description: The specific month the Upload is anticipated to have all required data uploaded, in the format YYYY-MM.
        required_on_create: false
        before_property_create_validators:
          - validate_anticipated_complete_date
        before_property_update_validators:
          - validate_anticipated_complete_date
      anticipated_dataset_count:
        type: integer
        indexed: true
        description: The total number of datasets that this Upload will eventually contain.
        required_on_create: false
        before_property_create_validators:
          - validate_anticipated_dataset_count
        before_property_update_validators:
          - validate_anticipated_dataset_count

  ############################################# EPICollection #############################################
  Epicollection:
    # This superclass property is optional
    superclass: Collection
    # EPICollection can not be derivation source but not target
    derivation:
      source: false
      target: false
    before_entity_update_validator:
      # Halt modification of entities which are "locked", such as a Dataset with status == 'Published'
      - validate_entity_not_locked_before_update
    properties:
      <<: *shared_collection_properties