diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index d30b5d6b1e..a0ec41740d 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -23,16 +23,16 @@ jobs: - uses: actions/checkout@v4 - name: Install Forge - uses: input-output-hk/catalyst-forge/actions/install@ci/v1.5.3 + uses: input-output-hk/catalyst-forge/actions/install@ci/v1.7.1 with: - version: 0.8.0 + version: 0.14.0 if: always() - name: Setup CI - uses: input-output-hk/catalyst-forge/actions/setup@ci/v1.5.3 + uses: input-output-hk/catalyst-forge/actions/setup@ci/v1.7.1 - name: Run tests - uses: input-output-hk/catalyst-forge/actions/run@ci/v1.5.3 + uses: input-output-hk/catalyst-forge/actions/run@ci/v1.7.1 if: always() continue-on-error: true with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a41b096b4f..ff7e5d69d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,6 @@ permissions: jobs: ci: - uses: input-output-hk/catalyst-forge/.github/workflows/ci.yml@ci/v1.5.3 + uses: input-output-hk/catalyst-forge/.github/workflows/ci.yml@ci/v1.7.1 with: - forge_version: 0.8.1 \ No newline at end of file + forge_version: 0.14.0 \ No newline at end of file diff --git a/Earthfile b/Earthfile index 6a3fe099a2..c7b327730a 100644 --- a/Earthfile +++ b/Earthfile @@ -92,7 +92,7 @@ all: END # Build and tag all Docker images - BUILD ./containers/event-db-migrations+publish --tag=$tag --registry=$registry_final + BUILD ./containers/event-db-migrations+package --tag=$tag --registry=$registry_final # Build crate images from the workspace BUILD ./src/jormungandr/jormungandr+docker --tag=$tag --registry=$registry_final @@ -101,7 +101,7 @@ all: BUILD ./src/voting-tools-rs+docker --tag=$tag --registry=$registry_final BUILD ./src/cat-data-service+publish --tag=$tag --registry=$registry_final - BUILD ./services/voting-node+publish --tag=$tag --registry=$registry_final + BUILD ./services/voting-node+package --tag=$tag --registry=$registry_final BUILD ./utilities/ideascale-importer+docker --tag=$tag --registry=$registry_final all-with-tags: @@ -147,9 +147,9 @@ tag-workspace: local: LOCALLY - BUILD ./containers/event-db-migrations+publish + BUILD ./containers/event-db-migrations+package BUILD ./src/cat-data-service+publish - BUILD ./services/voting-node+publish + BUILD ./services/voting-node+package RUN mkdir -p ./local COPY ./containers/dev-local+build/docker-compose.yml ./local/ diff --git a/blueprint.cue b/blueprint.cue new file mode 100644 index 0000000000..608f5fbdee --- /dev/null +++ b/blueprint.cue @@ -0,0 +1,60 @@ +version: "1.0" +global: { + ci: { + local: [ + "^check(-.*)?$", + "^build(-.*)?$", + "^package(-.*)?$", + "^test(-.*)?$", + ] + registries: [ + ci.providers.aws.ecr.registry, + ] + providers: { + aws: { + region: "eu-central-1" + ecr: registry: "332405224602.dkr.ecr.eu-central-1.amazonaws.com" + role: "arn:aws:iam::332405224602:role/ci" + } + + docker: credentials: { + provider: "aws" + path: "global/ci/docker" + } + + git: credentials: { + provider: "aws" + path: "global/ci/deploy" + } + + earthly: { + credentials: { + provider: "aws" + path: "global/ci/earthly" + } + org: "Catalyst" + satellite: "ci" + version: "0.8.15" + } + + github: registry: "ghcr.io" + + kcl: { + install: true + version: "v0.11.0" + } + } + secrets: [ + { + name: "GITHUB_TOKEN" + optional: true + provider: "env" + path: "GITHUB_TOKEN" + }, + ] + } + repo: { + defaultBranch: "main" + name: "input-output-hk/catalyst-core" + } +} diff --git a/containers/event-db-migrations/Earthfile b/containers/event-db-migrations/Earthfile index d690281084..dcf43d07a5 100644 --- a/containers/event-db-migrations/Earthfile +++ b/containers/event-db-migrations/Earthfile @@ -12,7 +12,7 @@ build: SAVE ARTIFACT ./bin/refinery refinery SAVE IMAGE --cache-hint -publish: +package: FROM debian:stable-slim ARG tag="latest" ARG data="historic" @@ -63,3 +63,11 @@ publish: # --load test:latest=+docker # RUN docker run test:latest # END + +docker: + FROM +package + + ARG container='migrations' + ARG tag="latest" + + SAVE IMAGE ${container}:${tag} \ No newline at end of file diff --git a/containers/event-db-migrations/blueprint.cue b/containers/event-db-migrations/blueprint.cue new file mode 100644 index 0000000000..c0c84f7200 --- /dev/null +++ b/containers/event-db-migrations/blueprint.cue @@ -0,0 +1,15 @@ +version: "1.0.0" +project: { + name: "event-db-migrations" + release: { + docker: { + on: { + merge: {} + tag: {} + } + config: { + tag: _ @forge(name="GIT_HASH_OR_TAG") + } + } + } +} diff --git a/services/voting-node/Earthfile b/services/voting-node/Earthfile index b02fe63de6..0b1d8c9fb5 100644 --- a/services/voting-node/Earthfile +++ b/services/voting-node/Earthfile @@ -43,7 +43,7 @@ deps: # Install package dependencies without the voting_node package RUN poetry install --only main --no-root RUN poetry self add poetry-plugin-export - + # Copy the voting_node source code COPY --dir voting_node README.md ./ @@ -91,7 +91,7 @@ pdoc: SAVE ARTIFACT /doc # Docker image built for distribution and use in production. -publish: +package: FROM python:3.11-slim-bullseye ARG tag="latest" @@ -165,3 +165,11 @@ integration-test: --service leader0 RUN docker-compose run leader0 echo "<<>>" END + +docker: + FROM +package + + ARG container='voting-node' + ARG tag="latest" + + SAVE IMAGE ${container}:${tag} \ No newline at end of file diff --git a/services/voting-node/blueprint.cue b/services/voting-node/blueprint.cue new file mode 100644 index 0000000000..c51438eda9 --- /dev/null +++ b/services/voting-node/blueprint.cue @@ -0,0 +1,15 @@ +version: "1.0.0" +project: { + name: "voting-node" + release: { + docker: { + on: { + merge: {} + tag: {} + } + config: { + tag: _ @forge(name="GIT_HASH_OR_TAG") + } + } + } +} diff --git a/src/event-db/migrations/V11__reviews.sql b/src/event-db/migrations/V11__reviews.sql deleted file mode 100644 index 0ae8cec2af..0000000000 --- a/src/event-db/migrations/V11__reviews.sql +++ /dev/null @@ -1,106 +0,0 @@ --- Catalyst Event Database - --- Subscription Table --- This table stores the subscriptions for users in the reviews module - -CREATE TABLE subscription -( - row_id SERIAL PRIMARY KEY, - user_id INTEGER NOT NULL, - event_id INTEGER NOT NULL, - role INTEGER NOT NULL, - status INTEGER NOT NULL, - extra JSONB NULL, - - FOREIGN KEY (user_id) REFERENCES catalyst_user(row_id) ON DELETE CASCADE, - FOREIGN KEY (event_id) REFERENCES event(row_id) ON DELETE CASCADE -); - -CREATE UNIQUE INDEX subscription_idx ON subscription(user_id,event_id,role); - -COMMENT ON TABLE subscription IS ' -A subscription describes the possible roles of a single user for an event. -A user can participate in multiple events active as multiple roles. -For this reason this dedicated table tracks all active roles for a user related -to a specific event and stores meta information about them. -Some of these subscriptions will be automatically created by the system. -The presence/status of subscriptions will determine user capabilities in the app. -'; - -COMMENT ON COLUMN subscription.user_id IS 'The user ID this subscription belongs to.'; -COMMENT ON COLUMN subscription.event_id IS 'The event ID this subscription belongs to.'; - -COMMENT ON COLUMN subscription.role IS -'This field describes the role of the user for this subscription. -Possible values: -0: For LV0 Community Reviewers -1: For LV1 Community Reviewers -2: For LV2 Moderators'; - -COMMENT ON COLUMN subscription.extra IS -'This field is used to store all meta information about a subscription. -Specifically: - -anonymous_id: str, -subscription_date: datetime, -preferred_categories: [int], -reward_address: str -'; - - --- Allocation - Defines the relationship between users and proposals or proposals_reviews --- to describe: --- - the ownership of a proposal. --- - the allocation of the reviews that needs to be done. --- - the allocation of moderations that needs to be done. - -CREATE TABLE allocation ( - row_id SERIAL PRIMARY KEY, - proposal_id INTEGER NULL, - review_id INTEGER NULL, - user_id INTEGER NOT NULL, - type INTEGER NOT NULL, - - FOREIGN KEY (proposal_id) REFERENCES proposal(row_id) ON DELETE CASCADE, - FOREIGN KEY (review_id) REFERENCES proposal_review(row_id) ON DELETE CASCADE, - FOREIGN KEY (user_id) REFERENCES catalyst_user(row_id) ON DELETE CASCADE -); - - -COMMENT ON TABLE allocation IS 'The relationship between users and proposals or proposals_reviews.'; -COMMENT ON COLUMN allocation.row_id IS 'Synthetic ID of this relationship.'; -COMMENT ON COLUMN allocation.proposal_id IS 'The proposal ID the relationship belongs to.'; -COMMENT ON COLUMN allocation.review_id IS 'The review ID the relationship belongs to.'; -COMMENT ON COLUMN allocation.user_id IS 'The user ID the relationship belongs to.'; -COMMENT ON COLUMN allocation.type IS 'The type of relationship stored. -Possible values: -0: proposal ownership relation. proposal_id and user_id are required -1: proposal allocated for review. proposal_id and user_id are required -2: review allocated for moderation. review_id and user_id are required'; - -CREATE INDEX idx_allocation_proposal_type ON allocation(proposal_id, type) WHERE proposal_id IS NOT NULL; -CREATE INDEX idx_allocation_review_type ON allocation(review_id, type) WHERE review_id IS NOT NULL; -CREATE INDEX idx_allocation_user_type ON allocation(user_id, type); - - --- Moderation - Defines the moderation submitted by users for each proposal_review. - -CREATE TABLE moderation ( - row_id SERIAL PRIMARY KEY, - review_id INTEGER NOT NULL, - user_id INTEGER NOT NULL, - classification INTEGER NOT NULL, - rationale VARCHAR, - UNIQUE (review_id, user_id), - - FOREIGN KEY (review_id) REFERENCES proposal_review(row_id) ON DELETE CASCADE, - FOREIGN KEY (user_id) REFERENCES catalyst_user(row_id) ON DELETE CASCADE -); - - -COMMENT ON TABLE moderation IS 'An individual moderation for a proposal review.'; -COMMENT ON COLUMN moderation.row_id IS 'Synthetic ID of this moderation.'; -COMMENT ON COLUMN moderation.review_id IS 'The review ID the moderation belongs to.'; -COMMENT ON COLUMN moderation.user_id IS 'The user ID the moderation belongs to.'; -COMMENT ON COLUMN moderation.classification IS 'The value used to describe the moderation (e.g. 0: excluded, 1: included).'; -COMMENT ON COLUMN moderation.rationale IS 'The rationale for the given classification.'; \ No newline at end of file diff --git a/src/event-db/migrations/V4__catalyst_id.sql b/src/event-db/migrations/V4__catalyst_id.sql deleted file mode 100644 index 69006c1e4c..0000000000 --- a/src/event-db/migrations/V4__catalyst_id.sql +++ /dev/null @@ -1,45 +0,0 @@ --- Catalyst Event Database - --- Catalyst ID map Table --- This table stores the relationship between a catalyst ID and an email address. - -CREATE TABLE catalyst_id_map -( - row_id SERIAL PRIMARY KEY, - catalyst_id VARCHAR NOT NULL, - email VARCHAR NOT NULL, - username VARCHAR NOT NULL, - signed_email VARCHAR NOT NULL, - status INTEGER NOT NULL, - confirmation_token VARCHAR, - confirmation_requested_at TIMESTAMP, - confirmed_at TIMESTAMP, - - UNIQUE(catalyst_id) -); - -CREATE UNIQUE INDEX catalyst_id_idx ON catalyst_id_map(catalyst_id, email); - -COMMENT ON TABLE catalyst_id_map IS ' -The `catalyst_id_map` table is used to store the map between the CatalystID and email/username for each user. -Because the Catalyst RBAC registration are stored on-chain, and for this reason are publicly accessible, -sensitive information like email address will be stored in a centralized DB, keeping a reference to the CatalystID. -The email address is stored only when its signature corresponds to the CatalystID. -'; -COMMENT ON COLUMN catalyst_id_map.catalyst_id IS ' -It contains the unique part of the CatalystID. -Given a full CatalystID like `id.catalyst://fred@preprod.cardano/FftxFnOrj2qmTuB2oZG2v0YEWJfKvQ9Gg8AgNAhDsKE` -The scheme and the username are omitted and only the unique part `preprod.cardano/FftxFnOrj2qmTuB2oZG2v0YEWJfKvQ9Gg8AgNAhDsKE` -is stored in this field. -'; -COMMENT ON COLUMN catalyst_id_map.email IS 'The email address stored in plaintext.'; -COMMENT ON COLUMN catalyst_id_map.username IS 'The username extracted from the CatalystID stored in plaintext.'; -COMMENT ON COLUMN catalyst_id_map.signed_email IS 'The signed document that includes the email address.'; -COMMENT ON COLUMN catalyst_id_map.status IS ' -Describes the status of an account: -0: Inactive -1: Active -2: Banned.'; -COMMENT ON COLUMN catalyst_id_map.confirmation_token IS 'The token that is generated for the email confirmation.'; -COMMENT ON COLUMN catalyst_id_map.confirmation_requested_at IS 'The timestamp to validate confirmation token validity.'; -COMMENT ON COLUMN catalyst_id_map.confirmed_at IS 'The timestamp of the email confirmation.'; \ No newline at end of file diff --git a/src/event-db/migrations/V6__proposal_tables.sql b/src/event-db/migrations/V4__proposal_tables.sql similarity index 90% rename from src/event-db/migrations/V6__proposal_tables.sql rename to src/event-db/migrations/V4__proposal_tables.sql index a364f53d91..158adb8f80 100644 --- a/src/event-db/migrations/V6__proposal_tables.sql +++ b/src/event-db/migrations/V4__proposal_tables.sql @@ -99,47 +99,52 @@ COMMENT ON COLUMN reviewer_level.event_id IS 'The specific Event ID this review CREATE TABLE proposal_review ( row_id SERIAL PRIMARY KEY, proposal_id INTEGER NOT NULL, - user_id INTEGER NOT NULL, assessor VARCHAR NOT NULL, assessor_level INTEGER, + reward_address TEXT, + -- These fields are deprecated and WILL BE removed in a future migration. + -- They MUST only be used for Vit-SS compatibility. impact_alignment_rating_given INTEGER, impact_alignment_note VARCHAR, feasibility_rating_given INTEGER, feasibility_note VARCHAR, auditability_rating_given INTEGER, auditability_note VARCHAR, - allocated INTEGER, ranking INTEGER, flags JSONB NULL, - FOREIGN KEY (user_id) REFERENCES catalyst_user(row_id) ON DELETE SET NULL, - FOREIGN KEY (proposal_id) REFERENCES proposal(row_id) ON DELETE CASCADE + FOREIGN KEY (proposal_id) REFERENCES proposal(row_id) ON DELETE CASCADE, + FOREIGN KEY (assessor_level) REFERENCES reviewer_level(row_id) ON DELETE CASCADE ); COMMENT ON TABLE proposal_review IS 'All Reviews.'; COMMENT ON COLUMN proposal_review.row_id IS 'Synthetic Unique Key.'; -COMMENT ON COLUMN proposal_review.proposal_id IS 'The Proposal id this review belongs to.'; -COMMENT ON COLUMN proposal_review.user_id IS 'The user id this review belongs to.'; -COMMENT ON COLUMN proposal_review.assessor IS 'Assessors Anonymized ID.'; +COMMENT ON COLUMN proposal_review.proposal_id IS 'The Proposal this review is for.'; +COMMENT ON COLUMN proposal_review.assessor IS 'Assessors Anonymized ID'; COMMENT ON COLUMN proposal_review.assessor_level IS 'Assessors level ID'; +COMMENT ON COLUMN proposal_review.reward_address IS 'Assessors reward address'; COMMENT ON COLUMN proposal_review.impact_alignment_rating_given IS -'The numeric rating assigned to the proposal by the assessor.'; +'The numeric rating assigned to the proposal by the assessor. +DEPRECATED: Only used for Vit-SS compatibility.'; COMMENT ON COLUMN proposal_review.impact_alignment_note IS -'A note about why the impact rating was given.'; +'A note about why the impact rating was given. +DEPRECATED: Only used for Vit-SS compatibility.'; + COMMENT ON COLUMN proposal_review.feasibility_rating_given IS -'The numeric feasibility rating given.'; +'The numeric feasibility rating given. +DEPRECATED: Only used for Vit-SS compatibility.'; COMMENT ON COLUMN proposal_review.feasibility_note IS -'A note about why the feasibility rating was given.'; +'A note about why the feasibility rating was given. +DEPRECATED: Only used for Vit-SS compatibility.'; + COMMENT ON COLUMN proposal_review.auditability_rating_given IS -'The numeric auditability rating given.'; +'The numeric auditability rating given. +DEPRECATED: Only used for Vit-SS compatibility.'; COMMENT ON COLUMN proposal_review.auditability_note IS -'A note about the auditability rating given.'; - -COMMENT ON COLUMN proposal_review.allocated IS -'Describes if the review was part of the original reviewer allocation. -'; +'A note about the auditability rating given. +DEPRECATED: Only used for Vit-SS compatibility.'; COMMENT ON COLUMN proposal_review.ranking IS 'Numeric Measure of quality of this review according to veteran community advisors. diff --git a/src/event-db/migrations/V5__user.sql b/src/event-db/migrations/V5__user.sql deleted file mode 100644 index 80ee4d9a6a..0000000000 --- a/src/event-db/migrations/V5__user.sql +++ /dev/null @@ -1,53 +0,0 @@ --- Catalyst Event Database - --- User Table --- This table stores the users for the reviews module - -CREATE TABLE catalyst_user -( - row_id SERIAL PRIMARY KEY, - catalyst_id INTEGER NOT NULL, - enc_password VARCHAR NOT NULL, - salt VARCHAR NOT NULL, - extra JSONB NULL, - - FOREIGN KEY(catalyst_id) REFERENCES catalyst_id_map(row_id) ON DELETE CASCADE -); - -CREATE UNIQUE INDEX user_catalyst_id_idx ON catalyst_user(catalyst_id); - - -COMMENT ON TABLE catalyst_user IS ' -This tables stores the user account for the Review Module. -It contains a reference to the catalyst_id that is used as public and unique identifier. -'; - -COMMENT ON COLUMN catalyst_user.catalyst_id IS 'The catalyst_id this account belongs to.'; -COMMENT ON COLUMN catalyst_user.enc_password IS 'The encrypted password.'; -COMMENT ON COLUMN catalyst_user.salt IS 'The salt for the password encryption.'; -COMMENT ON COLUMN catalyst_user.extra IS -'This field is used to store all meta information about a user that -are required by the application. Specifically: - -admin: bool, -username: str, -registration_date: datetime, -due_diligence: { - status: int, - due_diligence_id: str, - url: str -}, -historic_stats: { - "reviews": { - "active_funds": int, - "submitted": int, - "blank": int, - "valid": int - }, - "moderations": { - "active_funds": int, - "submitted": int, - "allocated": int, - } -} -'; diff --git a/src/event-db/migrations/V7__vote_plan.sql b/src/event-db/migrations/V5__vote_plan.sql similarity index 100% rename from src/event-db/migrations/V7__vote_plan.sql rename to src/event-db/migrations/V5__vote_plan.sql diff --git a/src/event-db/migrations/V8__snapshot_tables.sql b/src/event-db/migrations/V6__snapshot_tables.sql similarity index 100% rename from src/event-db/migrations/V8__snapshot_tables.sql rename to src/event-db/migrations/V6__snapshot_tables.sql diff --git a/src/event-db/migrations/V9__vote_tables.sql b/src/event-db/migrations/V7__vote_tables.sql similarity index 100% rename from src/event-db/migrations/V9__vote_tables.sql rename to src/event-db/migrations/V7__vote_tables.sql diff --git a/src/event-db/migrations/V10__catalyst_automation.sql b/src/event-db/migrations/V8__catalyst_automation.sql similarity index 100% rename from src/event-db/migrations/V10__catalyst_automation.sql rename to src/event-db/migrations/V8__catalyst_automation.sql diff --git a/src/event-db/migrations/V9__moderation_stage.sql b/src/event-db/migrations/V9__moderation_stage.sql new file mode 100644 index 0000000000..7afaa75696 --- /dev/null +++ b/src/event-db/migrations/V9__moderation_stage.sql @@ -0,0 +1,42 @@ +-- Catalyst Event Database + +-- ModerationAllocation - Defines the relationship between users and proposals_reviews +-- to describe the allocation of moderations that needs to be done. + +CREATE TABLE moderation_allocation ( + row_id SERIAL PRIMARY KEY, + review_id INTEGER NOT NULL, + user_id INTEGER NOT NULL, + + FOREIGN KEY (review_id) REFERENCES proposal_review(row_id) ON DELETE CASCADE, + FOREIGN KEY (user_id) REFERENCES config(row_id) ON DELETE CASCADE +); + + +COMMENT ON TABLE moderation_allocation IS 'The relationship between users and proposals_reviews.'; +COMMENT ON COLUMN moderation_allocation.row_id IS 'Synthetic ID of this relationship.'; +COMMENT ON COLUMN moderation_allocation.review_id IS 'The review the relationship is related to.'; +COMMENT ON COLUMN moderation_allocation.user_id IS 'The user the relationship is related to.'; + + +-- Moderation - Defines the moderation submitted by users for each proposal_review. + +CREATE TABLE moderation ( + row_id SERIAL PRIMARY KEY, + review_id INTEGER NOT NULL, + user_id INTEGER NOT NULL, + classification INTEGER NOT NULL, + rationale VARCHAR, + UNIQUE (review_id, user_id), + + FOREIGN KEY (review_id) REFERENCES proposal_review(row_id) ON DELETE CASCADE, + FOREIGN KEY (user_id) REFERENCES config(row_id) ON DELETE CASCADE +); + + +COMMENT ON TABLE moderation IS 'An individual moderation for a proposal review.'; +COMMENT ON COLUMN moderation.row_id IS 'Synthetic ID of this moderation.'; +COMMENT ON COLUMN moderation.review_id IS 'The review the moderation is related to.'; +COMMENT ON COLUMN moderation.user_id IS 'The user the moderation is submitted from.'; +COMMENT ON COLUMN moderation.classification IS 'The value used to describe the moderation (e.g. 0: excluded, 1: included).'; +COMMENT ON COLUMN moderation.rationale IS 'The rationale for the given classification.'; \ No newline at end of file diff --git a/utilities/ideascale-importer/Earthfile b/utilities/ideascale-importer/Earthfile index a0f179c60c..90b93920f9 100644 --- a/utilities/ideascale-importer/Earthfile +++ b/utilities/ideascale-importer/Earthfile @@ -16,7 +16,6 @@ build: # Install Poetry RUN curl -sSL https://install.python-poetry.org | python3 - --version 2.0.1 - # Set path to poetry ENV PATH=/root/.local/bin:$PATH @@ -30,7 +29,7 @@ build: RUN poetry config installer.max-workers 10 RUN poetry install --no-cache --no-root RUN poetry self add poetry-plugin-export - + # Build the distribution wheels and save them as artifacts RUN poetry export --without-hashes -f requirements.txt --output requirements.txt RUN poetry build --no-cache -f wheel