diff --git a/.github/actions/build_ami/action.yaml b/.github/actions/build_ami/action.yaml index 7390f9c45..7270494cf 100644 --- a/.github/actions/build_ami/action.yaml +++ b/.github/actions/build_ami/action.yaml @@ -59,7 +59,7 @@ runs: uses: actions/checkout@v4 - name: Get EIF for Release ${{ inputs.operator_release }} - uses: IABTechLab/uid2-operator/.github/actions/download_release_artifact@main + uses: ./.github/actions/download_release_artifact if: ${{ inputs.operator_release != '' }} with: github_token: ${{ inputs.github_token }} @@ -87,6 +87,11 @@ runs: FILE=$(echo $ARTIFACTS | jq -r '.[0].name') unzip -o -d ./scripts/aws/uid2-operator-ami/artifacts $FILE.zip rm $FILE.zip + cd "./scripts/aws/uid2-operator-ami/artifacts/" + zip "uid2operatoreif.zip" "uid2operator.eif" + cd - + rm ./scripts/aws/uid2-operator-ami/artifacts/uid2operator.eif + ls ./scripts/aws/uid2-operator-ami/artifacts/ -al - name: Configure UID2 AWS credentials uses: aws-actions/configure-aws-credentials@v4 diff --git a/.github/actions/build_aws_eif/action.yaml b/.github/actions/build_aws_eif/action.yaml index f17523a44..e7b7e9287 100644 --- a/.github/actions/build_aws_eif/action.yaml +++ b/.github/actions/build_aws_eif/action.yaml @@ -96,8 +96,9 @@ runs: cp ${{ steps.buildFolder.outputs.BUILD_FOLDER }}/identity_scope.txt ${ARTIFACTS_OUTPUT_DIR}/ cp ${{ steps.buildFolder.outputs.BUILD_FOLDER }}/version_number.txt ${ARTIFACTS_OUTPUT_DIR}/ - cp ./scripts/aws/start.sh ${ARTIFACTS_OUTPUT_DIR}/ - cp ./scripts/aws/stop.sh ${ARTIFACTS_OUTPUT_DIR}/ + cp ./scripts/aws/ec2.py ${ARTIFACTS_OUTPUT_DIR}/ + cp ./scripts/confidential_compute.py ${ARTIFACTS_OUTPUT_DIR}/ + cp ./scripts/aws/requirements.txt ${ARTIFACTS_OUTPUT_DIR}/ cp ./scripts/aws/proxies.host.yaml ${ARTIFACTS_OUTPUT_DIR}/ cp ./scripts/aws/sockd.conf ${ARTIFACTS_OUTPUT_DIR}/ cp ./scripts/aws/uid2operator.service ${ARTIFACTS_OUTPUT_DIR}/ @@ -116,10 +117,22 @@ runs: docker cp amazonlinux:/sockd ${ARTIFACTS_OUTPUT_DIR}/ docker cp amazonlinux:/vsockpx ${ARTIFACTS_OUTPUT_DIR}/ docker cp amazonlinux:/${{ inputs.identity_scope }}operator.eif ${ARTIFACTS_OUTPUT_DIR}/uid2operator.eif + + eifsize=$(wc -c < "${ARTIFACTS_OUTPUT_DIR}/uid2operator.eif") + if [ $eifsize -le 1 ]; then + echo "The eif was less then 1 byte. This indicates a build failure" + exit 1 + fi docker cp amazonlinux:/pcr0.txt ${{ steps.buildFolder.outputs.BUILD_FOLDER }} docker cp amazonlinux:/pcr0.txt ${ARTIFACTS_OUTPUT_DIR}/ echo "enclave_id=$(cat ${{ steps.buildFolder.outputs.BUILD_FOLDER}}/pcr0.txt)" >> $GITHUB_OUTPUT + + pcrsize=$(wc -c < "${{ steps.buildFolder.outputs.BUILD_FOLDER}}/pcr0.txt") + if [ $pcrsize -le 1 ]; then + echo "The pcr0.txt file was less then 1 byte. This indicates a build failure" + exit 1 + fi - name: Cleanup shell: bash diff --git a/.github/actions/build_eks_docker_image/action.yaml b/.github/actions/build_eks_docker_image/action.yaml index 1a7bca316..922136c5d 100644 --- a/.github/actions/build_eks_docker_image/action.yaml +++ b/.github/actions/build_eks_docker_image/action.yaml @@ -47,7 +47,7 @@ runs: mkdir ${{ inputs.artifacts_output_dir }} -p - name: Get EIF for Release ${{ inputs.operator_release }} - uses: IABTechLab/uid2-operator/.github/actions/download_release_artifact@main + uses: ./.github/actions/download_release_artifact if: ${{ inputs.operator_release != '' }} with: github_token: ${{ inputs.github_token }} diff --git a/.github/actions/install_az_cli/action.yaml b/.github/actions/install_az_cli/action.yaml new file mode 100644 index 000000000..19bdb382c --- /dev/null +++ b/.github/actions/install_az_cli/action.yaml @@ -0,0 +1,36 @@ +name: 'Install Azure CLI' +description: 'Install Azure CLI' +runs: + using: 'composite' + steps: + - name: uninstall azure-cli + shell: bash + run: | + sudo apt-get remove -y azure-cli + + - name: install azure-cli 2.61.0 + shell: bash + run: | + sudo apt-get update + sudo apt-get install apt-transport-https ca-certificates curl gnupg lsb-release + sudo mkdir -p /etc/apt/keyrings + curl -sLS https://packages.microsoft.com/keys/microsoft.asc | + gpg --dearmor | sudo tee /etc/apt/keyrings/microsoft.gpg > /dev/null + sudo chmod go+r /etc/apt/keyrings/microsoft.gpg + AZ_DIST=$(lsb_release -cs) + echo "Types: deb + URIs: https://packages.microsoft.com/repos/azure-cli/ + Suites: ${AZ_DIST} + Components: main + Architectures: $(dpkg --print-architecture) + Signed-by: /etc/apt/keyrings/microsoft.gpg" | sudo tee /etc/apt/sources.list.d/azure-cli.sources + sudo apt-get update + sudo apt-get install azure-cli + + apt-cache policy azure-cli + # Obtain the currently installed distribution + AZ_DIST=$(lsb_release -cs) + # Store an Azure CLI version of choice + AZ_VER=2.61.0 + # Install a specific version + sudo apt-get install azure-cli=${AZ_VER}-1~${AZ_DIST} --allow-downgrades diff --git a/.github/actions/update_operator_version/action.yaml b/.github/actions/update_operator_version/action.yaml index 1c66838e8..b6251e275 100644 --- a/.github/actions/update_operator_version/action.yaml +++ b/.github/actions/update_operator_version/action.yaml @@ -34,7 +34,7 @@ runs: steps: - name: Check branch and release type id: checkRelease - uses: IABTechLab/uid2-shared-actions/actions/check_branch_and_release_type@v2 + uses: IABTechLab/uid2-shared-actions/actions/check_branch_and_release_type@v3 with: release_type: ${{ inputs.release_type }} @@ -43,7 +43,7 @@ runs: uses: trstringer/manual-approval@v1 with: secret: ${{ github.token }} - approvers: thomasm-ttd,atarassov-ttd,cody-constine-ttd + approvers: atarassov-ttd,vishalegbert-ttd,sunnywu,cody-constine-ttd minimum-approvals: 1 issue-title: Creating Major version of UID2-Operator @@ -81,7 +81,7 @@ runs: - name: Set version number id: version - uses: IABTechLab/uid2-shared-actions/actions/version_number@v2 + uses: IABTechLab/uid2-shared-actions/actions/version_number@v3 with: type: ${{ inputs.release_type }} version_number: ${{ inputs.version_number_input }} diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index 00695f1db..aa13387c6 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -3,7 +3,7 @@ on: [pull_request, push, workflow_dispatch] jobs: build: - uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-build-and-test.yaml@v2 + uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-build-and-test.yaml@v3 with: java_version: 21 secrets: inherit \ No newline at end of file diff --git a/.github/workflows/build-uid2-ami.yaml b/.github/workflows/build-uid2-ami.yaml index a6c3143da..8439b33c6 100644 --- a/.github/workflows/build-uid2-ami.yaml +++ b/.github/workflows/build-uid2-ami.yaml @@ -42,7 +42,7 @@ jobs: - name: Build UID2 Operator AMI id: buildAMI - uses: IABTechLab/uid2-operator/.github/actions/build_ami@main + uses: ./.github/actions/build_ami with: identity_scope: uid2 eif_repo_owner: ${{ env.REPO_OWNER }} @@ -92,7 +92,7 @@ jobs: - name: Build EUID Operator AMI id: buildAMI - uses: IABTechLab/uid2-operator/.github/actions/build_ami@main + uses: ./.github/actions/build_ami with: identity_scope: euid eif_repo_owner: ${{ env.REPO_OWNER }} diff --git a/.github/workflows/publish-all-operators.yaml b/.github/workflows/publish-all-operators.yaml index c5db3a3b0..7e2529646 100644 --- a/.github/workflows/publish-all-operators.yaml +++ b/.github/workflows/publish-all-operators.yaml @@ -1,5 +1,5 @@ name: Publish All Operators -run-name: ${{ format('Publish All Operators - {0} Release', inputs.release_type) }} +run-name: ${{ format('Publish All Operators - {0} Release', github.event.inputs.release_type || 'scheduled') }} on: workflow_dispatch: inputs: @@ -18,6 +18,8 @@ on: - CRITICAL,HIGH - CRITICAL,HIGH,MEDIUM - CRITICAL (DO NOT use if JIRA ticket not raised) + schedule: + - cron: "0 0 * * *" jobs: start: @@ -26,13 +28,25 @@ jobs: outputs: new_version: ${{ steps.version.outputs.new_version }} commit_sha: ${{ steps.commit-and-tag.outputs.commit_sha }} + release_type: ${{ steps.set-env.outputs.release_type }} + vulnerability_severity: ${{ steps.set-env.outputs.vulnerability_severity }} + env: + RELEASE_TYPE: ${{ inputs.release_type || (github.event_name == 'schedule' && 'patch') }} + VULNERABILITY_SEVERITY: ${{ inputs.vulnerability_severity || (github.event_name == 'schedule' && 'CRITICAL,HIGH') }} steps: + - name: Set Environment Variables + id: set-env + run: | + echo "release_type=${{ inputs.release_type || (github.event_name == 'schedule' && 'patch') }}" >> $GITHUB_ENV + echo "vulnerability_severity=${{ inputs.vulnerability_severity || (github.event_name == 'schedule' && 'CRITICAL,HIGH') }}" >> $GITHUB_ENV + echo "release_type=${RELEASE_TYPE}" >> $GITHUB_OUTPUT + echo "vulnerability_severity=${VULNERABILITY_SEVERITY}" >> $GITHUB_OUTPUT - name: Approve Major release - if: inputs.release_type == 'Major' + if: env.RELEASE_TYPE == 'Major' uses: trstringer/manual-approval@v1 with: secret: ${{ github.token }} - approvers: thomasm-ttd,atarassov-ttd,cody-constine-ttd + approvers: atarassov-ttd,vishalegbert-ttd,sunnywu,cody-constine-ttd minimum-approvals: 1 issue-title: Creating Major version of UID2-Operator @@ -45,7 +59,7 @@ jobs: GITHUB_CONTEXT: ${{ toJson(github) }} - name: Check branch and release type - uses: IABTechLab/uid2-shared-actions/actions/check_branch_and_release_type@v2 + uses: IABTechLab/uid2-shared-actions/actions/check_branch_and_release_type@v3 with: release_type: ${{ inputs.release_type }} @@ -55,16 +69,17 @@ jobs: fetch-depth: 0 - name: Scan vulnerabilities - uses: IABTechLab/uid2-shared-actions/actions/vulnerability_scan_filesystem@v2 + uses: IABTechLab/uid2-shared-actions/actions/vulnerability_scan@v3 with: scan_severity: HIGH,CRITICAL failure_severity: CRITICAL + scan_type: 'fs' - name: Set version number id: version - uses: IABTechLab/uid2-shared-actions/actions/version_number@v2 + uses: IABTechLab/uid2-shared-actions/actions/version_number@v3 with: - type: ${{ inputs.release_type }} + type: ${{ env.RELEASE_TYPE }} branch_name: ${{ github.ref }} - name: Update pom.xml @@ -79,7 +94,7 @@ jobs: uses: IABTechLab/uid2-shared-actions/actions/commit_pr_and_merge@v3 with: add: 'pom.xml version.json' - message: 'Released ${{ inputs.release_type }} version: ${{ steps.version.outputs.new_version }}' + message: 'Released ${{ env.RELEASE_TYPE }} version: ${{ steps.version.outputs.new_version }}' tag: v${{ steps.version.outputs.new_version }} buildPublic: @@ -87,9 +102,9 @@ jobs: needs: start uses: ./.github/workflows/publish-public-operator-docker-image.yaml with: - release_type: ${{ inputs.release_type }} + release_type: ${{ needs.start.outputs.release_type }} version_number_input: ${{ needs.start.outputs.new_version }} - vulnerability_severity: ${{ inputs.vulnerability_severity }} + vulnerability_severity: ${{ needs.start.outputs.vulnerability_severity }} secrets: inherit buildGCP: @@ -97,10 +112,10 @@ jobs: needs: start uses: ./.github/workflows/publish-gcp-oidc-enclave-docker.yaml with: - release_type: ${{ inputs.release_type }} + release_type: ${{ needs.start.outputs.release_type }} version_number_input: ${{ needs.start.outputs.new_version }} commit_sha: ${{ needs.start.outputs.commit_sha }} - vulnerability_severity: ${{ inputs.vulnerability_severity }} + vulnerability_severity: ${{ needs.start.outputs.vulnerability_severity }} secrets: inherit buildAzure: @@ -108,10 +123,10 @@ jobs: needs: start uses: ./.github/workflows/publish-azure-cc-enclave-docker.yaml with: - release_type: ${{ inputs.release_type }} + release_type: ${{ needs.start.outputs.release_type }} version_number_input: ${{ needs.start.outputs.new_version }} commit_sha: ${{ needs.start.outputs.commit_sha }} - vulnerability_severity: ${{ inputs.vulnerability_severity }} + vulnerability_severity: ${{ needs.start.outputs.vulnerability_severity }} secrets: inherit buildAWS: @@ -119,7 +134,7 @@ jobs: needs: start uses: ./.github/workflows/publish-aws-nitro-eif.yaml with: - release_type: ${{ inputs.release_type }} + release_type: ${{ needs.start.outputs.release_type }} version_number_input: ${{ needs.start.outputs.new_version }} commit_sha: ${{ needs.start.outputs.commit_sha }} secrets: inherit @@ -132,18 +147,11 @@ jobs: operator_run_number: ${{ github.run_id }} secrets: inherit - buildEKS: - name: Build AWS EKS Docker - needs: [start, buildAWS] - uses: ./.github/workflows/publish-aws-eks-nitro-enclave-docker.yaml - with: - operator_run_number: ${{ github.run_id }} - secrets: inherit - createRelease: name: Create Release runs-on: ubuntu-latest - needs: [start, buildPublic, buildGCP, buildAzure, buildAWS, buildAMI, buildEKS] + if: github.event_name == 'workflow_dispatch' + needs: [start, buildPublic, buildGCP, buildAzure, buildAWS, buildAMI] steps: - name: Checkout repo uses: actions/checkout@v4 @@ -162,12 +170,18 @@ jobs: pattern: gcp-oidc-enclave-ids-* path: ./manifests/gcp_oidc_operator - - name: Download Azure manifest + - name: Download Azure CC manifest uses: actions/download-artifact@v4 with: pattern: azure-cc-enclave-id-* path: ./manifests/azure_cc_operator + - name: Download Azure AKS manifest + uses: actions/download-artifact@v4 + with: + pattern: azure-aks-enclave-id-* + path: ./manifests/azure_aks_operator + - name: Download EIF manifest uses: actions/download-artifact@v4 with: @@ -180,12 +194,6 @@ jobs: pattern: 'aws-ami-ids-*' path: ./manifests/aws_ami - - name: Download AWS EKS manifest - uses: actions/download-artifact@v4 - with: - pattern: 'aws-eks-enclave-ids-*' - path: ./manifests/aws_eks - - name: Download Deployment Files uses: actions/download-artifact@v4 with: @@ -216,6 +224,7 @@ jobs: (cd ./deployment/aws-euid-deployment-files-${{ needs.start.outputs.new_version }} && zip -r ../../aws-euid-deployment-files-${{ needs.start.outputs.new_version }}.zip . ) (cd ./deployment/aws-uid2-deployment-files-${{ needs.start.outputs.new_version }} && zip -r ../../aws-uid2-deployment-files-${{ needs.start.outputs.new_version }}.zip . ) (cd ./deployment/azure-cc-deployment-files-${{ needs.start.outputs.new_version }} && zip -r ../../azure-cc-deployment-files-${{ needs.start.outputs.new_version }}.zip . ) + (cd ./deployment/azure-aks-deployment-files-${{ needs.start.outputs.new_version }} && zip -r ../../azure-aks-deployment-files-${{ needs.start.outputs.new_version }}.zip . ) (cd ./deployment/gcp-oidc-deployment-files-${{ needs.start.outputs.new_version }} && zip -r ../../gcp-oidc-deployment-files-${{ needs.start.outputs.new_version }}.zip . ) (cd manifests && zip -r ../uid2-operator-release-manifests-${{ needs.start.outputs.new_version }}.zip .) @@ -229,5 +238,19 @@ jobs: ./aws-euid-deployment-files-${{ needs.start.outputs.new_version }}.zip ./aws-uid2-deployment-files-${{ needs.start.outputs.new_version }}.zip ./azure-cc-deployment-files-${{ needs.start.outputs.new_version }}.zip + ./azure-aks-deployment-files-${{ needs.start.outputs.new_version }}.zip ./gcp-oidc-deployment-files-${{ needs.start.outputs.new_version }}.zip ./uid2-operator-release-manifests-${{ needs.start.outputs.new_version }}.zip + notifyFailure: + name: Notify Slack on Failure + runs-on: ubuntu-latest + if: failure() && github.ref == 'refs/heads/main' + needs: [start, buildPublic, buildGCP, buildAzure, buildAWS, buildAMI] + steps: + - name: Send Slack Alert + env: + SLACK_COLOR: danger + SLACK_MESSAGE: ':x: Operator Pipeline failed' + SLACK_TITLE: Pipeline Failed in ${{ github.workflow }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + uses: rtCamp/action-slack-notify@v2 diff --git a/.github/workflows/publish-aws-eks-nitro-enclave-docker.yaml b/.github/workflows/publish-aws-eks-nitro-enclave-docker.yaml index eb602b422..0de600aac 100644 --- a/.github/workflows/publish-aws-eks-nitro-enclave-docker.yaml +++ b/.github/workflows/publish-aws-eks-nitro-enclave-docker.yaml @@ -1,4 +1,4 @@ -name: Publish EKS Operator Docker Images +name: Publish EKS Enclave Operator Docker Images run-name: >- ${{ inputs.operator_release == '' && format('Publish EKS Operator Docker Images for Operator Run Number: {0}', inputs.operator_run_number) || format('Publish EKS Operator Docker Images for Operator Release: {0}', inputs.operator_release)}} on: @@ -36,9 +36,12 @@ jobs: security-events: write packages: write steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build Docker Image for EKS Pod id: build_docker_image_uid - uses: IABTechLab/uid2-operator/.github/actions/build_eks_docker_image@main + uses: ./.github/actions/build_eks_docker_image with: identity_scope: uid2 artifacts_output_dir: ${{ env.ARTIFACTS_BASE_OUTPUT_DIR }}/uid2 @@ -61,9 +64,12 @@ jobs: security-events: write packages: write steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build Docker Image for EKS Pod id: build_docker_image_euid - uses: IABTechLab/uid2-operator/.github/actions/build_eks_docker_image@main + uses: ./.github/actions/build_eks_docker_image with: identity_scope: euid artifacts_output_dir: ${{ env.ARTIFACTS_BASE_OUTPUT_DIR }}/euid diff --git a/.github/workflows/publish-aws-nitro-eif.yaml b/.github/workflows/publish-aws-nitro-eif.yaml index 8783f6829..3c599c663 100644 --- a/.github/workflows/publish-aws-nitro-eif.yaml +++ b/.github/workflows/publish-aws-nitro-eif.yaml @@ -48,9 +48,12 @@ jobs: env: GITHUB_CONTEXT: ${{ toJson(github) }} + - name: Checkout + uses: actions/checkout@v4 + - name: Update Operator Version id: update_version - uses: IABTechLab/uid2-operator/.github/actions/update_operator_version@main + uses: ./.github/actions/update_operator_version with: release_type: ${{ inputs.release_type }} version_number_input: ${{ inputs.version_number_input }} @@ -68,9 +71,12 @@ jobs: runs-on: ubuntu-latest needs: start steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build UID2 AWS EIF id: build_uid2_eif - uses: IABTechLab/uid2-operator/.github/actions/build_aws_eif@main + uses: ./.github/actions/build_aws_eif with: identity_scope: uid2 artifacts_base_output_dir: ${{ env.ARTIFACTS_BASE_OUTPUT_DIR }}/uid2 @@ -104,9 +110,12 @@ jobs: runs-on: ubuntu-latest needs: start steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build EUID AWS EIF id: build_euid_eif - uses: IABTechLab/uid2-operator/.github/actions/build_aws_eif@main + uses: ./.github/actions/build_aws_eif with: identity_scope: euid artifacts_base_output_dir: ${{ env.ARTIFACTS_BASE_OUTPUT_DIR }}/euid diff --git a/.github/workflows/publish-azure-cc-enclave-docker.yaml b/.github/workflows/publish-azure-cc-enclave-docker.yaml index 0127a71f4..15064f94a 100644 --- a/.github/workflows/publish-azure-cc-enclave-docker.yaml +++ b/.github/workflows/publish-azure-cc-enclave-docker.yaml @@ -69,10 +69,16 @@ jobs: outputs: jar_version: ${{ steps.update_version.outputs.new_version }} image_tag: ${{ steps.update_version.outputs.image_tag }} + is_release: ${{ steps.update_version.outputs.is_release }} + docker_version: ${{ steps.meta.outputs.version }} + tags: ${{ steps.meta.outputs.tags }} steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Update Operator Version id: update_version - uses: IABTechLab/uid2-operator/.github/actions/update_operator_version@main + uses: ./.github/actions/update_operator_version with: release_type: ${{ inputs.release_type }} version_number_input: ${{ inputs.version_number_input }} @@ -92,6 +98,7 @@ jobs: echo "jar_version=$(mvn help:evaluate -Dexpression=project.version | grep -e '^[1-9][^\[]')" >> $GITHUB_OUTPUT echo "git_commit=$(git show --format="%h" --no-patch)" >> $GITHUB_OUTPUT cp -r target ${{ env.DOCKER_CONTEXT_PATH }}/ + cp scripts/confidential_compute.py ${{ env.DOCKER_CONTEXT_PATH }}/ - name: Log in to the Docker container registry uses: docker/login-action@v3 @@ -158,35 +165,17 @@ jobs: JAR_VERSION=${{ steps.update_version.outputs.new_version }} IMAGE_VERSION=${{ steps.update_version.outputs.new_version }} - - name: uninstall azure-cli - run: | - sudo apt-get remove -y azure-cli - - - name: install azure-cli 2.61.0 - run: | - sudo apt-get update - sudo apt-get install apt-transport-https ca-certificates curl gnupg lsb-release - sudo mkdir -p /etc/apt/keyrings - curl -sLS https://packages.microsoft.com/keys/microsoft.asc | - gpg --dearmor | sudo tee /etc/apt/keyrings/microsoft.gpg > /dev/null - sudo chmod go+r /etc/apt/keyrings/microsoft.gpg - AZ_DIST=$(lsb_release -cs) - echo "Types: deb - URIs: https://packages.microsoft.com/repos/azure-cli/ - Suites: ${AZ_DIST} - Components: main - Architectures: $(dpkg --print-architecture) - Signed-by: /etc/apt/keyrings/microsoft.gpg" | sudo tee /etc/apt/sources.list.d/azure-cli.sources - sudo apt-get update - sudo apt-get install azure-cli - - apt-cache policy azure-cli - # Obtain the currently installed distribution - AZ_DIST=$(lsb_release -cs) - # Store an Azure CLI version of choice - AZ_VER=2.61.0 - # Install a specific version - sudo apt-get install azure-cli=${AZ_VER}-1~${AZ_DIST} --allow-downgrades + azureCc: + name: Create Azure CC artifacts + runs-on: ubuntu-latest + permissions: {} + needs: buildImage + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Azure CLI + uses: ./.github/actions/install_az_cli - name: check azure-cli version run: | @@ -194,61 +183,80 @@ jobs: - name: Generate Azure deployment artifacts env: - IMAGE: ${{ steps.meta.outputs.tags }} + IMAGE: ${{ needs.buildImage.outputs.tags }} OUTPUT_DIR: ${{ env.ARTIFACTS_OUTPUT_DIR }} MANIFEST_DIR: ${{ env.MANIFEST_OUTPUT_DIR }} - VERSION_NUMBER: ${{ steps.update_version.outputs.new_version }} + VERSION_NUMBER: ${{ needs.buildImage.outputs.jar_version }} run: | bash ./scripts/azure-cc/deployment/generate-deployment-artifacts.sh - name: Upload deployment artifacts uses: actions/upload-artifact@v4 with: - name: azure-cc-deployment-files-${{ steps.update_version.outputs.new_version }} + name: azure-cc-deployment-files-${{ needs.buildImage.outputs.jar_version }} path: ${{ env.ARTIFACTS_OUTPUT_DIR }} if-no-files-found: error - name: Upload manifest uses: actions/upload-artifact@v4 with: - name: azure-cc-enclave-id-${{ steps.update_version.outputs.new_version }} + name: azure-cc-enclave-id-${{ needs.buildImage.outputs.jar_version }} path: ${{ env.MANIFEST_OUTPUT_DIR }} if-no-files-found: error - - name: Generate release archive - if: ${{ inputs.version_number_input == '' && steps.update_version.outputs.is_release == 'true' }} + e2eAzureCc: + name: E2E Azure CC + uses: ./.github/workflows/run-e2e-tests-on-operator.yaml + needs: [buildImage, azureCc] + with: + operator_type: azure + operator_image_version: ${{ needs.buildImage.outputs.image_tag }} + secrets: inherit + + azureAks: + name: Create Azure AKS artifacts + runs-on: ubuntu-latest + permissions: {} + needs: buildImage + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Azure CLI + uses: ./.github/actions/install_az_cli + + - name: check azure-cli version + run: | + az --version + + - name: Generate Azure deployment artifacts + env: + IMAGE: ${{ needs.buildImage.outputs.tags }} + OUTPUT_DIR: ${{ env.ARTIFACTS_OUTPUT_DIR }} + MANIFEST_DIR: ${{ env.MANIFEST_OUTPUT_DIR }} + VERSION_NUMBER: ${{ needs.buildImage.outputs.jar_version }} run: | - zip -j ${{ env.ARTIFACTS_OUTPUT_DIR }}/uid2-operator-deployment-artifacts-${{ steps.meta.outputs.version }}.zip ${{ env.ARTIFACTS_OUTPUT_DIR }}/* + bash ./scripts/azure-aks/deployment/generate-deployment-artifacts.sh - - name: Build changelog - id: github_release - if: ${{ inputs.version_number_input == '' && steps.update_version.outputs.is_release == 'true' }} - uses: mikepenz/release-changelog-builder-action@v4 + - name: Upload deployment artifacts + uses: actions/upload-artifact@v4 with: - configurationJson: | - { - "template": "#{{CHANGELOG}}\n## Installation\n```\ndocker pull ${{ steps.meta.outputs.tags }}\n```\n\n## Image reference to deploy: \n```\n${{ steps.update_version.outputs.image_tag }}\n```\n\n## Changelog\n#{{UNCATEGORIZED}}", - "pr_template": " - #{{TITLE}} - ( PR: ##{{NUMBER}} )" - } - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + name: azure-aks-deployment-files-${{ needs.buildImage.outputs.jar_version }} + path: ${{ env.ARTIFACTS_OUTPUT_DIR }} + if-no-files-found: error - - name: Create release - if: ${{ inputs.version_number_input == '' && steps.update_version.outputs.is_release == 'true' }} - uses: softprops/action-gh-release@v2 + - name: Upload manifest + uses: actions/upload-artifact@v4 with: - name: ${{ steps.update_version.outputs.new_version }} - body: ${{ steps.github_release.outputs.changelog }} - draft: true - files: | - ${{ env.ARTIFACTS_OUTPUT_DIR }}/uid2-operator-deployment-artifacts-${{ steps.update_version.outputs.new_version }}.zip - ${{ env.MANIFEST_OUTPUT_DIR }}/azure-cc-operator-digest-${{ steps.update_version.outputs.new_version }}.txt - - e2e: - name: E2E + name: azure-aks-enclave-id-${{ needs.buildImage.outputs.jar_version }} + path: ${{ env.MANIFEST_OUTPUT_DIR }} + if-no-files-found: error + + e2eAzureAks: + name: E2E Azure AKS uses: ./.github/workflows/run-e2e-tests-on-operator.yaml - needs: buildImage + needs: [buildImage, azureAks] with: - operator_type: azure + operator_type: aks operator_image_version: ${{ needs.buildImage.outputs.image_tag }} secrets: inherit diff --git a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml index 9f042a916..02977f83d 100644 --- a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml +++ b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml @@ -71,9 +71,12 @@ jobs: jar_version: ${{ steps.update_version.outputs.new_version }} image_tag: ${{ steps.update_version.outputs.image_tag }} steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Update Operator Version id: update_version - uses: IABTechLab/uid2-operator/.github/actions/update_operator_version@main + uses: ./.github/actions/update_operator_version with: release_type: ${{ inputs.release_type }} version_number_input: ${{ inputs.version_number_input }} @@ -93,6 +96,7 @@ jobs: echo "jar_version=$(mvn help:evaluate -Dexpression=project.version | grep -e '^[1-9][^\[]')" >> $GITHUB_OUTPUT echo "git_commit=$(git show --format="%h" --no-patch)" >> $GITHUB_OUTPUT cp -r target ${{ env.DOCKER_CONTEXT_PATH }}/ + cp scripts/confidential_compute.py ${{ env.DOCKER_CONTEXT_PATH }}/ - name: Log in to the Docker container registry uses: docker/login-action@v3 @@ -155,31 +159,13 @@ jobs: IMAGE_VERSION=${{ steps.update_version.outputs.new_version }} BUILD_TARGET=${{ env.ENCLAVE_PROTOCOL }} - - name: Generate Trivy vulnerability scan report - uses: aquasecurity/trivy-action@0.14.0 - with: - image-ref: ${{ steps.meta.outputs.tags }} - format: 'sarif' - exit-code: '0' - ignore-unfixed: true - severity: 'CRITICAL,HIGH' - output: 'trivy-results.sarif' - hide-progress: true - - - name: Upload Trivy scan report to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: 'trivy-results.sarif' - - - name: Test with Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.14.0 + - name: Vulnerability Scan + uses: IABTechLab/uid2-shared-actions/actions/vulnerability_scan@v3 with: - image-ref: ${{ steps.meta.outputs.tags }} - format: 'table' - exit-code: '1' - ignore-unfixed: true - severity: ${{ inputs.vulnerability_severity }} - hide-progress: true + image_ref: ${{ steps.meta.outputs.tags }} + scan_type: 'image' + skip_files: '/venv/lib/python3.12/site-packages/google/auth/crypt/__pycache__/_python_rsa.cpython-312.pyc' # Skip scanning this file as per UID2-4968 + failure_severity: ${{ (inputs.vulnerability_severity == 'CRITICAL (DO NOT use if JIRA ticket not raised)' && 'CRITICAL') || inputs.vulnerability_severity }} - name: Push to Docker id: push-to-docker diff --git a/.github/workflows/publish-public-operator-docker-image.yaml b/.github/workflows/publish-public-operator-docker-image.yaml index d55806c6b..db3c527c8 100644 --- a/.github/workflows/publish-public-operator-docker-image.yaml +++ b/.github/workflows/publish-public-operator-docker-image.yaml @@ -53,7 +53,7 @@ jobs: uses: trstringer/manual-approval@v1 with: secret: ${{ github.token }} - approvers: thomasm-ttd,atarassov-ttd,cody-constine-ttd + approvers: atarassov-ttd,vishalegbert-ttd,sunnywu,cody-constine-ttd minimum-approvals: 1 issue-title: Creating Major version of UID2-Operator diff --git a/.github/workflows/run-e2e-tests-on-operator.yaml b/.github/workflows/run-e2e-tests-on-operator.yaml index e57756c1b..462a992e1 100644 --- a/.github/workflows/run-e2e-tests-on-operator.yaml +++ b/.github/workflows/run-e2e-tests-on-operator.yaml @@ -1,10 +1,10 @@ name: Run Operator E2E Tests -run-name: ${{ format('Run Operator E2E Tests - {0} {1}', inputs.operator_type, inputs.identity_scope) }} by @${{ github.actor }} +run-name: ${{ format('Run Operator E2E Tests - {0} {1} {2}', inputs.operator_type, inputs.identity_scope, inputs.target_environment) }} by @${{ github.actor }} on: workflow_dispatch: inputs: operator_type: - description: The operator type [public, gcp, azure, aws, eks] + description: The operator type [public, gcp, azure, aws, aks] required: true type: choice options: @@ -12,7 +12,7 @@ on: - gcp - azure - aws - - eks + - aks identity_scope: description: The identity scope [UID2, EUID] required: true @@ -20,6 +20,19 @@ on: options: - UID2 - EUID + target_environment: + description: PRIVATE OPERATORS ONLY - The target environment [mock, integ, prod] + required: true + type: choice + options: + - mock + - integ + - prod + delay_operator_shutdown: + description: PRIVATE OPERATORS ONLY - If true, will delay operator shutdown by 24 hours. + required: true + type: boolean + default: false operator_image_version: description: 'Image: Operator image version (for gcp/azure, set appropriate image)' type: string @@ -51,22 +64,25 @@ on: "region": "us-east-1", "ami": "ami-xxxxx", "pcr0": "xxxxx" }' - eks: - description: The arguments for EKS operator - type: string - default: '{ - "pcr0": "xxxxx" }' workflow_call: inputs: operator_type: - description: The operator type [public, gcp, azure, aws, eks] + description: The operator type [public, gcp, azure, aws, aks] type: string default: public identity_scope: description: The identity scope [UID2, EUID] type: string default: UID2 + target_environment: + description: PRIVATE OPERATORS ONLY - The target environment [mock, integ, prod] + type: string + default: mock + delay_operator_shutdown: + description: PRIVATE OPERATORS ONLY - If true, will delay operator shutdown by 24 hours. + type: boolean + default: false operator_image_version: description: 'Image: Operator image version (for gcp/azure, set appropriate image)' type: string @@ -97,11 +113,6 @@ on: "region": "us-east-1", "ami": "ami-xxxxx", "pcr0": "xxxxx" }' - eks: - description: The arguments for EKS operator - type: string - default: '{ - "pcr0": "xxxxx" }' jobs: e2e-test: @@ -109,22 +120,21 @@ jobs: uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-run-e2e-tests.yaml@v3 with: operator_type: ${{ inputs.operator_type }} + identity_scope: ${{ inputs.identity_scope }} + target_environment: ${{ inputs.target_environment }} + delay_operator_shutdown: ${{ inputs.delay_operator_shutdown }} operator_image_version: ${{ inputs.operator_image_version }} core_image_version: ${{ inputs.core_image_version }} optout_image_version: ${{ inputs.optout_image_version }} e2e_image_version: ${{ inputs.e2e_image_version }} operator_branch: ${{ github.ref }} - branch_core: ${{ fromJson(inputs.branch).core }} - branch_optout: ${{ fromJson(inputs.branch).optout }} - branch_admin: ${{ fromJson(inputs.branch).admin }} - uid2_e2e_identity_scope: ${{ inputs.identity_scope }} + core_branch: ${{ fromJson(inputs.branch).core }} + optout_branch: ${{ fromJson(inputs.branch).optout }} + admin_branch: ${{ fromJson(inputs.branch).admin }} gcp_workload_identity_provider_id: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER_ID }} gcp_service_account: ${{ vars.GCP_SERVICE_ACCOUNT }} gcp_project: ${{ vars.GCP_PROJECT }} aws_region: ${{ fromJson(inputs.aws).region }} aws_ami: ${{ fromJson(inputs.aws).ami }} aws_pcr0: ${{ fromJson(inputs.aws).pcr0 }} - eks_pcr0: ${{ fromJson(inputs.eks).pcr0 }} - eks_test_cluster: ${{ vars.EKS_TEST_CLUSTER }} - eks_test_cluster_region: ${{ vars.EKS_TEST_CLUSTER_REGION }} secrets: inherit diff --git a/.github/workflows/validate-image.yaml b/.github/workflows/validate-image.yaml index 524f19102..37b4bf912 100644 --- a/.github/workflows/validate-image.yaml +++ b/.github/workflows/validate-image.yaml @@ -19,7 +19,7 @@ on: jobs: build-publish-docker-default: - uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v2 + uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v3 with: failure_severity: ${{ inputs.failure_severity || 'CRITICAL,HIGH' }} fail_on_error: ${{ inputs.fail_on_error || true }} @@ -27,7 +27,7 @@ jobs: java_version: 21 secrets: inherit build-publish-docker-aws: - uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v2 + uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v3 with: failure_severity: ${{ inputs.failure_severity || 'CRITICAL,HIGH' }} fail_on_error: ${{ inputs.fail_on_error || true }} @@ -36,7 +36,7 @@ jobs: secrets: inherit needs: [build-publish-docker-default] build-publish-docker-gcp: - uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v2 + uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v3 with: failure_severity: ${{ inputs.failure_severity || 'CRITICAL,HIGH' }} fail_on_error: ${{ inputs.fail_on_error || true }} @@ -45,7 +45,7 @@ jobs: secrets: inherit needs: [build-publish-docker-aws] build-publish-docker-azure: - uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v2 + uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v3 with: failure_severity: ${{ inputs.failure_severity || 'CRITICAL,HIGH' }} fail_on_error: ${{ inputs.fail_on_error || true }} diff --git a/.github/workflows/vulnerability-scan-failure-notify.yaml b/.github/workflows/vulnerability-scan-failure-notify.yaml new file mode 100644 index 000000000..7a87e06fc --- /dev/null +++ b/.github/workflows/vulnerability-scan-failure-notify.yaml @@ -0,0 +1,24 @@ +name: Vulnerability Scan Failure Slack Notify +on: + workflow_dispatch: + inputs: + vulnerability_severity: + description: The severity to fail the workflow if such vulnerability is detected. DO NOT override it unless a Jira ticket is raised. DO NOT use 'CRITICAL' unless a Jira ticket is raised. + type: choice + options: + - CRITICAL,HIGH + - CRITICAL,HIGH,MEDIUM + - CRITICAL + default: 'CRITICAL,HIGH' + schedule: + - cron: '0 16 * * *' # 9:00 AM GMT -7 + - cron: '0 0 * * *' # 5:00 PM GMT -7 + +jobs: + vulnerability-scan-failure-notify: + uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-vulnerability-scan-failure-notify.yaml@v3 + secrets: + SLACK_WEBHOOK : ${{ secrets.SLACK_WEBHOOK }} + with: + scan_type : image + java_version: "21" diff --git a/.trivyignore b/.trivyignore index 3aa85f54a..b5b45125c 100644 --- a/.trivyignore +++ b/.trivyignore @@ -1,5 +1,9 @@ # List any vulnerability that are to be accepted # See https://aquasecurity.github.io/trivy/v0.35/docs/vulnerability/examples/filter/ # for more details -# e.g. -# CVE-2022-3996 + +# https://thetradedesk.atlassian.net/browse/UID2-4460 +CVE-2024-47535 + +# https://thetradedesk.atlassian.net/browse/UID2-5186 +CVE-2024-8176 exp:2025-06-03 diff --git a/Dockerfile b/Dockerfile index c698202c2..647fb60bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ -# sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.4_7-jre-alpine/images/sha256-8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca -FROM eclipse-temurin@sha256:8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca +# sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.7_6-jre-alpine-3.21/images/sha256-62fa775039897e4420368514ba6c167741f6d45a0de9ff9125bee57e5aca8b75 +FROM eclipse-temurin@sha256:62fa775039897e4420368514ba6c167741f6d45a0de9ff9125bee57e5aca8b75 WORKDIR /app EXPOSE 8080 @@ -7,28 +7,25 @@ EXPOSE 8080 ARG JAR_NAME=uid2-operator ARG JAR_VERSION=1.0.0-SNAPSHOT ARG IMAGE_VERSION=1.0.0.unknownhash -ARG EXTRA_CONFIG ENV JAR_NAME=${JAR_NAME} ENV JAR_VERSION=${JAR_VERSION} ENV IMAGE_VERSION=${IMAGE_VERSION} ENV REGION=us-east-2 -ENV LOKI_HOSTNAME=loki -ENV LOGBACK_CONF=${LOGBACK_CONF:-./conf/logback.xml} COPY ./target/${JAR_NAME}-${JAR_VERSION}-jar-with-dependencies.jar /app/${JAR_NAME}-${JAR_VERSION}.jar COPY ./target/${JAR_NAME}-${JAR_VERSION}-sources.jar /app COPY ./target/${JAR_NAME}-${JAR_VERSION}-static.tar.gz /app/static.tar.gz -COPY ./conf/default-config.json ${EXTRA_CONFIG} /app/conf/ +COPY ./conf/default-config.json /app/conf/ COPY ./conf/*.xml /app/conf/ RUN tar xzvf /app/static.tar.gz --no-same-owner --no-same-permissions && rm -f /app/static.tar.gz -RUN adduser -D uid2-operator && mkdir -p /opt/uid2 && chmod 777 -R /opt/uid2 && mkdir -p /app && chmod 705 -R /app && mkdir -p /app/file-uploads && chmod 777 -R /app/file-uploads +RUN adduser -D uid2-operator && mkdir -p /opt/uid2 && chmod 777 -R /opt/uid2 && mkdir -p /app && chmod 705 -R /app && mkdir -p /app/file-uploads && chmod 777 -R /app/file-uploads && mkdir -p /app/pod_terminating && chmod 777 -R /app/pod_terminating USER uid2-operator CMD java \ -XX:MaxRAMPercentage=95 -XX:-UseCompressedOops -XX:+PrintFlagsFinal -XX:-OmitStackTraceInFastThrow \ -Djava.security.egd=file:/dev/./urandom \ -Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory \ - -Dlogback.configurationFile=${LOGBACK_CONF} \ + -Dlogback.configurationFile=/app/conf/logback.xml \ -jar ${JAR_NAME}-${JAR_VERSION}.jar diff --git a/Makefile.eif b/Makefile.eif index 395685024..38e47c13c 100644 --- a/Makefile.eif +++ b/Makefile.eif @@ -13,23 +13,14 @@ all: build_eif build_eif: uid2operator.eif euidoperator.eif -uid2operator.eif: build_artifacts build_configs build/proxies.nitro.yaml build/syslog-ng-client.conf build/syslog-ng-core_4.6.0-1_amd64.deb build/syslog-ng-ose-pub.asc build/entrypoint.sh build/vsockpx build/Dockerfile build/load_config.py build/make_config.py - cd build; docker build -t uid2operator . --build-arg JAR_VERSION=`cat package.version` --build-arg IMAGE_VERSION=`cat package.version`-`git show --format="%h" --no-patch`; docker save -o ./uid2operator.tar uid2operator; docker cp ./uid2operator.tar amazonlinux:/uid2operator.tar +uid2operator.eif: build_artifacts build_configs build/proxies.nitro.yaml build/syslog-ng-client.conf build/syslog-ng-core_4.6.0-1_amd64.deb build/syslog-ng-ose-pub.asc build/entrypoint.sh build/vsockpx build/Dockerfile + cd build; docker build -t uid2operator . --build-arg JAR_VERSION=`cat package.version` --build-arg IMAGE_VERSION=`cat package.version`-`git show --format="%h" --no-patch`; docker save -o ./uid2operator.tar uid2operator; docker cp ./uid2operator.tar amazonlinux:/uid2operator.tar; rm -f ./uid2operator.tar docker exec amazonlinux bash aws_nitro_eif.sh uid2operator -euidoperator.eif: build_artifacts build_configs build/proxies.nitro.yaml build/syslog-ng-client.conf build/syslog-ng-core_4.6.0-1_amd64.deb build/syslog-ng-ose-pub.asc build/entrypoint.sh build/vsockpx build/Dockerfile build/load_config.py build/make_config.py - cd build; docker build -t euidoperator . --build-arg IDENTITY_SCOPE='EUID' --build-arg JAR_VERSION=`cat package.version` --build-arg IMAGE_VERSION=`cat package.version`-`git show --format="%h" --no-patch`; docker save -o ./euidoperator.tar euidoperator; docker cp ./euidoperator.tar amazonlinux:/euidoperator.tar +euidoperator.eif: build_artifacts build_configs build/proxies.nitro.yaml build/syslog-ng-client.conf build/syslog-ng-core_4.6.0-1_amd64.deb build/syslog-ng-ose-pub.asc build/entrypoint.sh build/vsockpx build/Dockerfile + cd build; docker build -t euidoperator . --build-arg IDENTITY_SCOPE='EUID' --build-arg JAR_VERSION=`cat package.version` --build-arg IMAGE_VERSION=`cat package.version`-`git show --format="%h" --no-patch`; docker save -o ./euidoperator.tar euidoperator; docker cp ./euidoperator.tar amazonlinux:/euidoperator.tar; rm -f ./euidoperator.tar docker exec amazonlinux bash aws_nitro_eif.sh euidoperator -################################################################################################################################################################## - -# Config scripts - -build/load_config.py: ./scripts/aws/load_config.py - cp ./scripts/aws/load_config.py ./build/ - -build/make_config.py: ./scripts/aws/make_config.py - cp ./scripts/aws/make_config.py ./build/ ################################################################################################################################################################## @@ -37,26 +28,29 @@ build/make_config.py: ./scripts/aws/make_config.py .PHONY: build_configs -build_configs: build/conf/default-config.json build/conf/prod-uid2-config.json build/conf/integ-uid2-config.json build/conf/prod-euid-config.json build/conf/integ-euid-config.json build/conf/logback.xml +build_configs: build/conf/default-config.json build/conf/euid-integ-config.json build/conf/euid-prod-config.json build/conf/uid2-integ-config.json build/conf/uid2-prod-config.json build/conf/logback.xml build/conf/logback-debug.xml build/conf/default-config.json: build_artifacts ./scripts/aws/conf/default-config.json cp ./scripts/aws/conf/default-config.json ./build/conf/ -build/conf/prod-uid2-config.json: build_artifacts ./scripts/aws/conf/prod-uid2-config.json - cp ./scripts/aws/conf/prod-uid2-config.json ./build/conf/ +build/conf/euid-integ-config.json: build_artifacts ./scripts/aws/conf/euid-integ-config.json + cp ./scripts/aws/conf/euid-integ-config.json ./build/conf/ -build/conf/prod-euid-config.json: build_artifacts ./scripts/aws/conf/prod-euid-config.json - cp ./scripts/aws/conf/prod-euid-config.json ./build/conf/ +build/conf/euid-prod-config.json: build_artifacts ./scripts/aws/conf/euid-prod-config.json + cp ./scripts/aws/conf/euid-prod-config.json ./build/conf/ -build/conf/integ-uid2-config.json: build_artifacts ./scripts/aws/conf/integ-uid2-config.json - cp ./scripts/aws/conf/integ-uid2-config.json ./build/conf/ +build/conf/uid2-integ-config.json: build_artifacts ./scripts/aws/conf/uid2-integ-config.json + cp ./scripts/aws/conf/uid2-integ-config.json ./build/conf/ -build/conf/integ-euid-config.json: build_artifacts ./scripts/aws/conf/integ-euid-config.json - cp ./scripts/aws/conf/integ-euid-config.json ./build/conf/ +build/conf/uid2-prod-config.json: build_artifacts ./scripts/aws/conf/uid2-prod-config.json + cp ./scripts/aws/conf/uid2-prod-config.json ./build/conf/ build/conf/logback.xml: build_artifacts ./scripts/aws/conf/logback.xml cp ./scripts/aws/conf/logback.xml ./build/conf/ +build/conf/logback-debug.xml: build_artifacts ./scripts/aws/conf/logback-debug.xml + cp ./scripts/aws/conf/logback-debug.xml ./build/conf/ + build/Dockerfile: build_artifacts ./scripts/aws/Dockerfile cp ./scripts/aws/Dockerfile ./build/ diff --git a/conf/default-config.json b/conf/default-config.json index 224df8906..cf8024fc1 100644 --- a/conf/default-config.json +++ b/conf/default-config.json @@ -30,11 +30,14 @@ "salts_metadata_path": "salts/metadata.json", "services_metadata_path": "services/metadata.json", "service_links_metadata_path": "service_links/metadata.json", + "cloud_encryption_keys_metadata_path": "cloud_encryption_keys/metadata.json", + "encrypted_files": false, "optout_metadata_path": null, "optout_inmem_cache": false, "enclave_platform": null, "failure_shutdown_wait_hours": 120, "sharing_token_expiry_seconds": 2592000, - "operator_type": "public" - + "operator_type": "public", + "enable_remote_config": false, + "uid_instance_id_prefix": "local-operator" } diff --git a/conf/docker-config.json b/conf/docker-config.json index 648b922a8..213b4f426 100644 --- a/conf/docker-config.json +++ b/conf/docker-config.json @@ -4,7 +4,6 @@ "storage_mock": true, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, - "advertising_token_v3": false, "refresh_token_v3": false, "identity_v3": false, "identity_scope": "uid2", @@ -32,12 +31,17 @@ "salts_metadata_path": "/com.uid2.core/test/salts/metadata.json", "services_metadata_path": "/com.uid2.core/test/services/metadata.json", "service_links_metadata_path": "/com.uid2.core/test/service_links/metadata.json", + "cloud_encryption_keys_metadata_path": "/com.uid2.core/test/cloud_encryption_keys/metadata.json", + "runtime_config_metadata_path": "/com.uid2.core/test/runtime_config/metadata.json", + "encrypted_files": false, "identity_token_expires_after_seconds": 3600, "optout_metadata_path": null, "optout_inmem_cache": false, "enclave_platform": null, "failure_shutdown_wait_hours": 120, "salts_expired_shutdown_hours": 12, - "operator_type": "public" - + "operator_type": "public", + "disable_optout_token": true, + "enable_remote_config": false, + "uid_instance_id_prefix": "local-operator" } diff --git a/conf/integ-config.json b/conf/integ-config.json index f1cf90742..af8243946 100644 --- a/conf/integ-config.json +++ b/conf/integ-config.json @@ -13,7 +13,11 @@ "core_api_token": "trusted-partner-key", "optout_api_token": "test-operator-key", "optout_api_uri": "http://localhost:8081/optout/replicate", + "cloud_encryption_keys_metadata_path": "http://localhost:8088/cloud_encryption_keys/retrieve", + "runtime_config_metadata_path": "http://localhost:8088/operator/config", "salts_expired_shutdown_hours": 12, - "operator_type": "public" - + "operator_type": "public", + "disable_optout_token": true, + "enable_remote_config": false, + "uid_instance_id_prefix": "local-operator" } \ No newline at end of file diff --git a/conf/local-config.json b/conf/local-config.json index f19a4357d..6b407e5c6 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -9,12 +9,11 @@ "salts_metadata_path": "/com.uid2.core/test/salts/metadata.json", "services_metadata_path": "/com.uid2.core/test/services/metadata.json", "service_links_metadata_path": "/com.uid2.core/test/service_links/metadata.json", + "cloud_encryption_keys_metadata_path": "/com.uid2.core/test/cloud_encryption_keys/metadata.json", + "runtime_config_metadata_path": "/com.uid2.core/test/runtime_config/metadata.json", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, - "advertising_token_v3": false, - "advertising_token_v4_percentage": 0, - "site_ids_using_v4_tokens": "", "refresh_token_v3": false, "identity_v3": false, "identity_scope": "uid2", @@ -39,5 +38,9 @@ "key_sharing_endpoint_provide_app_names": true, "client_side_token_generate_log_invalid_http_origins": true, "salts_expired_shutdown_hours": 12, - "operator_type": "public" + "operator_type": "public", + "encrypted_files": false, + "disable_optout_token": true, + "enable_remote_config": false, + "uid_instance_id_prefix": "local-operator" } diff --git a/conf/local-e2e-docker-private-config.json b/conf/local-e2e-docker-private-config.json index ef05b8772..1aca0f7e8 100644 --- a/conf/local-e2e-docker-private-config.json +++ b/conf/local-e2e-docker-private-config.json @@ -11,10 +11,12 @@ "keysets_metadata_path": "http://core:8088/key/keyset/refresh", "keyset_keys_metadata_path": "http://core:8088/key/keyset-keys/refresh", "salts_metadata_path": "http://core:8088/salt/refresh", + "cloud_encryption_keys_metadata_path": "http://core:8088/cloud_encryption_keys/retrieve", + "runtime_config_metadata_path": "http://core:8088/operator/config", + "encrypted_files": true, "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, - "advertising_token_v3": false, "refresh_token_v3": true, "identity_v3": false, "identity_scope": "uid2", @@ -28,5 +30,7 @@ "optout_delta_rotate_interval": 60, "cloud_refresh_interval": 30, "salts_expired_shutdown_hours": 12, - "operator_type": "private" + "operator_type": "private", + "enable_remote_config": false, + "uid_instance_id_prefix": "local-private-operator" } diff --git a/conf/local-e2e-docker-public-config.json b/conf/local-e2e-docker-public-config.json index 60f0abd92..5d0671283 100644 --- a/conf/local-e2e-docker-public-config.json +++ b/conf/local-e2e-docker-public-config.json @@ -13,10 +13,12 @@ "salts_metadata_path": "http://core:8088/salt/refresh", "services_metadata_path": "http://core:8088/services/refresh", "service_links_metadata_path": "http://core:8088/service_links/refresh", + "cloud_encryption_keys_metadata_path": "http://core:8088/cloud_encryption_keys/retrieve", + "runtime_config_metadata_path": "http://core:8088/operator/config", + "encrypted_files": true, "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, - "advertising_token_v3": false, "refresh_token_v3": true, "identity_v3": false, "identity_scope": "uid2", @@ -34,6 +36,8 @@ "optout_status_api_enabled": true, "cloud_refresh_interval": 30, "salts_expired_shutdown_hours": 12, - "operator_type": "public" - + "operator_type": "public", + "disable_optout_token": true, + "enable_remote_config": false, + "uid_instance_id_prefix": "local-public-operator" } diff --git a/conf/local-e2e-private-config.json b/conf/local-e2e-private-config.json index e9d3f8b53..bff451b0a 100644 --- a/conf/local-e2e-private-config.json +++ b/conf/local-e2e-private-config.json @@ -13,10 +13,12 @@ "salts_metadata_path": "http://localhost:8088/salt/refresh", "services_metadata_path": "http://localhost:8088/services/refresh", "service_links_metadata_path": "http://localhost:8088/service_links/refresh", + "cloud_encryption_keys_metadata_path": "http://localhost:8088/cloud_encryption_keys/retrieve", + "runtime_config_metadata_path": "http://localhost:8088/operator/config", + "encrypted_files": true, "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, - "advertising_token_v3": false, "refresh_token_v3": true, "identity_v3": false, "identity_scope": "uid2", @@ -39,6 +41,7 @@ "client_side_token_generate_domain_name_check_enabled": false, "client_side_token_generate_log_invalid_http_origins": true, "salts_expired_shutdown_hours": 12, - "operator_type": "private" - + "operator_type": "private", + "enable_remote_config": false, + "uid_instance_id_prefix": "local-private-operator" } diff --git a/conf/local-e2e-public-config.json b/conf/local-e2e-public-config.json index cb635b103..e97322f40 100644 --- a/conf/local-e2e-public-config.json +++ b/conf/local-e2e-public-config.json @@ -13,10 +13,12 @@ "salts_metadata_path": "http://localhost:8088/salt/refresh", "services_metadata_path": "http://localhost:8088/services/refresh", "service_links_metadata_path": "http://localhost:8088/service_links/refresh", + "cloud_encryption_keys_metadata_path": "http://localhost:8088/cloud_encryption_keys/retrieve", + "runtime_config_metadata_path": "http://localhost:8088/operator/config", + "encrypted_files": true, "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, - "advertising_token_v3": false, "refresh_token_v3": true, "identity_v3": false, "identity_scope": "uid2", @@ -40,6 +42,8 @@ "key_sharing_endpoint_provide_app_names": true, "client_side_token_generate_log_invalid_http_origins": true, "salts_expired_shutdown_hours": 12, - "operator_type": "public" - + "operator_type": "public", + "disable_optout_token": true, + "enable_remote_config": false, + "uid_instance_id_prefix": "local-public-operator" } diff --git a/conf/logback.loki-local.xml b/conf/logback.loki-local.xml deleted file mode 100644 index ff0f0adb1..000000000 --- a/conf/logback.loki-local.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - - http://localhost:3100/loki/api/v1/push - - - - - l=%level h=${HOSTNAME} po=${port_offset:-0} c=%logger{20} t=%thread | %msg %ex - - true - - - - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %ex%n - - - - - - - - - \ No newline at end of file diff --git a/conf/logback.loki.xml b/conf/logback.loki.xml deleted file mode 100644 index d2358c272..000000000 --- a/conf/logback.loki.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - - http://${LOKI_HOSTNAME}:3100/loki/api/v1/push - - - - - l=%level h=${HOSTNAME} po=${port_offset:-0} c=%logger{20} t=%thread | %msg %ex - - true - - - - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %ex%n - - - - - - - - - diff --git a/conf/logback.xml b/conf/logback.xml index 3ad8100ae..bdd1f5726 100644 --- a/conf/logback.xml +++ b/conf/logback.xml @@ -4,12 +4,12 @@ ch.qos.logback.classic.encoder.PatternLayoutEncoder by default --> - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %ex%n + %d{HH:mm:ss.SSS} thread=%thread level=%-5level class=%logger{36} - %msg %ex%n - + - \ No newline at end of file + diff --git a/conf/validator-latest-e2e-docker-public-config.json b/conf/validator-latest-e2e-docker-public-config.json index cabf23380..8990bd5bd 100644 --- a/conf/validator-latest-e2e-docker-public-config.json +++ b/conf/validator-latest-e2e-docker-public-config.json @@ -14,10 +14,11 @@ "salts_metadata_path": "http://core:8088/salt/refresh", "services_metadata_path": "http://core:8088/services/refresh", "service_links_metadata_path": "http://core:8088/service_links/refresh", + "cloud_encryption_keys_metadata_path": "http://core:8088/cloud_encryption_keys/retrieve", + "encrypted_files": true, "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, - "advertising_token_v3": false, "refresh_token_v3": true, "identity_v3": false, "identity_scope": "uid2", @@ -33,6 +34,15 @@ "optout_api_uri": "http://optout:8081/optout/replicate", "optout_delta_rotate_interval": 60, "cloud_refresh_interval": 30, - "operator_type": "public" - + "operator_type": "public", + "runtime_config_store": { + "type": "http", + "config" : { + "url": "http://core:8088/operator/config" + }, + "config_scan_period_ms": 300000 + }, + "disable_optout_token": true, + "enable_remote_config": false, + "uid_instance_id_prefix": "local-public-operator" } diff --git a/pom.xml b/pom.xml index 082c67876..f405a68d9 100644 --- a/pom.xml +++ b/pom.xml @@ -6,11 +6,11 @@ com.uid2 uid2-operator - 5.40.86 + 5.56.43-alpha-201-SNAPSHOT UTF-8 - 4.5.3 + 4.5.13 1.0.22 5.11.2 5.11.2 @@ -20,9 +20,9 @@ 1.12.2 2.1.6 2.1.0 - 2.1.0 + 2.1.13 2.1.0 - 7.19.0 + 10.8.0 ${project.version} 21 21 @@ -32,8 +32,9 @@ - snapshots-repo - https://s01.oss.sonatype.org/content/repositories/snapshots + Central Portal Snapshots + central-portal-snapshots + https://central.sonatype.com/repository/maven-snapshots/ false @@ -162,11 +163,6 @@ logback-classic 1.5.8 - - com.github.loki4j - loki-logback-appender - 1.5.2 - net.logstash.logback logstash-logback-encoder @@ -206,6 +202,12 @@ 4.3.2 test + + org.mockito + mockito-junit-jupiter + 5.12.0 + test + @@ -306,7 +308,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.22.2 + 3.2.5 ${argLine} -Dfile.encoding=${project.build.sourceEncoding} diff --git a/scripts/aws/Dockerfile b/scripts/aws/Dockerfile index e210001c3..67aa17368 100644 --- a/scripts/aws/Dockerfile +++ b/scripts/aws/Dockerfile @@ -31,16 +31,12 @@ COPY ./target/${JAR_NAME}-${JAR_VERSION}-jar-with-dependencies.jar /app/${JAR_NA COPY ./static /app/static COPY ./libjnsm.so /app/lib/ COPY ./vsockpx /app/ -COPY ./make_config.py /app/ COPY ./entrypoint.sh /app/ COPY ./proxies.nitro.yaml /app/ -COPY ./conf/default-config.json /app/conf/ -COPY ./conf/prod-uid2-config.json /app/conf/ -COPY ./conf/integ-uid2-config.json /app/conf/ -COPY ./conf/prod-euid-config.json /app/conf/ -COPY ./conf/integ-euid-config.json /app/conf/ -COPY ./conf/*.xml /app/conf/ -COPY ./syslog-ng-client.conf /etc/syslog-ng/syslog-ng.conf +COPY ./conf/default-config.json /app/conf/ +COPY ./conf/*.json /app/conf/ +COPY ./conf/*.xml /app/conf/ +COPY ./syslog-ng-client.conf /etc/syslog-ng/syslog-ng.conf RUN chmod +x /app/vsockpx && chmod +x /app/entrypoint.sh diff --git a/scripts/aws/EUID_CloudFormation.template.yml b/scripts/aws/EUID_CloudFormation.template.yml index 9c5982488..09fefb18f 100644 --- a/scripts/aws/EUID_CloudFormation.template.yml +++ b/scripts/aws/EUID_CloudFormation.template.yml @@ -12,6 +12,9 @@ Parameters: AllowedValues: - prod - integ + ImageId: + Type: AWS::EC2::Image::Id + Default: ami-example1234567890 TrustNetworkCidr: Description: The IP address range that can be used to SSH and HTTPS to the EC2 instances Type: String @@ -21,7 +24,7 @@ Parameters: AllowedPattern: '(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})' ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x. InstanceType: - Description: EC2 instance type. Minimum 4 vCPUs needed. + Description: EC2 instance type. Minimum 8 vCPUs needed. Type: String Default: m5.2xlarge AllowedValues: @@ -104,20 +107,10 @@ Metadata: default: If choose to false for CustomizeEnclaceResource, enter memory for Enclave in MB EnclaveCPUCount: default: If choose to false for CustomizeEnclaceResource, enter CPU count for Enclave -Mappings: - RegionMap: - eu-central-1: - AMI: ami-xxxxxxxxxxxxxxxxx - eu-west-1: - AMI: ami-xxxxxxxxxxxxxxxxx - eu-west-2: - AMI: ami-xxxxxxxxxxxxxxxxx - eu-west-3: - AMI: ami-xxxxxxxxxxxxxxxxx - eu-south-1: - AMI: ami-xxxxxxxxxxxxxxxxx - eu-north-1: - AMI: ami-xxxxxxxxxxxxxxxxx +Conditions: + IsIntegEnvironment: !Equals + - !Ref DeployToEnvironment + - integ Resources: KMSKey: Type: AWS::KMS::Key @@ -154,13 +147,23 @@ Resources: Description: EUID Token KmsKeyId: !GetAtt KMSKey.Arn Name: !Sub 'euid-config-stack-${AWS::StackName}' - SecretString: !Sub '{ - "api_token":"${APIToken}", - "service_instances":6, - "enclave_cpu_count":6, - "enclave_memory_mb":24576, - "environment":"${DeployToEnvironment}" - }' + SecretString: !Join + - '' + - - '{' + - '"core_base_url": "' + - !If [IsIntegEnvironment, 'https://core.integ.euid.eu', 'https://core.prod.euid.eu'] + - '", "optout_base_url": "' + - !If [IsIntegEnvironment, 'https://optout.integ.euid.eu', 'https://optout.prod.euid.eu'] + - '", "operator_key": "' + - Ref: APIToken + - '"' + - ', "service_instances": 6' + - ', "enclave_cpu_count": 6' + - ', "enclave_memory_mb": 24576' + - ', "environment": "' + - Ref: DeployToEnvironment + - '"' + - '}' WorkerRole: Type: 'AWS::IAM::Role' Properties: @@ -239,7 +242,7 @@ Resources: VolumeType: gp3 IamInstanceProfile: Name: !Ref WorkerInstanceProfile - ImageId: !FindInMap [RegionMap, !Ref 'AWS::Region', AMI] + ImageId: !Ref ImageId InstanceType: !Ref InstanceType EnclaveOptions: Enabled: true @@ -253,6 +256,11 @@ Resources: sudo yum update -y --security while ! nc -z localhost 80;do sleep 10;done /opt/aws/bin/cfn-signal -e 0 --stack ${AWS::StackName} --resource AutoScalingGroup --region ${AWS::Region} + MetadataOptions: + HttpEndpoint: enabled + HttpTokens: required # Enforces IMDSv2 + HttpPutResponseHopLimit: 1 + InstanceMetadataTags: enabled AutoScalingGroup: Type: AWS::AutoScaling::AutoScalingGroup DependsOn: diff --git a/scripts/aws/UID_CloudFormation.template.yml b/scripts/aws/UID_CloudFormation.template.yml index 711d1ab0e..7a7f3d6a0 100644 --- a/scripts/aws/UID_CloudFormation.template.yml +++ b/scripts/aws/UID_CloudFormation.template.yml @@ -12,6 +12,9 @@ Parameters: AllowedValues: - prod - integ + ImageId: + Type: AWS::EC2::Image::Id + Default: ami-example1234567890 TrustNetworkCidr: Description: The IP address range that can be used to SSH and HTTPS to the EC2 instances Type: String @@ -21,7 +24,7 @@ Parameters: AllowedPattern: '(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})' ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x. InstanceType: - Description: EC2 instance type. Minimum 4 vCPUs needed. + Description: EC2 instance type. Minimum 8 vCPUs needed. Type: String Default: m5.2xlarge AllowedValues: @@ -104,48 +107,10 @@ Metadata: default: If choose to false for CustomizeEnclaceResource, enter memory for Enclave in MB EnclaveCPUCount: default: If choose to false for CustomizeEnclaceResource, enter CPU count for Enclave -Mappings: - RegionMap: - us-east-1: - AMI: ami-xxxxxxxxxxxxxxxxx - us-east-2: - AMI: ami-xxxxxxxxxxxxxxxxx - us-west-1: - AMI: ami-xxxxxxxxxxxxxxxxx - us-west-2: - AMI: ami-xxxxxxxxxxxxxxxxx - eu-central-1: - AMI: ami-xxxxxxxxxxxxxxxxx - eu-west-1: - AMI: ami-xxxxxxxxxxxxxxxxx - eu-west-2: - AMI: ami-xxxxxxxxxxxxxxxxx - eu-west-3: - AMI: ami-xxxxxxxxxxxxxxxxx - eu-south-1: - AMI: ami-xxxxxxxxxxxxxxxxx - eu-north-1: - AMI: ami-xxxxxxxxxxxxxxxxx - me-south-1: - AMI: ami-xxxxxxxxxxxxxxxxx - ap-east-1: - AMI: ami-xxxxxxxxxxxxxxxxx - ap-south-1: - AMI: ami-xxxxxxxxxxxxxxxxx - ap-northeast-1: - AMI: ami-xxxxxxxxxxxxxxxxx - ap-northeast-2: - AMI: ami-xxxxxxxxxxxxxxxxx - ap-southeast-1: - AMI: ami-xxxxxxxxxxxxxxxxx - ap-southeast-2: - AMI: ami-xxxxxxxxxxxxxxxxx - sa-east-1: - AMI: ami-xxxxxxxxxxxxxxxxx - ca-central-1: - AMI: ami-xxxxxxxxxxxxxxxxx - af-south-1: - AMI: ami-xxxxxxxxxxxxxxxxx +Conditions: + IsIntegEnvironment: !Equals + - !Ref DeployToEnvironment + - integ Resources: KMSKey: Type: AWS::KMS::Key @@ -182,13 +147,23 @@ Resources: Description: UID2 Token KmsKeyId: !GetAtt KMSKey.Arn Name: !Sub 'uid2-config-stack-${AWS::StackName}' - SecretString: !Sub '{ - "api_token":"${APIToken}", - "service_instances":6, - "enclave_cpu_count":6, - "enclave_memory_mb":24576, - "environment":"${DeployToEnvironment}" - }' + SecretString: !Join + - '' + - - '{' + - '"core_base_url": "' + - !If [IsIntegEnvironment, 'https://core-integ.uidapi.com', 'https://core-prod.uidapi.com'] + - '", "optout_base_url": "' + - !If [IsIntegEnvironment, 'https://optout-integ.uidapi.com', 'https://optout-prod.uidapi.com'] + - '", "operator_key": "' + - Ref: APIToken + - '"' + - ', "service_instances": 6' + - ', "enclave_cpu_count": 6' + - ', "enclave_memory_mb": 24576' + - ', "environment": "' + - Ref: DeployToEnvironment + - '"' + - '}' WorkerRole: Type: 'AWS::IAM::Role' Properties: @@ -267,7 +242,7 @@ Resources: VolumeType: gp3 IamInstanceProfile: Name: !Ref WorkerInstanceProfile - ImageId: !FindInMap [RegionMap, !Ref 'AWS::Region', AMI] + ImageId: !Ref ImageId InstanceType: !Ref InstanceType EnclaveOptions: Enabled: true @@ -281,6 +256,11 @@ Resources: sudo yum update -y --security while ! nc -z localhost 80;do sleep 10;done /opt/aws/bin/cfn-signal -e 0 --stack ${AWS::StackName} --resource AutoScalingGroup --region ${AWS::Region} + MetadataOptions: + HttpEndpoint: enabled + HttpTokens: required # Enforces IMDSv2 + HttpPutResponseHopLimit: 1 + InstanceMetadataTags: enabled AutoScalingGroup: Type: AWS::AutoScaling::AutoScalingGroup DependsOn: diff --git a/scripts/aws/conf/default-config.json b/scripts/aws/conf/default-config.json index 6db89fd29..6b910a63c 100644 --- a/scripts/aws/conf/default-config.json +++ b/scripts/aws/conf/default-config.json @@ -30,11 +30,13 @@ "service_links_metadata_path": "service_links/metadata.json", "optout_metadata_path": null, "optout_inmem_cache": false, - "enclave_platform": null, + "enclave_platform": "aws-nitro", "failure_shutdown_wait_hours": 120, "sharing_token_expiry_seconds": 2592000, "validate_service_links": false, - "advertising_token_v4_percentage": 100, - "site_ids_using_v4_tokens": "", - "operator_type": "private" -} + "identity_token_expires_after_seconds": 86400, + "refresh_token_expires_after_seconds": 2592000, + "refresh_identity_token_after_seconds": 3600, + "operator_type": "private", + "uid_instance_id_prefix": "unknown" +} \ No newline at end of file diff --git a/scripts/aws/conf/integ-euid-config.json b/scripts/aws/conf/euid-integ-config.json similarity index 86% rename from scripts/aws/conf/integ-euid-config.json rename to scripts/aws/conf/euid-integ-config.json index 45d3dbe94..702bc9ff2 100644 --- a/scripts/aws/conf/integ-euid-config.json +++ b/scripts/aws/conf/euid-integ-config.json @@ -10,6 +10,7 @@ "optout_metadata_path": "https://optout.integ.euid.eu/optout/refresh", "core_attest_url": "https://core.integ.euid.eu/attest", "optout_api_uri": "https://optout.integ.euid.eu/optout/replicate", + "cloud_encryption_keys_metadata_path": "https://core.integ.euid.eu/cloud_encryption_keys/retrieve", "optout_s3_folder": "optout/", - "allow_legacy_api": false -} + "identity_scope": "euid" +} \ No newline at end of file diff --git a/scripts/aws/conf/prod-euid-config.json b/scripts/aws/conf/euid-prod-config.json similarity index 80% rename from scripts/aws/conf/prod-euid-config.json rename to scripts/aws/conf/euid-prod-config.json index c7784a381..b9f043485 100644 --- a/scripts/aws/conf/prod-euid-config.json +++ b/scripts/aws/conf/euid-prod-config.json @@ -10,6 +10,7 @@ "service_links_metadata_path": "https://core.prod.euid.eu/service_links/refresh", "optout_metadata_path": "https://optout.prod.euid.eu/optout/refresh", "core_attest_url": "https://core.prod.euid.eu/attest", + "cloud_encryption_keys_metadata_path": "https://core.prod.euid.eu/cloud_encryption_keys/retrieve", "core_api_token": "your-api-token", "optout_s3_path_compat": false, "optout_api_uri": "https://optout.prod.euid.eu/optout/replicate", @@ -22,11 +23,16 @@ "identity_token_expires_after_seconds": 259200, "refresh_token_expires_after_seconds": 2592000, "refresh_identity_token_after_seconds": 3600, - "allow_legacy_api": false, "identity_scope": "euid", - "advertising_token_v3": true, "refresh_token_v3": true, - "enable_phone_support": false, + "enable_phone_support": true, "enable_v1_phone_support": false, - "enable_v2_encryption": true -} + "enable_v2_encryption": true, + "runtime_config_store": { + "type": "http", + "config" : { + "url": "https://core.prod.euid.eu/operator/config" + }, + "config_scan_period_ms": 300000 + } +} \ No newline at end of file diff --git a/scripts/aws/conf/logback-debug.xml b/scripts/aws/conf/logback-debug.xml new file mode 100644 index 000000000..c012f8d25 --- /dev/null +++ b/scripts/aws/conf/logback-debug.xml @@ -0,0 +1,15 @@ + + + + + + + REDACTED - S3 + \S+s3\.amazonaws\.com\/\S*X-Amz-Security-Token=\S+ + + + + + + + \ No newline at end of file diff --git a/scripts/aws/conf/integ-uid2-config.json b/scripts/aws/conf/uid2-integ-config.json similarity index 87% rename from scripts/aws/conf/integ-uid2-config.json rename to scripts/aws/conf/uid2-integ-config.json index a7272a26a..3c267a655 100644 --- a/scripts/aws/conf/integ-uid2-config.json +++ b/scripts/aws/conf/uid2-integ-config.json @@ -1,15 +1,16 @@ { + "core_attest_url": "https://core-integ.uidapi.com/attest", + "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate", "sites_metadata_path": "https://core-integ.uidapi.com/sites/refresh", "clients_metadata_path": "https://core-integ.uidapi.com/clients/refresh", + "client_side_keypairs_metadata_path": "https://core-integ.uidapi.com/client_side_keypairs/refresh", "keysets_metadata_path": "https://core-integ.uidapi.com/key/keyset/refresh", "keyset_keys_metadata_path": "https://core-integ.uidapi.com/key/keyset-keys/refresh", - "client_side_keypairs_metadata_path": "https://core-integ.uidapi.com/client_side_keypairs/refresh", "salts_metadata_path": "https://core-integ.uidapi.com/salt/refresh", "services_metadata_path": "https://core-integ.uidapi.com/services/refresh", "service_links_metadata_path": "https://core-integ.uidapi.com/service_links/refresh", "optout_metadata_path": "https://optout-integ.uidapi.com/optout/refresh", - "core_attest_url": "https://core-integ.uidapi.com/attest", - "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate", + "cloud_encryption_keys_metadata_path": "https://core-integ.uidapi.com/cloud_encryption_keys/retrieve", "optout_s3_folder": "uid-optout-integ/", - "allow_legacy_api": false + "identity_scope": "uid2" } diff --git a/scripts/aws/conf/prod-uid2-config.json b/scripts/aws/conf/uid2-prod-config.json similarity index 81% rename from scripts/aws/conf/prod-uid2-config.json rename to scripts/aws/conf/uid2-prod-config.json index 5da450033..25ad8c7af 100644 --- a/scripts/aws/conf/prod-uid2-config.json +++ b/scripts/aws/conf/uid2-prod-config.json @@ -10,6 +10,7 @@ "service_links_metadata_path": "https://core-prod.uidapi.com/service_links/refresh", "optout_metadata_path": "https://optout-prod.uidapi.com/optout/refresh", "core_attest_url": "https://core-prod.uidapi.com/attest", + "cloud_encryption_keys_metadata_path": "https://core-prod.uidapi.com/cloud_encryption_keys/retrieve", "core_api_token": "your-api-token", "optout_s3_path_compat": false, "optout_api_uri": "https://optout-prod.uidapi.com/optout/replicate", @@ -19,8 +20,15 @@ "optout_synthetic_logs_count": 0, "optout_inmem_cache": true, "optout_s3_folder": "optout-v2/", + "identity_scope": "uid2", "identity_token_expires_after_seconds": 259200, "refresh_token_expires_after_seconds": 2592000, "refresh_identity_token_after_seconds": 3600, - "allow_legacy_api": false -} + "runtime_config_store": { + "type": "http", + "config" : { + "url": "https://core-prod.uidapi.com/operator/config" + }, + "config_scan_period_ms": 300000 + } +} \ No newline at end of file diff --git a/scripts/aws/config-server/app.py b/scripts/aws/config-server/app.py index edb80e4d5..c0c94fc63 100644 --- a/scripts/aws/config-server/app.py +++ b/scripts/aws/config-server/app.py @@ -10,25 +10,6 @@ def get_config(): with open('/etc/secret/secret-value/config', 'r') as secret_file: secret_value = secret_file.read().strip() secret_value_json = json.loads(secret_value) - secret_value_json["environment"] = secret_value_json["environment"].lower() - if "core_base_url" in secret_value_json: - secret_value_json["core_base_url"] = secret_value_json["core_base_url"].lower() - if "optout_base_url" in secret_value_json: - secret_value_json["optout_base_url"] = secret_value_json["optout_base_url"].lower() - if "operator_type" in secret_value_json and secret_value_json["operator_type"].lower() == "public": - mount_path = '/etc/config/config-values' - if os.path.exists(mount_path): - config_keys = [f for f in os.listdir(mount_path) if os.path.isfile(os.path.join(mount_path, f))] - config = {} - for k in config_keys: - with open(os.path.join(mount_path, k), 'r') as value: - config[k] = value.read() - try: - json.loads(config[k]) - config[k] = json.loads(config[k]) - except Exception: - pass - secret_value_json.update(config) return json.dumps(secret_value_json) except Exception as e: return str(e), 500 diff --git a/scripts/aws/config-server/requirements.txt b/scripts/aws/config-server/requirements.txt index 57652a258..28938c283 100644 --- a/scripts/aws/config-server/requirements.txt +++ b/scripts/aws/config-server/requirements.txt @@ -1,3 +1,2 @@ Flask==2.3.2 -Werkzeug==3.0.3 -setuptools==70.0.0 +Werkzeug==3.0.6 \ No newline at end of file diff --git a/scripts/aws/ec2.py b/scripts/aws/ec2.py new file mode 100644 index 000000000..0fc1c5f2c --- /dev/null +++ b/scripts/aws/ec2.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 + +import boto3 +import json +import os +import subprocess +import re +import multiprocessing +import requests +import signal +import argparse +import logging +from botocore.exceptions import ClientError, NoCredentialsError +from typing import Dict, List +import sys +import time +import yaml +logging.basicConfig(level=logging.INFO) +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, InstanceProfileMissingError, OperatorKeyNotFoundError, ConfigurationValueError, ConfidentialComputeStartupError + +class AWSConfidentialComputeConfig(ConfidentialComputeConfig): + enclave_memory_mb: int + enclave_cpu_count: int + core_api_token: str + optout_api_token: str + +class AuxiliaryConfig: + FLASK_PORT: str = "27015" + LOCALHOST: str = "127.0.0.1" + AWS_METADATA: str = "169.254.169.254" + + @classmethod + def get_socks_url(cls) -> str: + return f"socks5://{cls.LOCALHOST}:3306" + + @classmethod + def get_config_url(cls) -> str: + return f"http://{cls.LOCALHOST}:{cls.FLASK_PORT}/getConfig" + + @classmethod + def get_user_data_url(cls) -> str: + return f"http://{cls.AWS_METADATA}/latest/user-data" + + @classmethod + def get_token_url(cls) -> str: + return f"http://{cls.AWS_METADATA}/latest/api/token" + + @classmethod + def get_meta_url(cls) -> str: + return f"http://{cls.AWS_METADATA}/latest/dynamic/instance-identity/document" + + +class EC2(ConfidentialCompute): + + def __init__(self): + super().__init__() + + def __get_aws_token(self) -> str: + """Fetches a temporary AWS EC2 metadata token.""" + try: + response = requests.put( + AuxiliaryConfig.get_token_url(), headers={"X-aws-ec2-metadata-token-ttl-seconds": "3600"}, timeout=2 + ) + return response.text + except requests.RequestException as e: + raise RuntimeError(f"Failed to fetch AWS token: {e}") + + def __get_current_region(self) -> str: + """Fetches the current AWS region from EC2 instance metadata.""" + token = self.__get_aws_token() + headers = {"X-aws-ec2-metadata-token": token} + try: + response = requests.get(AuxiliaryConfig.get_meta_url(), headers=headers, timeout=2) + response.raise_for_status() + return response.json()["region"] + except requests.RequestException as e: + raise RuntimeError(f"Failed to fetch region: {e}") + + def __get_ec2_instance_info(self) -> tuple[str, str]: + """Fetches the instance ID, and AMI ID from EC2 metadata.""" + token = self.__get_aws_token() + headers = {"X-aws-ec2-metadata-token": token} + try: + response = requests.get(AuxiliaryConfig.get_meta_url(), headers=headers, timeout=2) + response.raise_for_status() + data = response.json() + instance_id = data["instanceId"] + ami_id = data["imageId"] + return instance_id, ami_id + + except requests.RequestException as e: + raise RuntimeError(f"Failed to fetch instance info: {e}") + + def __validate_aws_specific_config(self): + if "enclave_memory_mb" in self.configs or "enclave_cpu_count" in self.configs: + max_capacity = self.__get_max_capacity() + if self.configs.get('enclave_memory_mb') < 11000 or self.configs.get('enclave_memory_mb') > max_capacity.get('enclave_memory_mb'): + raise ConfigurationValueError(self.__class__.__name__, f"enclave_memory_mb must be in range 11000 and {max_capacity.get('enclave_memory_mb')}") + if self.configs.get('enclave_cpu_count') < 2 or self.configs.get('enclave_cpu_count') > max_capacity.get('enclave_cpu_count'): + raise ConfigurationValueError(self.__class__.__name__, f"enclave_cpu_count must be in range 2 and {max_capacity.get('enclave_cpu_count')}") + + def _set_confidential_config(self, secret_identifier: str) -> None: + """Fetches a secret value from AWS Secrets Manager and adds defaults""" + + def add_defaults(configs: Dict[str, any]) -> AWSConfidentialComputeConfig: + """Adds default values to configuration if missing. Sets operator_key if only api_token is specified for backward compatibility """ + default_capacity = self.__get_max_capacity() + configs.setdefault("operator_key", configs.get("api_token")) + configs.setdefault("enclave_memory_mb", default_capacity["enclave_memory_mb"]) + configs.setdefault("enclave_cpu_count", default_capacity["enclave_cpu_count"]) + configs.setdefault("debug_mode", False) + configs.setdefault("core_api_token", configs.get("operator_key")) + configs.setdefault("optout_api_token", configs.get("operator_key")) + return configs + + region = self.__get_current_region() + logging.info(f"Running in {region}") + client = boto3.client("secretsmanager", region_name=region) + try: + self.configs = add_defaults(json.loads(client.get_secret_value(SecretId=secret_identifier)["SecretString"])) + instance_id, ami_id = self.__get_ec2_instance_info() + self.configs.setdefault("uid_instance_id_prefix", self.get_uid_instance_id(identifier=instance_id,version=ami_id)) + self.__validate_aws_specific_config() + except json.JSONDecodeError as e: + raise OperatorKeyNotFoundError(self.__class__.__name__, f"Can not parse secret {secret_identifier} in {region}") + except NoCredentialsError as _: + raise InstanceProfileMissingError(self.__class__.__name__) + except ClientError as _: + raise OperatorKeyNotFoundError(self.__class__.__name__, f"Secret Manager {secret_identifier} in {region}") + + @staticmethod + def __get_max_capacity(): + try: + with open("/etc/nitro_enclaves/allocator.yaml", "r") as file: + nitro_config = yaml.safe_load(file) + return {"enclave_memory_mb": nitro_config['memory_mib'], "enclave_cpu_count": nitro_config['cpu_count']} + except Exception as e: + raise RuntimeError("/etc/nitro_enclaves/allocator.yaml does not have CPU, memory allocated") + + def __setup_vsockproxy(self) -> None: + logging.info("Sets up the vSock proxy service") + thread_count = (multiprocessing.cpu_count() + 1) // 2 + command = [ + "/usr/bin/vsockpx", "-c", "/etc/uid2operator/proxy.yaml", + "--workers", str(thread_count), "--daemon" + ] + + debug_command = [ + "/usr/bin/vsockpx", "-c", "/etc/uid2operator/proxy.yaml", + "--workers", str(thread_count), "--log-level", "0" + ] + + self.run_service([command, debug_command], "vsock_proxy") + + def __run_config_server(self) -> None: + logging.info("Starts the Flask configuration server") + os.makedirs("/etc/secret/secret-value", exist_ok=True) + config_path = "/etc/secret/secret-value/config" + + # Save configs to a file + with open(config_path, 'w') as config_file: + json.dump(self.configs, config_file) + + os.chdir("/opt/uid2operator/config-server") + command = ["./bin/flask", "run", "--host", AuxiliaryConfig.LOCALHOST, "--port", AuxiliaryConfig.FLASK_PORT] + + self.run_service([command, command], "flask_config_server", separate_process=True) + + def __run_socks_proxy(self) -> None: + logging.info("Starts the SOCKS proxy service") + command = ["sockd", "-D"] + + # -d specifies debug level + debug_command = ["sockd", "-d", "0"] + + self.run_service([command, debug_command], "socks_proxy") + + def run_service(self, command: List[List[str]], log_filename: str, separate_process: bool = False) -> None: + """ + Runs a service command with logging if debug_mode is enabled. + + :param command: command[0] regular command, command[1] debug mode command + :param log_filename: Base name of the log file (e.g., "flask_config_server", "socks_proxy", "vsock_proxy") + :param separate_process: Whether to run in a separate process + """ + log_file = f"/var/log/{log_filename}.log" + + if self.configs.get("debug_mode") is True: + + # Remove old log file to start fresh + if os.path.exists(log_file): + os.remove(log_file) + + # Set up logging + logging.basicConfig( + filename=log_file, + filemode="w", + level=logging.DEBUG, + format="%(asctime)s %(levelname)s: %(message)s" + ) + + logging.info(f"Debug mode is on, logging into {log_file}") + + # Run debug mode command + with open(log_file, "a") as log: + self.run_command(command[1], separate_process=True, stdout=log, stderr=log) + else: + # Run regular command, possibly daemon + self.run_command(command[0], separate_process=separate_process) + + def __get_secret_name_from_userdata(self) -> str: + """Extracts the secret name from EC2 user data.""" + logging.info("Extracts the secret name from EC2 user data") + token = self.__get_aws_token() + response = requests.get(AuxiliaryConfig.get_user_data_url(), headers={"X-aws-ec2-metadata-token": token}) + user_data = response.text + + with open("/opt/uid2operator/identity_scope.txt") as file: + identity_scope = file.read().strip() + + default_name = f"{identity_scope.lower()}-operator-config-key" + hardcoded_value = f"{identity_scope.upper()}_CONFIG_SECRET_KEY" + match = re.search(rf'^export {hardcoded_value}="(.+?)"$', user_data, re.MULTILINE) + return match.group(1) if match else default_name + + def _setup_auxiliaries(self) -> None: + """Sets up the vsock tunnel, socks proxy and flask server""" + self.__setup_vsockproxy() + self.__run_config_server() + self.__run_socks_proxy() + logging.info("Finished setting up all auxiliaries") + + def _validate_auxiliaries(self) -> None: + """Validates connection to flask server direct and through socks proxy.""" + logging.info("Validating auxiliaries") + try: + for attempt in range(10): + try: + response = requests.get(AuxiliaryConfig.get_config_url()) + logging.info("Config server is reachable") + break + except requests.exceptions.ConnectionError as e: + logging.error(f"Connecting to config server, attempt {attempt + 1} failed with ConnectionError: {e}") + time.sleep(1) + else: + raise RuntimeError(f"Config server unreachable") + response.raise_for_status() + except requests.RequestException as e: + raise RuntimeError(f"Failed to get config from config server: {e}") + proxies = {"http": AuxiliaryConfig.get_socks_url(), "https": AuxiliaryConfig.get_socks_url()} + try: + response = requests.get(AuxiliaryConfig.get_config_url(), proxies=proxies) + response.raise_for_status() + except requests.RequestException as e: + raise RuntimeError(f"Cannot connect to config server via SOCKS proxy: {e}") + logging.info("Connectivity check to config server passes") + + def __run_nitro_enclave(self): + command = [ + "nitro-cli", "run-enclave", + "--eif-path", "/opt/uid2operator/uid2operator.eif", + "--memory", str(self.configs["enclave_memory_mb"]), + "--cpu-count", str(self.configs["enclave_cpu_count"]), + "--enclave-cid", "42", + "--enclave-name", "uid2operator" + ] + if self.configs.get('debug_mode', False): + logging.info("Running nitro in debug_mode") + command += ["--debug-mode", "--attach-console"] + self.run_command(command, separate_process=False) + + def run_compute(self) -> None: + """Main execution flow for confidential compute.""" + secret_manager_key = self.__get_secret_name_from_userdata() + self._set_confidential_config(secret_manager_key) + logging.info(f"Fetched configs from {secret_manager_key}") + if not self.configs.get("skip_validations"): + self.validate_configuration() + self._setup_auxiliaries() + self._validate_auxiliaries() + self.__run_nitro_enclave() + + def cleanup(self) -> None: + """Terminates the Nitro Enclave and auxiliary processes.""" + try: + self.run_command(["nitro-cli", "terminate-enclave", "--all"]) + self.__kill_auxiliaries() + except subprocess.SubprocessError as e: + raise (f"Error during cleanup: {e}") + + def __kill_auxiliaries(self) -> None: + """Kills all auxiliary processes spawned.""" + for process_name in ["vsockpx", "sockd", "flask"]: + try: + result = subprocess.run(["pgrep", "-f", process_name], stdout=subprocess.PIPE, text=True, check=False) + if result.stdout.strip(): + for pid in result.stdout.strip().split("\n"): + os.kill(int(pid), signal.SIGKILL) + logging.info(f"Killed process '{process_name}'.") + else: + logging.info(f"No process named '{process_name}' found.") + except Exception as e: + logging.error(f"Error killing process '{process_name}': {e}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Manage EC2-based confidential compute workflows.") + parser.add_argument("-o", "--operation", choices=["stop", "start"], default="start", help="Operation to perform.") + args = parser.parse_args() + + try: + ec2 = EC2() + if args.operation == "stop": + ec2.cleanup() + else: + ec2.run_compute() + except ConfidentialComputeStartupError as e: + logging.error(f"Failed starting up Confidential Compute. Please checks the logs for errors and retry {e}") + except Exception as e: + logging.error(f"Unexpected failure while starting up Confidential Compute. Please contact UID support team with this log {e}") + diff --git a/scripts/aws/eks-pod/entrypoint.sh b/scripts/aws/eks-pod/entrypoint.sh index c506d6cbf..2dc0483e2 100644 --- a/scripts/aws/eks-pod/entrypoint.sh +++ b/scripts/aws/eks-pod/entrypoint.sh @@ -3,6 +3,7 @@ CID=42 EIF_PATH=/home/uid2operator.eif MEMORY_MB=24576 CPU_COUNT=6 +DEBUG_MODE="false" set -x @@ -26,7 +27,7 @@ function setup_vsockproxy() { echo "setup_vsockproxy" VSOCK_PROXY=${VSOCK_PROXY:-/home/vsockpx} VSOCK_CONFIG=${VSOCK_CONFIG:-/home/proxies.host.yaml} - VSOCK_THREADS=${VSOCK_THREADS:-$(( $(nproc) * 2 )) } + VSOCK_THREADS=${VSOCK_THREADS:-$(( ( $(nproc) + 1 ) / 2 )) } VSOCK_LOG_LEVEL=${VSOCK_LOG_LEVEL:-3} echo "starting vsock proxy at $VSOCK_PROXY with $VSOCK_THREADS worker threads..." $VSOCK_PROXY -c $VSOCK_CONFIG --workers $VSOCK_THREADS --log-level $VSOCK_LOG_LEVEL --daemon @@ -87,12 +88,20 @@ function update_config() { { set +x; } 2>/dev/null; { CPU_COUNT=$(echo $IDENTITY_SERVICE_CONFIG | jq -r '.enclave_cpu_count'); set -x; } { set +x; } 2>/dev/null; { MEMORY_MB=$(echo $IDENTITY_SERVICE_CONFIG | jq -r '.enclave_memory_mb'); set -x; } fi + + { set +x; } 2>/dev/null; { DEBUG_MODE=$(echo $IDENTITY_SERVICE_CONFIG | jq -r '.debug_mode'); set -x; } + shopt -u nocasematch } function run_enclave() { - echo "starting enclave... --cpu-count $CPU_COUNT --memory $MEMORY_MB --eif-path $EIF_PATH --enclave-cid $CID" - nitro-cli run-enclave --cpu-count $CPU_COUNT --memory $MEMORY_MB --eif-path $EIF_PATH --enclave-cid $CID --enclave-name uid2-operator + if [ "$DEBUG_MODE" == "true" ]; then + echo "starting enclave... --cpu-count $CPU_COUNT --memory $MEMORY_MB --eif-path $EIF_PATH --enclave-cid $CID --debug-mode --attach-console" + nitro-cli run-enclave --cpu-count $CPU_COUNT --memory $MEMORY_MB --eif-path $EIF_PATH --enclave-cid $CID --enclave-name uid2-operator --debug-mode --attach-console + else + echo "starting enclave... --cpu-count $CPU_COUNT --memory $MEMORY_MB --eif-path $EIF_PATH --enclave-cid $CID" + nitro-cli run-enclave --cpu-count $CPU_COUNT --memory $MEMORY_MB --eif-path $EIF_PATH --enclave-cid $CID --enclave-name uid2-operator + fi } echo "starting ..." diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index 32db563fa..6d4fbe15e 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -5,8 +5,10 @@ LOG_FILE="/home/start.txt" set -x -exec > $LOG_FILE -exec 2>&1 +exec &> >(tee -a "$LOG_FILE") + +PARAMETERIZED_CONFIG="/app/conf/config-overrides.json" +OPERATOR_CONFIG="/tmp/final-config.json" set -o pipefail ulimit -n 65536 @@ -14,80 +16,74 @@ ulimit -n 65536 # -- setup loopback device echo "Setting up loopback device..." ifconfig lo 127.0.0.1 +/usr/sbin/syslog-ng --verbose # -- start vsock proxy echo "Starting vsock proxy..." -/app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( $(nproc) * 2 )) --log-level 3 - -# -- setup syslog-ng -echo "Starting syslog-ng..." -/usr/sbin/syslog-ng --verbose - -# -- load config from identity service -echo "Loading config from identity service via proxy..." - -#wait for config service, then download config -OVERRIDES_CONFIG="/app/conf/config-overrides.json" - -RETRY_COUNT=0 -MAX_RETRY=20 -until curl -s -f -o "${OVERRIDES_CONFIG}" -x socks5h://127.0.0.1:3305 http://127.0.0.1:27015/getConfig -do - echo "Waiting for config service to be available" - RETRY_COUNT=$(( RETRY_COUNT + 1)) - if [ $RETRY_COUNT -gt $MAX_RETRY ]; then - echo "Config Server did not return a response. Exiting" +/app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( ( $(nproc) + 3 ) / 4 )) --log-level 3 + +build_parameterized_config() { + curl -s -f -o "${PARAMETERIZED_CONFIG}" -x socks5h://127.0.0.1:3305 http://127.0.0.1:27015/getConfig + REQUIRED_KEYS=("optout_base_url" "core_base_url" "core_api_token" "optout_api_token" "environment" "uid_instance_id_prefix") + for key in "${REQUIRED_KEYS[@]}"; do + if ! jq -e "has(\"${key}\")" "${PARAMETERIZED_CONFIG}" > /dev/null; then + echo "Error: Key '${key}' is missing. Please add it to flask config server" + exit 1 + fi + done + FILTER=$(printf '. | {') + for key in "${REQUIRED_KEYS[@]}"; do + FILTER+="$key: .${key}, " + done + FILTER+="debug_mode: .debug_mode, " + FILTER=${FILTER%, }'}' + jq "${FILTER}" "${PARAMETERIZED_CONFIG}" > "${PARAMETERIZED_CONFIG}.tmp" && mv "${PARAMETERIZED_CONFIG}.tmp" "${PARAMETERIZED_CONFIG}" +} + +build_operator_config() { + CORE_BASE_URL=$(jq -r ".core_base_url" < "${PARAMETERIZED_CONFIG}") + OPTOUT_BASE_URL=$(jq -r ".optout_base_url" < "${PARAMETERIZED_CONFIG}") + DEPLOYMENT_ENVIRONMENT=$(jq -r ".environment" < "${PARAMETERIZED_CONFIG}") + DEBUG_MODE=$(jq -r ".debug_mode" < "${PARAMETERIZED_CONFIG}") + + IDENTITY_SCOPE_LOWER=$(echo "${IDENTITY_SCOPE}" | tr '[:upper:]' '[:lower:]') + DEPLOYMENT_ENVIRONMENT_LOWER=$(echo "${DEPLOYMENT_ENVIRONMENT}" | tr '[:upper:]' '[:lower:]') + DEFAULT_CONFIG="/app/conf/${IDENTITY_SCOPE_LOWER}-${DEPLOYMENT_ENVIRONMENT_LOWER}-config.json" + + jq -s '.[0] * .[1]' "${DEFAULT_CONFIG}" "${PARAMETERIZED_CONFIG}" > "${OPERATOR_CONFIG}" + + if [[ "$DEPLOYMENT_ENVIRONMENT" == "prod" ]]; then + if [[ "$DEBUG_MODE" == "true" ]]; then + echo "Cannot run in DEBUG_MODE in production environment. Exiting." exit 1 + fi fi - sleep 2 -done - -# check the config is valid. Querying for a known missing element (empty) makes jq parse the file, but does not echo the results -if jq empty "${OVERRIDES_CONFIG}"; then - echo "Identity service returned valid config" -else - echo "Failed to get a valid config from identity service" - exit 1 -fi -export DEPLOYMENT_ENVIRONMENT=$(jq -r ".environment" < "${OVERRIDES_CONFIG}") -export CORE_BASE_URL=$(jq -r ".core_base_url" < "${OVERRIDES_CONFIG}") -export OPTOUT_BASE_URL=$(jq -r ".optout_base_url" < "${OVERRIDES_CONFIG}") -echo "DEPLOYMENT_ENVIRONMENT=${DEPLOYMENT_ENVIRONMENT}" -if [ -z "${DEPLOYMENT_ENVIRONMENT}" ]; then - echo "DEPLOYMENT_ENVIRONMENT cannot be empty" - exit 1 -fi -if [ "${DEPLOYMENT_ENVIRONMENT}" != "prod" ] && [ "${DEPLOYMENT_ENVIRONMENT}" != "integ" ]; then - echo "Unrecognized DEPLOYMENT_ENVIRONMENT ${DEPLOYMENT_ENVIRONMENT}" - exit 1 -fi + #TODO: Remove below logic after remote config management is implemented + + if [[ "$DEPLOYMENT_ENVIRONMENT" != "prod" ]]; then + #Allow override of base URL in non-prod environments + CORE_PATTERN="https://core.*uidapi.com" + OPTOUT_PATTERN="https://optout.*uidapi.com" + if [[ "$IDENTITY_SCOPE_LOWER" == "euid" ]]; then + CORE_PATTERN="https://core.*euid.eu" + OPTOUT_PATTERN="https://optout.*euid.eu" + fi + sed -i "s#${CORE_PATTERN}#${CORE_BASE_URL}#g" "${OPERATOR_CONFIG}" + sed -i "s#${OPTOUT_PATTERN}#${OPTOUT_BASE_URL}#g" "${OPERATOR_CONFIG}" + fi + +} -echo "Loading config final..." -export FINAL_CONFIG="/app/conf/config-final.json" -if [ "${IDENTITY_SCOPE}" = "UID2" ]; then - python3 /app/make_config.py /app/conf/prod-uid2-config.json /app/conf/integ-uid2-config.json ${OVERRIDES_CONFIG} "$(nproc)" > ${FINAL_CONFIG} -elif [ "${IDENTITY_SCOPE}" = "EUID" ]; then - python3 /app/make_config.py /app/conf/prod-euid-config.json /app/conf/integ-euid-config.json ${OVERRIDES_CONFIG} "$(nproc)" > ${FINAL_CONFIG} -else - echo "Unrecognized IDENTITY_SCOPE ${IDENTITY_SCOPE}" - exit 1 -fi +build_parameterized_config +build_operator_config -# -- replace base URLs if both CORE_BASE_URL and OPTOUT_BASE_URL are provided -# -- using hardcoded domains is fine because they should not be changed frequently -if [ -n "${CORE_BASE_URL}" ] && [ "${CORE_BASE_URL}" != "null" ] && [ -n "${OPTOUT_BASE_URL}" ] && [ "${OPTOUT_BASE_URL}" != "null" ] && [ "${DEPLOYMENT_ENVIRONMENT}" != "prod" ]; then - echo "Replacing core and optout URLs by ${CORE_BASE_URL} and ${OPTOUT_BASE_URL}..." - sed -i "s#https://core-integ.uidapi.com#${CORE_BASE_URL}#g" "${FINAL_CONFIG}" - sed -i "s#https://core-prod.uidapi.com#${CORE_BASE_URL}#g" "${FINAL_CONFIG}" - sed -i "s#https://core.integ.euid.eu#${CORE_BASE_URL}#g" "${FINAL_CONFIG}" - sed -i "s#https://core.prod.euid.eu#${CORE_BASE_URL}#g" "${FINAL_CONFIG}" +DEBUG_MODE=$(jq -r ".debug_mode" < "${OPERATOR_CONFIG}") +LOGBACK_CONF="./conf/logback.xml" - sed -i "s#https://optout-integ.uidapi.com#${OPTOUT_BASE_URL}#g" "${FINAL_CONFIG}" - sed -i "s#https://optout-prod.uidapi.com#${OPTOUT_BASE_URL}#g" "${FINAL_CONFIG}" - sed -i "s#https://optout.integ.euid.eu#${OPTOUT_BASE_URL}#g" "${FINAL_CONFIG}" - sed -i "s#https://optout.prod.euid.eu#${OPTOUT_BASE_URL}#g" "${FINAL_CONFIG}" +if [[ "$DEBUG_MODE" == "true" ]]; then + LOGBACK_CONF="./conf/logback-debug.xml" fi # -- set pwd to /app so we can find default configs @@ -95,12 +91,14 @@ cd /app # -- start operator echo "Starting Java application..." + java \ -XX:MaxRAMPercentage=95 -XX:-UseCompressedOops -XX:+PrintFlagsFinal \ -Djava.security.egd=file:/dev/./urandom \ -Djava.library.path=/app/lib \ - -Dvertx-config-path="${FINAL_CONFIG}" \ + -Dvertx-config-path="${OPERATOR_CONFIG}" \ -Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory \ - -Dlogback.configurationFile=./conf/logback.xml \ + -Dlogback.configurationFile=${LOGBACK_CONF} \ -Dhttp_proxy=socks5://127.0.0.1:3305 \ -jar /app/"${JAR_NAME}"-"${JAR_VERSION}".jar + diff --git a/scripts/aws/load_config.py b/scripts/aws/load_config.py deleted file mode 100644 index 9f0446a49..000000000 --- a/scripts/aws/load_config.py +++ /dev/null @@ -1,41 +0,0 @@ -import os -import boto3 -import base64 -import json -from botocore.exceptions import ClientError - -secret_name = os.environ['UID2_CONFIG_SECRET_KEY'] -region_name = os.environ['AWS_REGION_NAME'] -aws_access_key_id = os.environ['AWS_ACCESS_KEY_ID'] -secret_key = os.environ['AWS_SECRET_KEY'] -session_token = os.environ['AWS_SESSION_TOKEN'] - -def get_secret(): - session = boto3.session.Session() - client = session.client( - service_name='secretsmanager', - region_name=region_name, - aws_access_key_id = aws_access_key_id, - aws_secret_access_key = secret_key, - aws_session_token = session_token - ) - try: - get_secret_value_response = client.get_secret_value( - SecretId=secret_name - ) - except ClientError as e: - raise e - else: - if 'SecretString' in get_secret_value_response: - secret = get_secret_value_response['SecretString'] - else: - decoded_binary_secret = base64.b64decode(get_secret_value_response['SecretBinary']) - - return secret - -def get_config(): - result = get_secret() - conf = json.loads(result) - print(result) - -get_config() diff --git a/scripts/aws/make_config.py b/scripts/aws/make_config.py deleted file mode 100644 index 5777dce61..000000000 --- a/scripts/aws/make_config.py +++ /dev/null @@ -1,57 +0,0 @@ -import json -import sys - - -def load_json(path): - with open(path, 'r') as f: - return json.load(f) - - -def apply_override(config, overrides, key, type): - value = overrides.get(key) - if value is not None: - config[key] = type(value) - - -config_path = sys.argv[1] -integ_config_path = sys.argv[2] -overrides_path = sys.argv[3] -thread_count = int(sys.argv[4]) - -config = load_json(config_path) -overrides = load_json(overrides_path) - -# set API key -config['core_api_token'] = overrides['api_token'] -config['optout_api_token'] = overrides['api_token'] - -# number of threads -config['service_instances'] = thread_count - -# environment -if overrides.get('environment') == 'integ': - integ_config = load_json(integ_config_path) - apply_override(config, integ_config, 'sites_metadata_path', str) - apply_override(config, integ_config, 'clients_metadata_path', str) - apply_override(config, integ_config, 'keysets_metadata_path', str) - apply_override(config, integ_config, 'keyset_keys_metadata_path', str) - apply_override(config, integ_config, 'client_side_keypairs_metadata_path', str) - apply_override(config, integ_config, 'salts_metadata_path', str) - apply_override(config, integ_config, 'services_metadata_path', str) - apply_override(config, integ_config, 'service_links_metadata_path', str) - apply_override(config, integ_config, 'optout_metadata_path', str) - apply_override(config, integ_config, 'core_attest_url', str) - apply_override(config, integ_config, 'optout_api_uri', str) - apply_override(config, integ_config, 'optout_s3_folder', str) - - -apply_override(config, overrides, 'operator_type', str) -if 'operator_type' in config and config['operator_type'] == 'public': - config.update(overrides) -else: - # allowed overrides - apply_override(config, overrides, 'loki_enabled', bool) - apply_override(config, overrides, 'optout_synthetic_logs_enabled', bool) - apply_override(config, overrides, 'optout_synthetic_logs_count', int) - -print(json.dumps(config)) diff --git a/scripts/aws/pipeline/amazonlinux2023.Dockerfile b/scripts/aws/pipeline/amazonlinux2023.Dockerfile index 2914c9ee3..79bcd66df 100644 --- a/scripts/aws/pipeline/amazonlinux2023.Dockerfile +++ b/scripts/aws/pipeline/amazonlinux2023.Dockerfile @@ -4,8 +4,9 @@ FROM amazonlinux:2023 RUN dnf update -y # systemd is not a hard requirement for Amazon ECS Anywhere, but the installation script currently only supports systemd to run. # Amazon ECS Anywhere can be used without systemd, if you set up your nodes and register them into your ECS cluster **without** the installation script. -RUN dnf -y groupinstall "Development Tools" -RUN dnf -y install systemd vim-common wget git tar libstdc++-static.x86_64 cmake cmake3 aws-nitro-enclaves-cli aws-nitro-enclaves-cli-devel +RUN dnf -y groupinstall "Development Tools" \ + && dnf -y install systemd vim-common wget git tar libstdc++-static.x86_64 cmake cmake3 aws-nitro-enclaves-cli aws-nitro-enclaves-cli-devel \ + && dnf clean all RUN systemctl enable docker @@ -14,12 +15,14 @@ RUN wget https://www.inet.no/dante/files/dante-1.4.3.tar.gz \ && sha256sum --check dante_checksum \ && tar -xf dante-1.4.3.tar.gz \ && cd dante-1.4.3; ./configure; make; cd .. \ - && cp dante-1.4.3/sockd/sockd ./ + && cp dante-1.4.3/sockd/sockd ./ \ + && rm -rf dante-1.4.3 dante-1.4.3.tar.gz RUN git clone https://github.com/IABTechLab/uid2-aws-enclave-vsockproxy.git \ && mkdir uid2-aws-enclave-vsockproxy/build \ && cd uid2-aws-enclave-vsockproxy/build; cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo; make; cd ../.. \ - && cp uid2-aws-enclave-vsockproxy/build/vsock-bridge/src/vsock-bridge ./vsockpx + && cp uid2-aws-enclave-vsockproxy/build/vsock-bridge/src/vsock-bridge ./vsockpx \ + && rm -rf uid2-aws-enclave-vsockproxy COPY ./scripts/aws/pipeline/aws_nitro_eif.sh /aws_nitro_eif.sh diff --git a/scripts/aws/pipeline/aws_nitro_eif.sh b/scripts/aws/pipeline/aws_nitro_eif.sh index 2d8f0216b..904d3f3ea 100644 --- a/scripts/aws/pipeline/aws_nitro_eif.sh +++ b/scripts/aws/pipeline/aws_nitro_eif.sh @@ -10,5 +10,6 @@ while (! docker stats --no-stream >/dev/null 2>&1); do sleep 1 done docker load -i $1.tar +rm -f $1.tar nitro-cli build-enclave --docker-uri $1 --output-file $1.eif nitro-cli describe-eif --eif-path $1.eif | jq -r '.Measurements.PCR0' | xxd -r -p | base64 > pcr0.txt diff --git a/scripts/aws/requirements.txt b/scripts/aws/requirements.txt new file mode 100644 index 000000000..421faba98 --- /dev/null +++ b/scripts/aws/requirements.txt @@ -0,0 +1,4 @@ +requests[socks]==2.32.3 +boto3==1.35.59 +urllib3==1.26.20 +PyYAML===6.0.2 \ No newline at end of file diff --git a/scripts/aws/sockd.conf b/scripts/aws/sockd.conf index 6e8814445..1f903407c 100644 --- a/scripts/aws/sockd.conf +++ b/scripts/aws/sockd.conf @@ -3,10 +3,11 @@ external: ens5 user.notprivileged: ec2-user clientmethod: none socksmethod: none +logoutput: stderr client pass { from: 127.0.0.1/32 to: 127.0.0.1/32 - log: error # connect disconnect iooperation + log: error connect # disconnect iooperation } socks pass { diff --git a/scripts/aws/start.sh b/scripts/aws/start.sh deleted file mode 100644 index 440ae58d7..000000000 --- a/scripts/aws/start.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/bash - -echo "$HOSTNAME" > /etc/uid2operator/HOSTNAME -EIF_PATH=${EIF_PATH:-/opt/uid2operator/uid2operator.eif} -IDENTITY_SCOPE=${IDENTITY_SCOPE:-$(cat /opt/uid2operator/identity_scope.txt)} -CID=${CID:-42} -TOKEN=$(curl --request PUT "http://169.254.169.254/latest/api/token" --header "X-aws-ec2-metadata-token-ttl-seconds: 3600") -USER_DATA=$(curl -s http://169.254.169.254/latest/user-data --header "X-aws-ec2-metadata-token: $TOKEN") -AWS_REGION_NAME=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document/ --header "X-aws-ec2-metadata-token: $TOKEN" | jq -r '.region') -if [ "$IDENTITY_SCOPE" = 'UID2' ]; then - UID2_CONFIG_SECRET_KEY=$([[ "$(echo "${USER_DATA}" | grep UID2_CONFIG_SECRET_KEY=)" =~ ^export\ UID2_CONFIG_SECRET_KEY=\"(.*)\"$ ]] && echo "${BASH_REMATCH[1]}" || echo "uid2-operator-config-key") -elif [ "$IDENTITY_SCOPE" = 'EUID' ]; then - UID2_CONFIG_SECRET_KEY=$([[ "$(echo "${USER_DATA}" | grep EUID_CONFIG_SECRET_KEY=)" =~ ^export\ EUID_CONFIG_SECRET_KEY=\"(.*)\"$ ]] && echo "${BASH_REMATCH[1]}" || echo "euid-operator-config-key") -else - echo "Unrecognized IDENTITY_SCOPE $IDENTITY_SCOPE" - exit 1 -fi -CORE_BASE_URL=$([[ "$(echo "${USER_DATA}" | grep CORE_BASE_URL=)" =~ ^export\ CORE_BASE_URL=\"(.*)\"$ ]] && echo "${BASH_REMATCH[1]}" || echo "") -OPTOUT_BASE_URL=$([[ "$(echo "${USER_DATA}" | grep OPTOUT_BASE_URL=)" =~ ^export\ OPTOUT_BASE_URL=\"(.*)\"$ ]] && echo "${BASH_REMATCH[1]}" || echo "") - -echo "UID2_CONFIG_SECRET_KEY=${UID2_CONFIG_SECRET_KEY}" -echo "CORE_BASE_URL=${CORE_BASE_URL}" -echo "OPTOUT_BASE_URL=${OPTOUT_BASE_URL}" -echo "AWS_REGION_NAME=${AWS_REGION_NAME}" - -function terminate_old_enclave() { - ENCLAVE_ID=$(nitro-cli describe-enclaves | jq -r ".[0].EnclaveID") - [ "$ENCLAVE_ID" != "null" ] && nitro-cli terminate-enclave --enclave-id ${ENCLAVE_ID} -} - -function config_aws() { - aws configure set default.region $AWS_REGION_NAME -} - -function default_cpu() { - target=$(( $(nproc) * 3 / 4 )) - if [ $target -lt 2 ]; then - target="2" - fi - echo $target -} - -function default_mem() { - target=$(( $(grep MemTotal /proc/meminfo | awk '{print $2}') * 3 / 4000 )) - if [ $target -lt 24576 ]; then - target="24576" - fi - echo $target -} - -function read_allocation() { - USER_CUSTOMIZED=$(aws secretsmanager get-secret-value --secret-id "$UID2_CONFIG_SECRET_KEY" | jq -r '.SecretString' | jq -r '.customize_enclave') - shopt -s nocasematch - if [ "$USER_CUSTOMIZED" = "true" ]; then - echo "Applying user customized CPU/Mem allocation..." - CPU_COUNT=${CPU_COUNT:-$(aws secretsmanager get-secret-value --secret-id "$UID2_CONFIG_SECRET_KEY" | jq -r '.SecretString' | jq -r '.enclave_cpu_count')} - MEMORY_MB=${MEMORY_MB:-$(aws secretsmanager get-secret-value --secret-id "$UID2_CONFIG_SECRET_KEY" | jq -r '.SecretString' | jq -r '.enclave_memory_mb')} - else - echo "Applying default CPU/Mem allocation..." - CPU_COUNT=6 - MEMORY_MB=24576 - fi - shopt -u nocasematch -} - - -function update_allocation() { - ALLOCATOR_YAML=/etc/nitro_enclaves/allocator.yaml - if [ -z "$CPU_COUNT" ] || [ -z "$MEMORY_MB" ]; then - echo 'No CPU_COUNT or MEMORY_MB set, cannot start enclave' - exit 1 - fi - echo "updating allocator: CPU_COUNT=$CPU_COUNT, MEMORY_MB=$MEMORY_MB..." - systemctl stop nitro-enclaves-allocator.service - sed -r "s/^(\s*memory_mib\s*:\s*).*/\1$MEMORY_MB/" -i $ALLOCATOR_YAML - sed -r "s/^(\s*cpu_count\s*:\s*).*/\1$CPU_COUNT/" -i $ALLOCATOR_YAML - systemctl start nitro-enclaves-allocator.service && systemctl enable nitro-enclaves-allocator.service - echo "nitro-enclaves-allocator restarted" -} - -function setup_vsockproxy() { - VSOCK_PROXY=${VSOCK_PROXY:-/usr/bin/vsockpx} - VSOCK_CONFIG=${VSOCK_CONFIG:-/etc/uid2operator/proxy.yaml} - VSOCK_THREADS=${VSOCK_THREADS:-$(( $(nproc) * 2 )) } - VSOCK_LOG_LEVEL=${VSOCK_LOG_LEVEL:-3} - echo "starting vsock proxy at $VSOCK_PROXY with $VSOCK_THREADS worker threads..." - $VSOCK_PROXY -c $VSOCK_CONFIG --workers $VSOCK_THREADS --log-level $VSOCK_LOG_LEVEL --daemon - echo "vsock proxy now running in background." -} - -function setup_dante() { - sockd -D -} - -function run_config_server() { - mkdir -p /etc/secret/secret-value - { - set +x; # Disable tracing within this block - 2>/dev/null; - SECRET_JSON=$(aws secretsmanager get-secret-value --secret-id "$UID2_CONFIG_SECRET_KEY" | jq -r '.SecretString') - echo "${SECRET_JSON}" > /etc/secret/secret-value/config; - } - echo $(jq ".core_base_url = \"$CORE_BASE_URL\"" /etc/secret/secret-value/config) > /etc/secret/secret-value/config - echo $(jq ".optout_base_url = \"$OPTOUT_BASE_URL\"" /etc/secret/secret-value/config) > /etc/secret/secret-value/config - echo "run_config_server" - cd /opt/uid2operator/config-server - ./bin/flask run --host 127.0.0.1 --port 27015 & -} - -function run_enclave() { - echo "starting enclave..." - nitro-cli run-enclave --eif-path $EIF_PATH --memory $MEMORY_MB --cpu-count $CPU_COUNT --enclave-cid $CID --enclave-name uid2operator -} - -terminate_old_enclave -config_aws -read_allocation -# update_allocation -setup_vsockproxy -setup_dante -run_config_server -run_enclave - -echo "Done!" diff --git a/scripts/aws/stop.sh b/scripts/aws/stop.sh deleted file mode 100644 index c37bdc729..000000000 --- a/scripts/aws/stop.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -function terminate_old_enclave() { - echo "Terminating Enclave..." - ENCLAVE_ID=$(nitro-cli describe-enclaves | jq -r ".[0].EnclaveID") - if [ "$ENCLAVE_ID" != "null" ]; then - nitro-cli terminate-enclave --enclave-id $ENCLAVE_ID - else - echo "no running enclaves to terminate" - fi -} - -function kill_process() { - echo "Shutting down $1..." - pid=$(pidof $1) - if [ -z "$pid" ]; then - echo "process $1 not found" - else - kill -9 $pid - echo "$1 exited" - fi -} - -terminate_old_enclave -kill_process vsockpx -kill_process sockd -# we start aws vsock-proxy via nohup -kill_process vsock-proxy -kill_process nohup - -echo "Done!" diff --git a/scripts/aws/syslog-ng/syslog-ng-client.conf b/scripts/aws/syslog-ng/syslog-ng-client.conf index dd86abe8f..7f136c0d5 100644 --- a/scripts/aws/syslog-ng/syslog-ng-client.conf +++ b/scripts/aws/syslog-ng/syslog-ng-client.conf @@ -6,15 +6,6 @@ options { chain_hostnames(no); }; -source s_local { - system(); - internal(); -}; - -source s_dev_nitro { - pipe("/dev/nitro_enclaves"); -}; - source s_startup_file { file("/home/start.txt"); }; @@ -24,8 +15,6 @@ destination d_syslog_tcp { }; log { - source(s_local); - source(s_dev_nitro); source(s_startup_file); destination(d_syslog_tcp); }; diff --git a/scripts/aws/syslog-ng/syslog-ng-server.conf b/scripts/aws/syslog-ng/syslog-ng-server.conf index aa9b52e1c..45c3e5b1a 100644 --- a/scripts/aws/syslog-ng/syslog-ng-server.conf +++ b/scripts/aws/syslog-ng/syslog-ng-server.conf @@ -9,11 +9,6 @@ options { chain_hostnames(yes); }; -source s_local { - system(); - internal(); -}; - source s_network { network( ip(0.0.0.0) @@ -31,7 +26,6 @@ destination d_file { }; log { - source(s_local); source(s_network); destination(d_file); }; diff --git a/scripts/aws/uid2-operator-ami/ansible/playbook.yml b/scripts/aws/uid2-operator-ami/ansible/playbook.yml index 84c6c6f14..a5ec77809 100644 --- a/scripts/aws/uid2-operator-ami/ansible/playbook.yml +++ b/scripts/aws/uid2-operator-ami/ansible/playbook.yml @@ -70,34 +70,47 @@ requirements: /opt/uid2operator/config-server/requirements.txt virtualenv_command: 'python3 -m venv' + - name: Install requirements.txt for enclave init + ansible.builtin.copy: + src: /tmp/artifacts/requirements.txt + dest: /opt/uid2operator/requirements.txt + remote_src: yes + - name: Install starter script ansible.builtin.copy: - src: /tmp/artifacts/start.sh - dest: /opt/uid2operator/start.sh + src: /tmp/artifacts/ec2.py + dest: /opt/uid2operator/ec2.py remote_src: yes - name: Make starter script executable ansible.builtin.file: - path: /opt/uid2operator/start.sh + path: /opt/uid2operator/ec2.py mode: '0755' - - name: Install stopper script + - name: Copy confidential_compute script ansible.builtin.copy: - src: /tmp/artifacts/stop.sh - dest: /opt/uid2operator/stop.sh + src: /tmp/artifacts/confidential_compute.py + dest: /opt/uid2operator/confidential_compute.py remote_src: yes - - name: Make starter script executable - ansible.builtin.file: - path: /opt/uid2operator/stop.sh - mode: '0755' + - name: Create virtualenv for eif init + ansible.builtin.pip: + virtualenv: /opt/uid2operator/init + requirements: /opt/uid2operator/requirements.txt + virtualenv_command: 'python3.11 -m venv' - - name: Install Operator EIF + - name: Copy Operator EIF ansible.builtin.copy: - src: /tmp/artifacts/uid2operator.eif - dest: /opt/uid2operator/uid2operator.eif + src: /tmp/artifacts/uid2operatoreif.zip + dest: /opt/uid2operator/uid2operatoreif.zip remote_src: yes + - name: Unzip Operator EIF + ansible.builtin.unarchive: + src: /opt/uid2operator/uid2operatoreif.zip + dest: /opt/uid2operator/ + remote_src: yes + - name: Install Identity Scope ansible.builtin.copy: src: /tmp/artifacts/identity_scope.txt diff --git a/scripts/aws/uid2operator.service b/scripts/aws/uid2operator.service index 1d36b7a91..56559e3c2 100644 --- a/scripts/aws/uid2operator.service +++ b/scripts/aws/uid2operator.service @@ -8,8 +8,8 @@ RemainAfterExit=true StandardOutput=journal StandardError=journal SyslogIdentifier=uid2operator -ExecStart=/opt/uid2operator/start.sh -ExecStop=/opt/uid2operator/stop.sh +ExecStart=/opt/uid2operator/init/bin/python /opt/uid2operator/ec2.py +ExecStop=/opt/uid2operator/init/bin/python /opt/uid2operator/ec2.py -o stop [Install] -WantedBy=multi-user.target \ No newline at end of file +WantedBy=multi-user.target diff --git a/scripts/azure-aks/deployment/generate-deployment-artifacts.sh b/scripts/azure-aks/deployment/generate-deployment-artifacts.sh new file mode 100644 index 000000000..a9c8d8dd8 --- /dev/null +++ b/scripts/azure-aks/deployment/generate-deployment-artifacts.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +set -x + +# Following environment variables must be set +# - IMAGE: uid2-operator image +# - OUTPUT_DIR: output directory to store the artifacts +# - MANIFEST_DIR: output directory to store the manifest for the enclave Id +# - VERSION_NUMBER: the version number of the build + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +INPUT_DIR=${SCRIPT_DIR} + +if [[ -z ${IMAGE} ]]; then + echo "IMAGE cannot be empty" + exit 1 +fi +IMAGE_VERSION=$(echo $IMAGE | awk -F':' '{print $2}') +if [[ -z ${IMAGE_VERSION} ]]; then + echo "Failed to extract image version from ${IMAGE}" + exit 1 +fi + +if [[ -z ${OUTPUT_DIR} ]]; then + echo "OUTPUT_DIR cannot be empty" + exit 1 +fi + +mkdir -p ${OUTPUT_DIR} +if [[ $? -ne 0 ]]; then + echo "Failed to create ${OUTPUT_DIR}" + exit 1 +fi + +mkdir -p ${MANIFEST_DIR} +if [[ $? -ne 0 ]]; then + echo "Failed to create ${MANIFEST_DIR}" + exit 1 +fi + +# Input files +INPUT_FILES=( + operator.yaml +) + +# Copy input files to output dir +for f in ${INPUT_FILES[@]}; do + cp ${INPUT_DIR}/${f} ${OUTPUT_DIR}/${f} + if [[ $? -ne 0 ]]; then + echo "Failed to copy ${INPUT_DIR}/${f} to ${OUTPUT_DIR}" + exit 1 + fi +done + +az version +# Install confcom extension, az is originally available in GitHub workflow environment +az extension add --name confcom +if [[ $? -ne 0 ]]; then + echo "Failed to install Azure confcom extension" + exit 1 +fi + +# Required by az confcom +sudo usermod -aG docker ${USER} +if [[ $? -ne 0 ]]; then + echo "Failed to add current user to docker group" + exit 1 +fi + +# Generate operator template +sed -i "s#IMAGE_PLACEHOLDER#${IMAGE}#g" ${OUTPUT_DIR}/operator.yaml +# && \ +# sed -i "s#IMAGE_VERSION_PLACEHOLDER#${IMAGE_VERSION}#g" ${OUTPUT_DIR}/operator.yaml +if [[ $? -ne 0 ]]; then + echo "Failed to pre-process operator template file" + exit 1 +fi + +# Export the policy, update it to turn off allow_environment_variable_dropping, and then insert it into the template +# note that the EnclaveId is generated by generate.py on the raw policy, not the base64 version +POLICY_DIGEST_FILE=azure-aks-operator-digest-$VERSION_NUMBER.txt +az confcom acipolicygen --virtual-node-yaml ${OUTPUT_DIR}/operator.yaml --print-policy > ${INPUT_DIR}/policy.base64 +if [[ $? -ne 0 ]]; then + echo "Failed to generate ACI policy" + exit 1 +fi + +base64 -di < ${INPUT_DIR}/policy.base64 > ${INPUT_DIR}/generated.rego +if [[ $? -ne 0 ]]; then + echo "Failed to base64-decode policy" + exit 1 +fi + +sed --in-place \ + -e 's#{"pattern":"DEPLOYMENT_ENVIRONMENT=DEPLOYMENT_ENVIRONMENT_PLACEHOLDER","required":false,"strategy":"string"}#{"pattern":"DEPLOYMENT_ENVIRONMENT=.+","required":false,"strategy":"re2"}#g' \ + -e 's#{"pattern":"VAULT_NAME=VAULT_NAME_PLACEHOLDER","required":false,"strategy":"string"}#{"pattern":"VAULT_NAME=.+","required":false,"strategy":"re2"}#g' \ + -e 's#{"pattern":"OPERATOR_KEY_SECRET_NAME=OPERATOR_KEY_SECRET_NAME_PLACEHOLDER","required":false,"strategy":"string"}#{"pattern":"OPERATOR_KEY_SECRET_NAME=.+","required":false,"strategy":"re2"}#g' \ + ${INPUT_DIR}/generated.rego +if [[ $? -ne 0 ]]; then + echo "Failed to replace placeholders in policy file" + exit 1 +fi +cat ${INPUT_DIR}/generated.rego + +base64 -w0 < ${INPUT_DIR}/generated.rego > ${INPUT_DIR}/generated.rego.base64 +if [[ $? -ne 0 ]]; then + echo "Failed to base64-encode policy file" + exit 1 +fi + +python3 ${SCRIPT_DIR}/../../azure-cc/deployment/generate.py ${INPUT_DIR}/generated.rego > ${MANIFEST_DIR}/${POLICY_DIGEST_FILE} +if [[ $? -ne 0 ]]; then + echo "Failed to generate digest from policy file" + exit 1 +fi + +sed --in-place "s#CCE_POLICY_PLACEHOLDER#$(cat ${INPUT_DIR}/generated.rego.base64)#g" ${OUTPUT_DIR}/operator.yaml +if [[ $? -ne 0 ]]; then + echo "Failed to replace placeholder in operator.yaml" + exit 1 +fi + diff --git a/scripts/azure-aks/deployment/operator.yaml b/scripts/azure-aks/deployment/operator.yaml new file mode 100644 index 000000000..234bd4a23 --- /dev/null +++ b/scripts/azure-aks/deployment/operator.yaml @@ -0,0 +1,93 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: operator-deployment +spec: + replicas: 3 + selector: + matchLabels: + app.kubernetes.io/name: operator + template: + metadata: + labels: + app.kubernetes.io/name: operator + annotations: + microsoft.containerinstance.virtualnode.ccepolicy: CCE_POLICY_PLACEHOLDER + microsoft.containerinstance.virtualnode.identity: IDENTITY_PLACEHOLDER + microsoft.containerinstance.virtualnode.injectdns: "false" + spec: + containers: + - image: "mcr.microsoft.com/aci/skr:2.7" + imagePullPolicy: Always + name: skr + resources: + limits: + cpu: 2250m + memory: 2256Mi + requests: + cpu: 100m + memory: 512Mi + env: + - name: Port + value: "9000" + volumeMounts: + - mountPath: /opt/confidential-containers/share/kata-containers/reference-info-base64 + name: endorsement-location + command: + - /skr.sh + - name: uid2-operator + image: IMAGE_PLACEHOLDER + resources: + limits: + memory: "8Gi" + imagePullPolicy: Always + securityContext: + runAsUser: 1000 + env: + - name: VAULT_NAME + value: VAULT_NAME_PLACEHOLDER + - name: OPERATOR_KEY_SECRET_NAME + value: OPERATOR_KEY_SECRET_NAME_PLACEHOLDER + - name: DEPLOYMENT_ENVIRONMENT + value: DEPLOYMENT_ENVIRONMENT_PLACEHOLDER + - name: IMAGE_NAME + value: IMAGE_PLACEHOLDER + ports: + - containerPort: 8080 + protocol: TCP + - name: prometheus + containerPort: 9080 + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ops/healthcheck + port: 8080 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + volumes: + - name: endorsement-location + hostPath: + path: /opt/confidential-containers/share/kata-containers/reference-info-base64 + nodeSelector: + virtualization: virtualnode2 + tolerations: + - effect: NoSchedule + key: virtual-kubelet.io/provider + operator: Exists +--- +apiVersion: v1 +kind: Service +metadata: + name: operator-svc +spec: + type: LoadBalancer + selector: + app.kubernetes.io/name: operator + ports: + - protocol: TCP + port: 80 + targetPort: 8080 diff --git a/scripts/azure-cc/Dockerfile b/scripts/azure-cc/Dockerfile index bb0c96b70..1fe6cfc67 100644 --- a/scripts/azure-cc/Dockerfile +++ b/scripts/azure-cc/Dockerfile @@ -1,13 +1,24 @@ -# sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.4_7-jre-alpine/images/sha256-8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca -FROM eclipse-temurin@sha256:8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca +# sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.7_6-jre-alpine-3.21/images/sha256-62fa775039897e4420368514ba6c167741f6d45a0de9ff9125bee57e5aca8b75 +FROM eclipse-temurin@sha256:62fa775039897e4420368514ba6c167741f6d45a0de9ff9125bee57e5aca8b75 -# Install Packages -RUN apk update && apk add jq +# Install necessary packages and set up virtual environment +RUN apk update && apk add --no-cache jq python3 py3-pip && \ + python3 -m venv /venv && \ + . /venv/bin/activate && \ + pip install --no-cache-dir requests azure-identity azure-keyvault-secrets && \ + rm -rf /var/cache/apk/* +# Set virtual environment path +ENV PATH="/venv/bin:$PATH" + +# Working directory WORKDIR /app + +# Expose necessary ports EXPOSE 8080 EXPOSE 9080 +# ARG and ENV variables ARG JAR_NAME=uid2-operator ARG JAR_VERSION=1.0.0-SNAPSHOT ARG IMAGE_VERSION=1.0.0.unknownhash @@ -15,20 +26,29 @@ ENV JAR_NAME=${JAR_NAME} ENV JAR_VERSION=${JAR_VERSION} ENV IMAGE_VERSION=${IMAGE_VERSION} ENV REGION=default -ENV LOKI_HOSTNAME=loki +# Copy application files COPY ./target/${JAR_NAME}-${JAR_VERSION}-jar-with-dependencies.jar /app/${JAR_NAME}-${JAR_VERSION}.jar COPY ./target/${JAR_NAME}-${JAR_VERSION}-sources.jar /app COPY ./target/${JAR_NAME}-${JAR_VERSION}-static.tar.gz /app/static.tar.gz COPY ./conf/*.json /app/conf/ COPY ./conf/*.xml /app/conf/ -RUN tar xzvf /app/static.tar.gz --no-same-owner --no-same-permissions && rm -f /app/static.tar.gz +# Extract and clean up tar.gz +RUN tar xzvf /app/static.tar.gz --no-same-owner --no-same-permissions && \ + rm -f /app/static.tar.gz + +COPY ./azr.py /app +COPY ./confidential_compute.py /app +RUN chmod a+x /app/*.py -COPY ./entrypoint.sh /app/ -RUN chmod a+x /app/entrypoint.sh +# Create and configure non-root user +RUN adduser -D uid2-operator && \ + mkdir -p /opt/uid2 && chmod 777 -R /opt/uid2 && \ + chmod 705 -R /app && mkdir -p /app/file-uploads && chmod 777 -R /app/file-uploads -RUN adduser -D uid2-operator && mkdir -p /opt/uid2 && chmod 777 -R /opt/uid2 && mkdir -p /app && chmod 705 -R /app && mkdir -p /app/file-uploads && chmod 777 -R /app/file-uploads +# Switch to non-root user USER uid2-operator -CMD ["/app/entrypoint.sh"] +# Run the Python entry point +CMD python3 /app/azr.py \ No newline at end of file diff --git a/scripts/azure-cc/azr.py b/scripts/azure-cc/azr.py new file mode 100644 index 000000000..5ccfcb301 --- /dev/null +++ b/scripts/azure-cc/azr.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 + +import json +import os +import time +from typing import Dict +import sys +import shutil +import requests +import logging +from datetime import datetime +from confidential_compute import ConfidentialCompute, ConfigurationMissingError, OperatorKeyPermissionError, OperatorKeyNotFoundError, ConfidentialComputeStartupError +from azure.keyvault.secrets import SecretClient +from azure.identity import DefaultAzureCredential, CredentialUnavailableError +from azure.core.exceptions import ResourceNotFoundError, ClientAuthenticationError +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +class AZR(ConfidentialCompute): + kv_name = os.getenv("VAULT_NAME") + secret_name = os.getenv("OPERATOR_KEY_SECRET_NAME") + env_name = os.getenv("DEPLOYMENT_ENVIRONMENT") + jar_name = os.getenv("JAR_NAME", "default-jar-name") + jar_version = os.getenv("JAR_VERSION", "default-jar-version") + default_core_endpoint = f"https://core-{env_name}.uidapi.com".lower() + default_optout_endpoint = f"https://optout-{env_name}.uidapi.com".lower() + + FINAL_CONFIG = "/tmp/final-config.json" + + def __init__(self): + super().__init__() + + def __check_env_variables(self): + # Check essential env variables + if AZR.kv_name is None: + raise ConfigurationMissingError(self.__class__.__name__, ["VAULT_NAME"]) + if AZR.secret_name is None: + raise ConfigurationMissingError(self.__class__.__name__, ["OPERATOR_KEY_SECRET_NAME"]) + if AZR.env_name is None: + raise ConfigurationMissingError(self.__class__.__name__, ["DEPLOYMENT_ENVIRONMENT"]) + logging.info("Environment variables validation success") + + def __create_final_config(self): + TARGET_CONFIG = f"/app/conf/{AZR.env_name}-uid2-config.json" + if not os.path.isfile(TARGET_CONFIG): + logging.error(f"Unrecognized config {TARGET_CONFIG}") + sys.exit(1) + + logging.info(f"-- copying {TARGET_CONFIG} to {AZR.FINAL_CONFIG}") + try: + shutil.copy(TARGET_CONFIG, AZR.FINAL_CONFIG) + except IOError as e: + logging.error(f"Failed to create {AZR.FINAL_CONFIG} with error: {e}") + sys.exit(1) + + logging.info(f"-- replacing URLs by {self.configs["core_base_url"]} and {self.configs["optout_base_url"]}") + with open(AZR.FINAL_CONFIG, "r") as file: + config = file.read() + + config = config.replace("https://core.uidapi.com", self.configs["core_base_url"]) + config = config.replace("https://optout.uidapi.com", self.configs["optout_base_url"]) + config = config.replace("unknown", self.configs["uid_instance_id_prefix"]) + with open(AZR.FINAL_CONFIG, "w") as file: + file.write(config) + + with open(AZR.FINAL_CONFIG, "r") as file: + logging.info(file.read()) + + def __set_operator_key(self): + try: + credential = DefaultAzureCredential() + kv_URL = f"https://{AZR.kv_name}.vault.azure.net" + secret_client = SecretClient(vault_url=kv_URL, credential=credential) + secret = secret_client.get_secret(AZR.secret_name) + self.configs["operator_key"] = secret.value + + except (CredentialUnavailableError, ClientAuthenticationError) as auth_error: + logging.error(f"Read operator key, authentication error: {auth_error}") + raise OperatorKeyPermissionError(self.__class__.__name__, str(auth_error)) + except ResourceNotFoundError as not_found_error: + logging.error(f"Read operator key, secret not found: {AZR.secret_name}. Error: {not_found_error}") + raise OperatorKeyNotFoundError(self.__class__.__name__, str(not_found_error)) + + def __get_azure_image_info(self) -> str: + """ + Fetches Image version from non-modifiable environment variable. + """ + try: + return os.getenv("IMAGE_NAME") + except Exception as e: + raise RuntimeError(f"Failed to fetch Azure image info: {e}") + + + def _set_confidential_config(self, secret_identifier: str = None): + """Builds and sets ConfidentialComputeConfig""" + self.configs["skip_validations"] = os.getenv("SKIP_VALIDATIONS", "false").lower() == "true" + self.configs["debug_mode"] = os.getenv("DEBUG_MODE", "false").lower() == "true" + self.configs["environment"] = AZR.env_name + self.configs["core_base_url"] = os.getenv("CORE_BASE_URL") if os.getenv("CORE_BASE_URL") and AZR.env_name == "integ" else AZR.default_core_endpoint + self.configs["optout_base_url"] = os.getenv("OPTOUT_BASE_URL") if os.getenv("OPTOUT_BASE_URL") and AZR.env_name == "integ" else AZR.default_optout_endpoint + image_version = self.__get_azure_image_info() + self.configs["uid_instance_id_prefix"] = self.get_uid_instance_id(identifier=datetime.now().strftime("%H:%M:%S"), version=image_version) + self.__set_operator_key() + + def __run_operator(self): + os.environ["azure_vault_name"] = AZR.kv_name + os.environ["azure_secret_name"] = AZR.secret_name + java_command = [ + "java", + "-XX:MaxRAMPercentage=95", "-XX:-UseCompressedOops", "-XX:+PrintFlagsFinal", + "-Djava.security.egd=file:/dev/./urandom", + "-Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory", + "-Dlogback.configurationFile=/app/conf/logback.xml", + f"-Dvertx-config-path={AZR.FINAL_CONFIG}", + "-jar", + f"{AZR.jar_name}-{AZR.jar_version}.jar" + ] + logging.info("-- starting java operator application") + self.run_command(java_command, separate_process=False) + + def _validate_auxiliaries(self): + logging.info("Waiting for sidecar ...") + + MAX_RETRIES = 15 + PING_URL = "http://169.254.169.254/ping" + delay = 1 + + for attempt in range(1, MAX_RETRIES + 1): + try: + response = requests.get(PING_URL, timeout=5) + if response.status_code in [200, 204]: + logging.info("Sidecar started successfully.") + return + else: + logging.warning( + f"Attempt {attempt}: Unexpected status code {response.status_code}. Response: {response.text}" + ) + except Exception as e: + logging.info(f"Attempt {attempt}: Error during request - {e}") + + if attempt == MAX_RETRIES: + raise RuntimeError(f"Unable to detect sidecar running after {MAX_RETRIES} attempts. Exiting.") + + logging.info(f"Retrying in {delay} seconds... (Attempt {attempt}/{MAX_RETRIES})") + time.sleep(delay) + delay += 1 + + def run_compute(self) -> None: + """Main execution flow for confidential compute.""" + self.__check_env_variables() + self._set_confidential_config() + if not self.configs.get("skip_validations"): + self.validate_configuration() + self.__create_final_config() + self._setup_auxiliaries() + self.__run_operator() + + def _setup_auxiliaries(self) -> None: + """ setup auxiliary services are running.""" + pass + +if __name__ == "__main__": + + logging.basicConfig(level=logging.INFO) + logging.info("Start Azure") + try: + operator = AZR() + operator.run_compute() + except ConfidentialComputeStartupError as e: + logging.error(f"Failed starting up Azure Confidential Compute. Please checks the logs for errors and retry {e}", exc_info=True) + except Exception as e: + logging.error(f"Unexpected failure while starting up Azure Confidential Compute. Please contact UID support team with this log {e}", exc_info=True) diff --git a/scripts/azure-cc/conf/default-config.json b/scripts/azure-cc/conf/default-config.json index fbe3e7184..c7e8d6ab3 100644 --- a/scripts/azure-cc/conf/default-config.json +++ b/scripts/azure-cc/conf/default-config.json @@ -34,11 +34,10 @@ "identity_token_expires_after_seconds": 86400, "refresh_token_expires_after_seconds": 2592000, "refresh_identity_token_after_seconds": 3600, - "allow_legacy_api": false, "failure_shutdown_wait_hours": 120, "sharing_token_expiry_seconds": 2592000, "validate_service_links": false, - "advertising_token_v4_percentage": 100, - "site_ids_using_v4_tokens": "", - "operator_type": "private" + "operator_type": "private", + "enable_remote_config": false, + "uid_instance_id_prefix": "unknown" } diff --git a/scripts/azure-cc/conf/integ-uid2-config.json b/scripts/azure-cc/conf/integ-uid2-config.json index 2cd4be5c3..010a184ea 100644 --- a/scripts/azure-cc/conf/integ-uid2-config.json +++ b/scripts/azure-cc/conf/integ-uid2-config.json @@ -1,14 +1,23 @@ { - "sites_metadata_path": "https://core-integ.uidapi.com/sites/refresh", - "clients_metadata_path": "https://core-integ.uidapi.com/clients/refresh", - "keysets_metadata_path": "https://core-integ.uidapi.com/key/keyset/refresh", - "keyset_keys_metadata_path": "https://core-integ.uidapi.com/key/keyset-keys/refresh", - "client_side_keypairs_metadata_path": "https://core-integ.uidapi.com/client_side_keypairs/refresh", - "salts_metadata_path": "https://core-integ.uidapi.com/salt/refresh", - "services_metadata_path": "https://core-integ.uidapi.com/services/refresh", - "service_links_metadata_path": "https://core-integ.uidapi.com/service_links/refresh", - "optout_metadata_path": "https://optout-integ.uidapi.com/optout/refresh", - "core_attest_url": "https://core-integ.uidapi.com/attest", - "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate", - "optout_s3_folder": "uid-optout-integ/" + "sites_metadata_path": "https://core.uidapi.com/sites/refresh", + "clients_metadata_path": "https://core.uidapi.com/clients/refresh", + "keysets_metadata_path": "https://core.uidapi.com/key/keyset/refresh", + "keyset_keys_metadata_path": "https://core.uidapi.com/key/keyset-keys/refresh", + "client_side_keypairs_metadata_path": "https://core.uidapi.com/client_side_keypairs/refresh", + "salts_metadata_path": "https://core.uidapi.com/salt/refresh", + "services_metadata_path": "https://core.uidapi.com/services/refresh", + "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh", + "optout_metadata_path": "https://optout.uidapi.com/optout/refresh", + "core_attest_url": "https://core.uidapi.com/attest", + "optout_api_uri": "https://optout.uidapi.com/optout/replicate", + "cloud_encryption_keys_metadata_path": "https://core.uidapi.com/cloud_encryption_keys/retrieve", + "optout_s3_folder": "uid-optout-integ/", + "uid_instance_id_prefix": "unknown", + "runtime_config_store": { + "type": "http", + "config" : { + "url": "https://core.uidapi.com/operator/config" + }, + "config_scan_period_ms": 300000 + } } diff --git a/scripts/azure-cc/conf/logback.xml b/scripts/azure-cc/conf/logback.xml index 56c8e8df2..bdd1f5726 100644 --- a/scripts/azure-cc/conf/logback.xml +++ b/scripts/azure-cc/conf/logback.xml @@ -4,12 +4,12 @@ ch.qos.logback.classic.encoder.PatternLayoutEncoder by default --> - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %ex%n + %d{HH:mm:ss.SSS} thread=%thread level=%-5level class=%logger{36} - %msg %ex%n - + diff --git a/scripts/azure-cc/conf/prod-uid2-config.json b/scripts/azure-cc/conf/prod-uid2-config.json index 02e2cde20..9e0ec902f 100644 --- a/scripts/azure-cc/conf/prod-uid2-config.json +++ b/scripts/azure-cc/conf/prod-uid2-config.json @@ -1,15 +1,24 @@ { - "sites_metadata_path": "https://core-prod.uidapi.com/sites/refresh", - "clients_metadata_path": "https://core-prod.uidapi.com/clients/refresh", - "keysets_metadata_path": "https://core-prod.uidapi.com/key/keyset/refresh", - "keyset_keys_metadata_path": "https://core-prod.uidapi.com/key/keyset-keys/refresh", - "client_side_keypairs_metadata_path": "https://core-prod.uidapi.com/client_side_keypairs/refresh", - "salts_metadata_path": "https://core-prod.uidapi.com/salt/refresh", - "services_metadata_path": "https://core-prod.uidapi.com/services/refresh", - "service_links_metadata_path": "https://core-prod.uidapi.com/service_links/refresh", - "optout_metadata_path": "https://optout-prod.uidapi.com/optout/refresh", - "core_attest_url": "https://core-prod.uidapi.com/attest", - "optout_api_uri": "https://optout-prod.uidapi.com/optout/replicate", + "sites_metadata_path": "https://core.uidapi.com/sites/refresh", + "clients_metadata_path": "https://core.uidapi.com/clients/refresh", + "keysets_metadata_path": "https://core.uidapi.com/key/keyset/refresh", + "keyset_keys_metadata_path": "https://core.uidapi.com/key/keyset-keys/refresh", + "client_side_keypairs_metadata_path": "https://core.uidapi.com/client_side_keypairs/refresh", + "salts_metadata_path": "https://core.uidapi.com/salt/refresh", + "services_metadata_path": "https://core.uidapi.com/services/refresh", + "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh", + "optout_metadata_path": "https://optout.uidapi.com/optout/refresh", + "core_attest_url": "https://core.uidapi.com/attest", + "cloud_encryption_keys_metadata_path": "https://core.uidapi.com/cloud_encryption_keys/retrieve", + "optout_api_uri": "https://optout.uidapi.com/optout/replicate", "optout_s3_folder": "optout-v2/", - "identity_token_expires_after_seconds": 259200 + "identity_token_expires_after_seconds": 259200, + "uid_instance_id_prefix": "unknown", + "runtime_config_store": { + "type": "http", + "config" : { + "url": "https://core.uidapi.com/operator/config" + }, + "config_scan_period_ms": 300000 + } } diff --git a/scripts/azure-cc/deployment/operator.json b/scripts/azure-cc/deployment/operator.json index b50ecced9..43d395c1b 100644 --- a/scripts/azure-cc/deployment/operator.json +++ b/scripts/azure-cc/deployment/operator.json @@ -54,6 +54,16 @@ "metadata": { "description": "Operator Key" } + }, + "skipValidations": { + "type": "string", + "metadata": { + "description": "Whether to skip pre-init validations" + }, + "allowedValues": [ + "true", + "false" + ] } }, "variables": { @@ -111,6 +121,10 @@ } }, "environmentVariables": [ + { + "name": "IMAGE_NAME", + "value": "IMAGE_PLACEHOLDER" + }, { "name": "VAULT_NAME", "value": "[parameters('vaultName')]" @@ -122,6 +136,10 @@ { "name": "DEPLOYMENT_ENVIRONMENT", "value": "[parameters('deploymentEnvironment')]" + }, + { + "name": "SKIP_VALIDATIONS", + "value": "[parameters('skipValidations')]" } ] } diff --git a/scripts/azure-cc/deployment/operator.parameters.json b/scripts/azure-cc/deployment/operator.parameters.json index 776690776..5095746ea 100644 --- a/scripts/azure-cc/deployment/operator.parameters.json +++ b/scripts/azure-cc/deployment/operator.parameters.json @@ -22,6 +22,9 @@ }, "deploymentEnvironment": { "value": "integ" + }, + "skipValidations": { + "value": "false" } } } diff --git a/scripts/azure-cc/entrypoint.sh b/scripts/azure-cc/entrypoint.sh deleted file mode 100644 index 14875c9bf..000000000 --- a/scripts/azure-cc/entrypoint.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/sh -# -# This script must be compatible with Ash (provided in eclipse-temurin Docker image) and Bash - -function wait_for_sidecar() { - url="http://169.254.169.254/ping" - delay=1 - max_retries=15 - - while true; do - if wget -q --spider --tries=1 --timeout 5 "$url" > /dev/null; then - echo "side car started" - break - else - echo "side car not started. Retrying in $delay seconds..." - sleep $delay - if [ $delay -gt $max_retries ]; then - echo "side car failed to start" - break - fi - delay=$((delay + 1)) - fi - done -} - -TMP_FINAL_CONFIG="/tmp/final-config.tmp" - -if [ -z "${VAULT_NAME}" ]; then - echo "VAULT_NAME cannot be empty" - exit 1 -fi - -if [ -z "${OPERATOR_KEY_SECRET_NAME}" ]; then - echo "OPERATOR_KEY_SECRET_NAME cannot be empty" - exit 1 -fi - -export azure_vault_name="${VAULT_NAME}" -export azure_secret_name="${OPERATOR_KEY_SECRET_NAME}" - -# -- locate config file -if [ -z "${DEPLOYMENT_ENVIRONMENT}" ]; then - echo "DEPLOYMENT_ENVIRONMENT cannot be empty" - exit 1 -fi -if [ "${DEPLOYMENT_ENVIRONMENT}" != 'prod' -a "${DEPLOYMENT_ENVIRONMENT}" != 'integ' ]; then - echo "Unrecognized DEPLOYMENT_ENVIRONMENT ${DEPLOYMENT_ENVIRONMENT}" - exit 1 -fi - -TARGET_CONFIG="/app/conf/${DEPLOYMENT_ENVIRONMENT}-uid2-config.json" -if [ ! -f "${TARGET_CONFIG}" ]; then - echo "Unrecognized config ${TARGET_CONFIG}" - exit 1 -fi - -FINAL_CONFIG="/tmp/final-config.json" -echo "-- copying ${TARGET_CONFIG} to ${FINAL_CONFIG}" -cp ${TARGET_CONFIG} ${FINAL_CONFIG} -if [ $? -ne 0 ]; then - echo "Failed to create ${FINAL_CONFIG} with error code $?" - exit 1 -fi - -# -- replace base URLs if both CORE_BASE_URL and OPTOUT_BASE_URL are provided -# -- using hardcoded domains is fine because they should not be changed frequently -if [ -n "${CORE_BASE_URL}" -a -n "${OPTOUT_BASE_URL}" -a "${DEPLOYMENT_ENVIRONMENT}" != 'prod' ]; then - echo "-- replacing URLs by ${CORE_BASE_URL} and ${OPTOUT_BASE_URL}" - sed -i "s#https://core-integ.uidapi.com#${CORE_BASE_URL}#g" ${FINAL_CONFIG} - - sed -i "s#https://optout-integ.uidapi.com#${OPTOUT_BASE_URL}#g" ${FINAL_CONFIG} -fi - -cat $FINAL_CONFIG - -# delay the start of the operator until the side car has started correctly -wait_for_sidecar - -# -- start operator -echo "-- starting java application" -java \ - -XX:MaxRAMPercentage=95 -XX:-UseCompressedOops -XX:+PrintFlagsFinal \ - -Djava.security.egd=file:/dev/./urandom \ - -Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory \ - -Dlogback.configurationFile=/app/conf/logback.xml \ - -Dvertx-config-path=${FINAL_CONFIG} \ - -jar ${JAR_NAME}-${JAR_VERSION}.jar diff --git a/scripts/confidential_compute.py b/scripts/confidential_compute.py new file mode 100644 index 000000000..3fd696f70 --- /dev/null +++ b/scripts/confidential_compute.py @@ -0,0 +1,161 @@ +import requests +import re +import socket +from urllib.parse import urlparse +from abc import ABC, abstractmethod +from typing import TypedDict, NotRequired, get_type_hints +import subprocess +import logging +import hashlib + +class ConfidentialComputeConfig(TypedDict): + operator_key: str + core_base_url: str + optout_base_url: str + environment: str + uid_instance_id_prefix: str + skip_validations: NotRequired[bool] + debug_mode: NotRequired[bool] + +class ConfidentialComputeStartupError(Exception): + def __init__(self, error_name, provider, extra_message=None): + urls = { + "EC2": "https://unifiedid.com/docs/guides/operator-guide-aws-marketplace#uid2-operator-error-codes", + "AZR": "https://unifiedid.com/docs/guides/operator-guide-azure-enclave#uid2-operator-error-codes", + "GCP": "https://unifiedid.com/docs/guides/operator-private-gcp-confidential-space#uid2-operator-error-codes", + } + url = urls.get(provider) + super().__init__(f"{error_name}\n" + (extra_message if extra_message else "") + f"\nVisit {url} for more details") + +class InstanceProfileMissingError(ConfidentialComputeStartupError): + def __init__(self, cls, message = None): + super().__init__(error_name=f"E01: {self.__class__.__name__}", provider=cls, extra_message=message) + +class OperatorKeyNotFoundError(ConfidentialComputeStartupError): + def __init__(self, cls, message = None): + super().__init__(error_name=f"E02: {self.__class__.__name__}", provider=cls, extra_message=message) + +class ConfigurationMissingError(ConfidentialComputeStartupError): + def __init__(self, cls, missing_keys): + super().__init__(error_name=f"E03: {self.__class__.__name__}", provider=cls, extra_message=', '.join(missing_keys)) + +class ConfigurationValueError(ConfidentialComputeStartupError): + def __init__(self, cls, config_key = None): + super().__init__(error_name=f"E04: {self.__class__.__name__} " , provider=cls, extra_message=config_key) + +class OperatorKeyValidationError(ConfidentialComputeStartupError): + def __init__(self, cls): + super().__init__(error_name=f"E05: {self.__class__.__name__}", provider=cls) + +class UID2ServicesUnreachableError(ConfidentialComputeStartupError): + def __init__(self, cls, ip=None): + super().__init__(error_name=f"E06: {self.__class__.__name__}", provider=cls, extra_message=ip) + +class OperatorKeyPermissionError(ConfidentialComputeStartupError): + def __init__(self, cls, message = None): + super().__init__(error_name=f"E08: {self.__class__.__name__}", provider=cls, extra_message=message) + +class ConfidentialCompute(ABC): + + def __init__(self): + self.configs: ConfidentialComputeConfig = {} + + def validate_configuration(self): + """ Validates the paramters specified through configs/secret manager .""" + logging.info("Validating configurations provided") + def validate_operator_key(): + """ Validates the operator key format and its environment alignment.""" + operator_key = self.configs.get("operator_key") + pattern = r"^(UID2|EUID)-.\-(I|P|L)-\d+-.*$" + if re.match(pattern, operator_key): + env = self.configs.get("environment", "").lower() + debug_mode = self.configs.get("debug_mode", False) + expected_env = "I" if debug_mode or env == "integ" else "P" + if operator_key.split("-")[2] != expected_env: + raise OperatorKeyValidationError(self.__class__.__name__) + logging.info("Validated operator key matches environment") + else: + logging.info("Skipping operator key validation") + + def validate_url(url_key, environment): + """URL should include environment except in prod""" + if environment != "prod" and environment not in self.configs[url_key]: + raise ConfigurationValueError(self.__class__.__name__, url_key) + parsed_url = urlparse(self.configs[url_key]) + if parsed_url.scheme != 'https' and parsed_url.path: + raise ConfigurationValueError(self.__class__.__name__, url_key) + logging.info(f"Validated {self.configs[url_key]} matches other config parameters") + + def validate_connectivity() -> None: + """ Validates that the core URL is accessible.""" + try: + core_url = self.configs["core_base_url"] + core_ip = socket.gethostbyname(urlparse(core_url).netloc) + requests.get(core_url, timeout=5) + logging.info(f"Validated connectivity to {core_url}") + except (requests.ConnectionError, requests.Timeout) as e: + raise UID2ServicesUnreachableError(self.__class__.__name__, core_ip) + except Exception as e: + raise UID2ServicesUnreachableError(self.__class__.__name__) + + type_hints = get_type_hints(ConfidentialComputeConfig, include_extras=True) + required_keys = [field for field, hint in type_hints.items() if "NotRequired" not in str(hint)] + missing_keys = [key for key in required_keys if key not in self.configs or self.configs[key] == None] + if missing_keys: + raise ConfigurationMissingError(self.__class__.__name__, missing_keys) + + environment = self.configs["environment"] + if environment not in ["integ", "prod"]: + raise ConfigurationValueError(self.__class__.__name__, "environment") + + if self.configs.get("debug_mode") and environment == "prod": + raise ConfigurationValueError(self.__class__.__name__, "debug_mode") + + validate_url("core_base_url", environment) + validate_url("optout_base_url", environment) + validate_operator_key() + validate_connectivity() + logging.info("Completed static validation of confidential compute config values") + + @abstractmethod + def _set_confidential_config(self, secret_identifier: str) -> None: + """ + Set ConfidentialComputeConfig + """ + pass + + @abstractmethod + def _setup_auxiliaries(self) -> None: + """ Sets up auxiliary processes required for confidential computing. """ + pass + + @abstractmethod + def _validate_auxiliaries(self) -> None: + """ Validates auxiliary services are running.""" + pass + + @abstractmethod + def run_compute(self) -> None: + """ Runs confidential computing.""" + pass + + def get_uid_instance_id(self, identifier, version): + logging.info(f"Generating UID Indentifier for {identifier} running version: {version}") + identifier_hash = hashlib.sha256(identifier.encode()).hexdigest()[:6] + cloud_provider = self.__class__.__name__.lower() + uid_instance_id = f"{cloud_provider}-{identifier_hash}-{version}" + logging.info(f"Generated and using uid_instance_id: {uid_instance_id}") + return uid_instance_id + + @staticmethod + def run_command(command, separate_process=False, stdout=None, stderr=None): + logging.info(f"Running command: {' '.join(command)}") + try: + if separate_process: + subprocess.Popen(command, stdout=stdout, stderr=stderr) + else: + subprocess.run(command, check=True, stdout=stdout, stderr=stderr) + + except Exception as e: + logging.error(f"Failed to run command: {e}", exc_info=True) + raise RuntimeError (f"Failed to start {' '.join(command)} ") diff --git a/scripts/gcp-oidc/Dockerfile b/scripts/gcp-oidc/Dockerfile index 76b302e30..1ec846ed2 100644 --- a/scripts/gcp-oidc/Dockerfile +++ b/scripts/gcp-oidc/Dockerfile @@ -1,11 +1,15 @@ -# sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.4_7-jre-alpine/images/sha256-8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca -FROM eclipse-temurin@sha256:8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca +# sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.7_6-jre-alpine-3.21/images/sha256-62fa775039897e4420368514ba6c167741f6d45a0de9ff9125bee57e5aca8b75 +FROM eclipse-temurin@sha256:62fa775039897e4420368514ba6c167741f6d45a0de9ff9125bee57e5aca8b75 -LABEL "tee.launch_policy.allow_env_override"="API_TOKEN_SECRET_NAME,DEPLOYMENT_ENVIRONMENT,CORE_BASE_URL,OPTOUT_BASE_URL" +LABEL "tee.launch_policy.allow_env_override"="API_TOKEN_SECRET_NAME,DEPLOYMENT_ENVIRONMENT,CORE_BASE_URL,OPTOUT_BASE_URL,DEBUG_MODE,SKIP_VALIDATIONS" LABEL "tee.launch_policy.log_redirect"="always" # Install Packages -RUN apk update && apk add jq +RUN apk update && apk add --no-cache jq python3 py3-pip && \ + python3 -m venv /venv && \ + . /venv/bin/activate && \ + pip install --no-cache-dir google-cloud-secret-manager google-auth google-api-core && \ + rm -rf /var/cache/apk/* WORKDIR /app EXPOSE 8080 @@ -18,7 +22,6 @@ ENV JAR_NAME=${JAR_NAME} ENV JAR_VERSION=${JAR_VERSION} ENV IMAGE_VERSION=${IMAGE_VERSION} ENV REGION=default -ENV LOKI_HOSTNAME=loki COPY ./target/${JAR_NAME}-${JAR_VERSION}-jar-with-dependencies.jar /app/${JAR_NAME}-${JAR_VERSION}.jar COPY ./target/${JAR_NAME}-${JAR_VERSION}-sources.jar /app @@ -28,9 +31,10 @@ COPY ./conf/*.xml /app/conf/ RUN tar xzvf /app/static.tar.gz --no-same-owner --no-same-permissions && rm -f /app/static.tar.gz -COPY ./entrypoint.sh /app/ -RUN chmod a+x /app/entrypoint.sh +COPY ./gcp.py /app/ +COPY ./confidential_compute.py /app +RUN chmod a+x /app/gcp.py RUN mkdir -p /opt/uid2 && chmod 777 -R /opt/uid2 && mkdir -p /app && chmod 705 -R /app && mkdir -p /app/file-uploads && chmod 777 -R /app/file-uploads -CMD ["/app/entrypoint.sh"] +CMD ["/venv/bin/python", "/app/gcp.py"] diff --git a/scripts/gcp-oidc/conf/default-config.json b/scripts/gcp-oidc/conf/default-config.json index 302a8c3c3..bd09cac63 100644 --- a/scripts/gcp-oidc/conf/default-config.json +++ b/scripts/gcp-oidc/conf/default-config.json @@ -1,44 +1,42 @@ -{ - "service_verbose": true, - "service_instances": 12, - "core_s3_bucket": null, - "core_attest_url": null, - "core_api_token": null, - "storage_mock": false, - "optout_s3_bucket": null, - "optout_s3_folder": "optout/", - "optout_s3_path_compat": false, - "optout_data_dir": "/opt/uid2/operator-optout/", - "optout_api_token": null, - "optout_api_uri": null, - "optout_bloom_filter_size": 8192, - "optout_delta_rotate_interval": 300, - "optout_delta_backtrack_in_days": 1, - "optout_partition_interval": 86400, - "optout_max_partitions": 30, - "optout_heap_default_capacity": 8192, - "cloud_download_threads": 8, - "cloud_upload_threads": 2, - "cloud_refresh_interval": 60, - "sites_metadata_path": "sites/metadata.json", - "clients_metadata_path": "clients/metadata.json", - "client_side_keypairs_metadata_path": "client_side_keypairs/metadata.json", - "keysets_metadata_path": "keysets/metadata.json", - "keyset_keys_metadata_path": "keyset_keys/metadata.json", - "salts_metadata_path": "salts/metadata.json", - "services_metadata_path": "services/metadata.json", - "service_links_metadata_path": "service_links/metadata.json", - "optout_metadata_path": null, - "enclave_platform": "gcp-oidc", - "optout_inmem_cache": true, - "identity_token_expires_after_seconds": 86400, - "refresh_token_expires_after_seconds": 2592000, - "refresh_identity_token_after_seconds": 3600, - "allow_legacy_api": false, - "failure_shutdown_wait_hours": 120, - "sharing_token_expiry_seconds": 2592000, - "validate_service_links": false, - "advertising_token_v4_percentage": 100, - "site_ids_using_v4_tokens": "", - "operator_type": "private" -} +{ + "service_verbose": true, + "service_instances": 12, + "core_s3_bucket": null, + "core_attest_url": null, + "core_api_token": null, + "storage_mock": false, + "optout_s3_bucket": null, + "optout_s3_folder": "optout/", + "optout_s3_path_compat": false, + "optout_data_dir": "/opt/uid2/operator-optout/", + "optout_api_token": null, + "optout_api_uri": null, + "optout_bloom_filter_size": 8192, + "optout_delta_rotate_interval": 300, + "optout_delta_backtrack_in_days": 1, + "optout_partition_interval": 86400, + "optout_max_partitions": 30, + "optout_heap_default_capacity": 8192, + "cloud_download_threads": 8, + "cloud_upload_threads": 2, + "cloud_refresh_interval": 60, + "sites_metadata_path": "sites/metadata.json", + "clients_metadata_path": "clients/metadata.json", + "client_side_keypairs_metadata_path": "client_side_keypairs/metadata.json", + "keysets_metadata_path": "keysets/metadata.json", + "keyset_keys_metadata_path": "keyset_keys/metadata.json", + "salts_metadata_path": "salts/metadata.json", + "services_metadata_path": "services/metadata.json", + "service_links_metadata_path": "service_links/metadata.json", + "optout_metadata_path": null, + "enclave_platform": "gcp-oidc", + "optout_inmem_cache": true, + "identity_token_expires_after_seconds": 86400, + "refresh_token_expires_after_seconds": 2592000, + "refresh_identity_token_after_seconds": 3600, + "failure_shutdown_wait_hours": 120, + "sharing_token_expiry_seconds": 2592000, + "validate_service_links": false, + "operator_type": "private", + "uid_instance_id_prefix": "unknown" +} \ No newline at end of file diff --git a/scripts/gcp-oidc/conf/integ-config.json b/scripts/gcp-oidc/conf/integ-config.json new file mode 100644 index 000000000..e25b332d0 --- /dev/null +++ b/scripts/gcp-oidc/conf/integ-config.json @@ -0,0 +1,23 @@ +{ + "sites_metadata_path": "https://core.uidapi.com/sites/refresh", + "clients_metadata_path": "https://core.uidapi.com/clients/refresh", + "keysets_metadata_path": "https://core.uidapi.com/key/keyset/refresh", + "keyset_keys_metadata_path": "https://core.uidapi.com/key/keyset-keys/refresh", + "client_side_keypairs_metadata_path": "https://core.uidapi.com/client_side_keypairs/refresh", + "salts_metadata_path": "https://core.uidapi.com/salt/refresh", + "services_metadata_path": "https://core.uidapi.com/services/refresh", + "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh", + "optout_metadata_path": "https://optout.uidapi.com/optout/refresh", + "core_attest_url": "https://core.uidapi.com/attest", + "cloud_encryption_keys_metadata_path": "https://core.uidapi.com/cloud_encryption_keys/retrieve", + "optout_api_uri": "https://optout.uidapi.com/optout/replicate", + "uid_instance_id_prefix": "unknown", + "optout_s3_folder": "uid-optout-integ/", + "runtime_config_store": { + "type": "http", + "config" : { + "url": "https://core.uidapi.com/operator/config" + }, + "config_scan_period_ms": 300000 + } +} diff --git a/scripts/gcp-oidc/conf/integ-uid2-config.json b/scripts/gcp-oidc/conf/integ-uid2-config.json deleted file mode 100644 index 935514b5a..000000000 --- a/scripts/gcp-oidc/conf/integ-uid2-config.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "sites_metadata_path": "https://core.uidapi.com/sites/refresh", - "clients_metadata_path": "https://core.uidapi.com/clients/refresh", - "keysets_metadata_path": "https://core.uidapi.com/key/keyset/refresh", - "keyset_keys_metadata_path": "https://core.uidapi.com/key/keyset-keys/refresh", - "client_side_keypairs_metadata_path": "https://core.uidapi.com/client_side_keypairs/refresh", - "salts_metadata_path": "https://core.uidapi.com/salt/refresh", - "services_metadata_path": "https://core.uidapi.com/services/refresh", - "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh", - "optout_metadata_path": "https://optout.uidapi.com/optout/refresh", - "core_attest_url": "https://core.uidapi.com/attest", - "optout_api_uri": "https://optout.uidapi.com/optout/replicate", - "optout_s3_folder": "uid-optout-integ/" -} diff --git a/scripts/gcp-oidc/conf/logback.xml b/scripts/gcp-oidc/conf/logback.xml index 6d6dfab78..a3ec18bf7 100644 --- a/scripts/gcp-oidc/conf/logback.xml +++ b/scripts/gcp-oidc/conf/logback.xml @@ -4,12 +4,12 @@ ch.qos.logback.classic.encoder.PatternLayoutEncoder by default --> - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %ex%n + %d{HH:mm:ss.SSS} thread=%thread level=%-5level class=%logger{36} - %msg %ex%n - + diff --git a/scripts/gcp-oidc/conf/prod-uid2-config.json b/scripts/gcp-oidc/conf/prod-config.json similarity index 70% rename from scripts/gcp-oidc/conf/prod-uid2-config.json rename to scripts/gcp-oidc/conf/prod-config.json index f5445a9ec..9e0ec902f 100644 --- a/scripts/gcp-oidc/conf/prod-uid2-config.json +++ b/scripts/gcp-oidc/conf/prod-config.json @@ -1,15 +1,24 @@ -{ - "sites_metadata_path": "https://core.uidapi.com/sites/refresh", - "clients_metadata_path": "https://core.uidapi.com/clients/refresh", - "keysets_metadata_path": "https://core.uidapi.com/key/keyset/refresh", - "keyset_keys_metadata_path": "https://core.uidapi.com/key/keyset-keys/refresh", - "client_side_keypairs_metadata_path": "https://core.uidapi.com/client_side_keypairs/refresh", - "salts_metadata_path": "https://core.uidapi.com/salt/refresh", - "services_metadata_path": "https://core.uidapi.com/services/refresh", - "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh", - "optout_metadata_path": "https://optout.uidapi.com/optout/refresh", - "core_attest_url": "https://core.uidapi.com/attest", - "optout_api_uri": "https://optout.uidapi.com/optout/replicate", - "optout_s3_folder": "optout-v2/", - "identity_token_expires_after_seconds": 259200 -} +{ + "sites_metadata_path": "https://core.uidapi.com/sites/refresh", + "clients_metadata_path": "https://core.uidapi.com/clients/refresh", + "keysets_metadata_path": "https://core.uidapi.com/key/keyset/refresh", + "keyset_keys_metadata_path": "https://core.uidapi.com/key/keyset-keys/refresh", + "client_side_keypairs_metadata_path": "https://core.uidapi.com/client_side_keypairs/refresh", + "salts_metadata_path": "https://core.uidapi.com/salt/refresh", + "services_metadata_path": "https://core.uidapi.com/services/refresh", + "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh", + "optout_metadata_path": "https://optout.uidapi.com/optout/refresh", + "core_attest_url": "https://core.uidapi.com/attest", + "cloud_encryption_keys_metadata_path": "https://core.uidapi.com/cloud_encryption_keys/retrieve", + "optout_api_uri": "https://optout.uidapi.com/optout/replicate", + "optout_s3_folder": "optout-v2/", + "identity_token_expires_after_seconds": 259200, + "uid_instance_id_prefix": "unknown", + "runtime_config_store": { + "type": "http", + "config" : { + "url": "https://core.uidapi.com/operator/config" + }, + "config_scan_period_ms": 300000 + } +} diff --git a/scripts/gcp-oidc/entrypoint.sh b/scripts/gcp-oidc/entrypoint.sh deleted file mode 100644 index 133b54486..000000000 --- a/scripts/gcp-oidc/entrypoint.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/sh -# -# This script must be compatible with Ash (provided in eclipse-temurin Docker image) and Bash - -# -- set API tokens -if [ -z "${API_TOKEN_SECRET_NAME}" ]; then - echo "API_TOKEN_SECRET_NAME cannot be empty" - exit 1 -fi - -if [ -z "${CORE_BASE_URL}" ]; then - echo "CORE_BASE_URL cannot be empty" - exit 1 -fi - -if [ -z "${OPTOUT_BASE_URL}" ]; then - echo "OPTOUT_BASE_URL cannot be empty" - exit 1 -fi - -export gcp_secret_version_name="${API_TOKEN_SECRET_NAME}" - -# -- locate config file -if [ -z "${DEPLOYMENT_ENVIRONMENT}" ]; then - echo "DEPLOYMENT_ENVIRONMENT cannot be empty" - exit 1 -fi -if [ "${DEPLOYMENT_ENVIRONMENT}" != 'prod' -a "${DEPLOYMENT_ENVIRONMENT}" != 'integ' ]; then - echo "Unrecognized DEPLOYMENT_ENVIRONMENT ${DEPLOYMENT_ENVIRONMENT}" - exit 1 -fi - -TARGET_CONFIG="/app/conf/${DEPLOYMENT_ENVIRONMENT}-uid2-config.json" -if [ ! -f "${TARGET_CONFIG}" ]; then - echo "Unrecognized config ${TARGET_CONFIG}" - exit 1 -fi - -FINAL_CONFIG="/tmp/final-config.json" -echo "-- copying ${TARGET_CONFIG} to ${FINAL_CONFIG}" -cp ${TARGET_CONFIG} ${FINAL_CONFIG} -if [ $? -ne 0 ]; then - echo "Failed to create ${FINAL_CONFIG} with error code $?" - exit 1 -fi - -# -- using hardcoded domains is fine because they should not be changed frequently -echo "-- replacing URLs by ${CORE_BASE_URL} and ${OPTOUT_BASE_URL}" -sed -i "s#https://core.uidapi.com#${CORE_BASE_URL}#g" ${FINAL_CONFIG} - -sed -i "s#https://optout.uidapi.com#${OPTOUT_BASE_URL}#g" ${FINAL_CONFIG} - - -cat $FINAL_CONFIG - -# -- start operator -echo "-- starting java application" -java \ - -XX:MaxRAMPercentage=95 -XX:-UseCompressedOops -XX:+PrintFlagsFinal \ - -Djava.security.egd=file:/dev/./urandom \ - -Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory \ - -Dlogback.configurationFile=/app/conf/logback.xml \ - -Dvertx-config-path=${FINAL_CONFIG} \ - -jar ${JAR_NAME}-${JAR_VERSION}.jar diff --git a/scripts/gcp-oidc/gcp.py b/scripts/gcp-oidc/gcp.py new file mode 100644 index 000000000..6b3525e71 --- /dev/null +++ b/scripts/gcp-oidc/gcp.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 + +import os +import shutil +from typing import Dict +import sys +import logging +import requests +import re +from google.cloud import secretmanager +from google.auth.exceptions import DefaultCredentialsError +from google.api_core.exceptions import PermissionDenied, NotFound +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from confidential_compute import ConfidentialCompute, ConfigurationMissingError, OperatorKeyNotFoundError, OperatorKeyPermissionError, ConfidentialComputeStartupError + + +class AuxiliaryConfig: + GCP_METADATA: str = "169.254.169.254" + GCP_HEADER: dict = {"Metadata-Flavor": "Google"} + + @classmethod + def get_gcp_instance_id_url(cls) -> str: + return f"http://{cls.GCP_METADATA}/computeMetadata/v1/instance/id" + + @classmethod + def get_gcp_image_url(cls) -> str: + return f"http://{cls.GCP_METADATA}/computeMetadata/v1/instance/image" + +class GCP(ConfidentialCompute): + + def __init__(self): + super().__init__() + + def _set_confidential_config(self, secret_identifier=None) -> None: + + keys_mapping = { + "core_base_url": "CORE_BASE_URL", + "optout_base_url": "OPTOUT_BASE_URL", + "environment": "DEPLOYMENT_ENVIRONMENT", + "skip_validations": "SKIP_VALIDATIONS", + "debug_mode": "DEBUG_MODE", + } + self.configs = { + key: (os.environ[env_var].lower() == "true" if key in ["skip_validations", "debug_mode"] else os.environ[env_var]) + for key, env_var in keys_mapping.items() if env_var in os.environ + } + + if not os.getenv("API_TOKEN_SECRET_NAME"): + raise ConfigurationMissingError(self.__class__.__name__, ["API_TOKEN_SECRET_NAME"]) + try: + client = secretmanager.SecretManagerServiceClient() + secret_version_name = f"{os.getenv("API_TOKEN_SECRET_NAME")}" + response = client.access_secret_version(name=secret_version_name) + secret_value = response.payload.data.decode("UTF-8") + except (PermissionDenied, DefaultCredentialsError) as e: + raise OperatorKeyPermissionError(self.__class__.__name__, str(e)) + except NotFound: + raise OperatorKeyNotFoundError(self.__class__.__name__, f"Secret Manager {os.getenv("API_TOKEN_SECRET_NAME")}") + self.configs["operator_key"] = secret_value + instance_id, version = self.__get_gcp_instance_info() + self.configs["uid_instance_id_prefix"] = self.get_uid_instance_id(identifier=instance_id, version=version) + + def __populate_operator_config(self, destination): + target_config = f"/app/conf/{self.configs["environment"].lower()}-config.json" + shutil.copy(target_config, destination) + with open(destination, 'r') as file: + config = file.read() + config = config.replace("https://core.uidapi.com", self.configs.get("core_base_url")) + config = config.replace("https://optout.uidapi.com", self.configs.get("optout_base_url")) + config = config.replace("unknown", self.configs.get("uid_instance_id_prefix")) + with open(destination, 'w') as file: + file.write(config) + + def _setup_auxiliaries(self) -> None: + """ No Auxiliariy service required for GCP Confidential compute. """ + pass + + def _validate_auxiliaries(self) -> None: + """ No Auxiliariy service required for GCP Confidential compute. """ + pass + + def __get_gcp_instance_info(self) -> tuple[str, str]: + """Fetches the GCP instance ID, and image version.""" + try: + response = requests.get(AuxiliaryConfig.get_gcp_instance_id_url(), headers=AuxiliaryConfig.GCP_HEADER, timeout=2) + response.raise_for_status() + instance_id = response.text.strip() + image_response = requests.get(AuxiliaryConfig.get_gcp_image_url(), headers=AuxiliaryConfig.GCP_HEADER, timeout=2) + image_response.raise_for_status() + image = image_response.text.strip() + match = re.search(r":(\d+\.\d+\.\d+)", image) + version = match.group(1) if match else "unknown" + return instance_id, version + except requests.RequestException as e: + raise RuntimeError(f"Failed to fetch GCP instance info: {e}") + + def run_compute(self) -> None: + self._set_confidential_config() + logging.info("Fetched configs") + if not self.configs.get("skip_validations"): + self.validate_configuration() + config_locaton = "/tmp/final-config.json" + self.__populate_operator_config(config_locaton) + os.environ["gcp_secret_version_name"] = os.getenv("API_TOKEN_SECRET_NAME") + java_command = [ + "java", + "-XX:MaxRAMPercentage=95", + "-XX:-UseCompressedOops", + "-XX:+PrintFlagsFinal", + "-Djava.security.egd=file:/dev/./urandom", + "-Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory", + "-Dlogback.configurationFile=/app/conf/logback.xml", + f"-Dvertx-config-path={config_locaton}", + "-jar", + f"{os.getenv("JAR_NAME")}-{os.getenv("JAR_VERSION")}.jar" + ] + self.run_command(java_command) + +if __name__ == "__main__": + try: + gcp = GCP() + gcp.run_compute() + except ConfidentialComputeStartupError as e: + logging.error(f"Failed starting up Confidential Compute. Please checks the logs for errors and retry {e}") + except Exception as e: + logging.error(f"Unexpected failure while starting up Confidential Compute. Please contact UID support team with this log {e}") + diff --git a/scripts/gcp-oidc/terraform/main.tf b/scripts/gcp-oidc/terraform/main.tf index aefb68362..3a600e26e 100644 --- a/scripts/gcp-oidc/terraform/main.tf +++ b/scripts/gcp-oidc/terraform/main.tf @@ -106,6 +106,7 @@ resource "google_compute_instance_template" "uid_operator" { tee-image-reference = var.uid_operator_image tee-container-log-redirect = true tee-restart-policy = "Never" + tee-env-DEBUG_MODE = var.debug_mode tee-env-DEPLOYMENT_ENVIRONMENT = var.uid_deployment_env tee-env-API_TOKEN_SECRET_NAME = module.secret-manager.secret_versions[0] tee-env-CORE_BASE_URL = var.uid_deployment_env == "integ" ? "https://core-integ.uidapi.com" : "https://core-prod.uidapi.com" diff --git a/scripts/gcp/conf/integ-config.json b/scripts/gcp/conf/integ-config.json index d3fb9e9ff..99d74dc7e 100644 --- a/scripts/gcp/conf/integ-config.json +++ b/scripts/gcp/conf/integ-config.json @@ -13,7 +13,6 @@ "refresh_identity_token_after_seconds": 3600, "enclave_platform": "gcp-vmid", "service_instances": 16, - "allow_legacy_api": false, "sharing_token_expiry_seconds": 2592000, "operator_type": "private" } diff --git a/scripts/gcp/conf/prod-config.json b/scripts/gcp/conf/prod-config.json index 836349c19..f8e1bed78 100644 --- a/scripts/gcp/conf/prod-config.json +++ b/scripts/gcp/conf/prod-config.json @@ -13,7 +13,6 @@ "refresh_identity_token_after_seconds": 3600, "enclave_platform": "gcp-vmid", "service_instances": 16, - "allow_legacy_api": false, "sharing_token_expiry_seconds": 2592000, "operator_type": "private" } diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index 4d32b9034..31dcd4a4d 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -11,7 +11,6 @@ public class Config extends com.uid2.shared.Const.Config { public static final String StorageMockProp = "storage_mock"; public static final String StatsCollectorEventBus = "StatsCollector"; public static final String FailureShutdownWaitHoursProp = "failure_shutdown_wait_hours"; - public static final String AllowLegacyAPIProp = "allow_legacy_api"; public static final String SharingTokenExpiryProp = "sharing_token_expiry_seconds"; public static final String MaxBidstreamLifetimeSecondsProp = "max_bidstream_lifetime_seconds"; public static final String AllowClockSkewSecondsProp = "allow_clock_skew_seconds"; @@ -20,6 +19,8 @@ public class Config extends com.uid2.shared.Const.Config { public static final String ValidateServiceLinks = "validate_service_links"; public static final String OperatorTypeProp = "operator_type"; public static final String EnclavePlatformProp = "enclave_platform"; + public static final String EncryptedFiles = "encrypted_files"; + public static final String PodTerminationCheckInterval = "pod_termination_check_interval"; public static final String AzureVaultNameProp = "azure_vault_name"; public static final String AzureSecretNameProp = "azure_secret_name"; @@ -29,5 +30,11 @@ public class Config extends com.uid2.shared.Const.Config { public static final String OptOutStatusMaxRequestSize = "optout_status_max_request_size"; public static final String MaxInvalidPaths = "logging_limit_max_invalid_paths_per_interval"; public static final String MaxVersionBucketsPerSite = "logging_limit_max_version_buckets_per_site"; + + public static final String ConfigScanPeriodMsProp = "config_scan_period_ms"; + public static final String IdentityV3Prop = "identity_v3"; + public static final String DisableOptoutTokenProp = "disable_optout_token"; + public static final String EnableRemoteConfigProp = "enable_remote_config"; + public static final String RuntimeConfigMetadataPathProp = "runtime_config_metadata_path"; } } diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index dad32611d..9fb10d1b8 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -8,27 +8,32 @@ import com.uid2.operator.monitoring.IStatsCollectorQueue; import com.uid2.operator.monitoring.OperatorMetrics; import com.uid2.operator.monitoring.StatsCollectorVerticle; -import com.uid2.operator.service.SecureLinkValidatorService; -import com.uid2.operator.service.ShutdownService; +import com.uid2.operator.reader.RotatingCloudEncryptionKeyApiProvider; +import com.uid2.operator.service.*; +import com.uid2.operator.store.*; +import com.uid2.operator.store.BootstrapConfigStore; import com.uid2.operator.vertx.Endpoints; import com.uid2.operator.vertx.OperatorShutdownHandler; -import com.uid2.operator.store.CloudSyncOptOutStore; -import com.uid2.operator.store.OptOutCloudStorage; import com.uid2.operator.vertx.UIDOperatorVerticle; import com.uid2.shared.ApplicationVersion; import com.uid2.shared.Utils; import com.uid2.shared.attest.*; +import com.uid2.shared.audit.UidInstanceIdProvider; import com.uid2.shared.cloud.*; import com.uid2.shared.jmx.AdminApi; import com.uid2.shared.optout.OptOutCloudSync; import com.uid2.shared.store.CloudPath; -import com.uid2.shared.store.RotatingSaltProvider; +import com.uid2.shared.store.salt.EncryptedRotatingSaltProvider; +import com.uid2.shared.store.salt.RotatingSaltProvider; import com.uid2.shared.store.reader.*; import com.uid2.shared.store.scope.GlobalScope; +import com.uid2.shared.util.HTTPPathMetricFilter; import com.uid2.shared.vertx.CloudSyncVerticle; import com.uid2.shared.vertx.ICloudSync; import com.uid2.shared.vertx.RotatingStoreVerticle; import com.uid2.shared.vertx.VertxUtils; +import com.uid2.shared.health.HealthManager; +import com.uid2.shared.health.PodTerminationMonitor; import io.micrometer.core.instrument.Gauge; import io.micrometer.core.instrument.Meter; import io.micrometer.core.instrument.MeterRegistry; @@ -37,9 +42,9 @@ import io.micrometer.core.instrument.distribution.DistributionStatisticConfig; import io.micrometer.prometheus.PrometheusMeterRegistry; import io.micrometer.prometheus.PrometheusRenameFilter; +import io.vertx.config.ConfigRetriever; import io.vertx.core.*; import io.vertx.core.http.HttpServerOptions; -import io.vertx.core.http.impl.HttpUtils; import io.vertx.core.json.JsonObject; import io.vertx.micrometer.*; import io.vertx.micrometer.backends.BackendRegistries; @@ -57,6 +62,7 @@ import java.util.*; import java.util.function.Supplier; +import static com.uid2.operator.Const.Config.EnableRemoteConfigProp; import static io.micrometer.core.instrument.Metrics.globalRegistry; public class Main { @@ -67,6 +73,7 @@ public class Main { private final ApplicationVersion appVersion; private final ICloudStorage fsLocal; private final ICloudStorage fsOptOut; + private final IConfigStore configStore; private final RotatingSiteStore siteProvider; private final RotatingClientKeyProvider clientKeyProvider; private final RotatingKeysetKeyStore keysetKeyStore; @@ -74,19 +81,24 @@ public class Main { private final RotatingClientSideKeypairStore clientSideKeypairProvider; private final RotatingSaltProvider saltProvider; private final CloudSyncOptOutStore optOutStore; + private final boolean encryptedCloudFilesEnabled; private OperatorShutdownHandler shutdownHandler = null; private final OperatorMetrics metrics; + private final boolean useRemoteConfig; private final boolean clientSideTokenGenerate; private final boolean validateServiceLinks; private IStatsCollectorQueue _statsCollectorQueue; private RotatingServiceStore serviceProvider; private RotatingServiceLinkStore serviceLinkProvider; + private RotatingCloudEncryptionKeyApiProvider cloudEncryptionKeyProvider; + private final UidInstanceIdProvider uidInstanceIdProvider; public Main(Vertx vertx, JsonObject config) throws Exception { this.vertx = vertx; this.config = config; this.appVersion = ApplicationVersion.load("uid2-operator", "uid2-shared", "uid2-attestation-api"); + HealthManager.instance.registerGenericComponent(new PodTerminationMonitor(config.getInteger(Const.Config.PodTerminationCheckInterval, 3000))); // allow to switch between in-mem optout file cache and on-disk file cache if (config.getBoolean(Const.Config.OptOutInMemCacheProp)) { @@ -96,9 +108,12 @@ public Main(Vertx vertx, JsonObject config) throws Exception { } boolean useStorageMock = config.getBoolean(Const.Config.StorageMockProp, false); + this.useRemoteConfig = config.getBoolean(EnableRemoteConfigProp, false); this.clientSideTokenGenerate = config.getBoolean(Const.Config.EnableClientSideTokenGenerate, false); this.validateServiceLinks = config.getBoolean(Const.Config.ValidateServiceLinks, false); + this.encryptedCloudFilesEnabled = config.getBoolean(Const.Config.EncryptedFiles, false); this.shutdownHandler = new OperatorShutdownHandler(Duration.ofHours(12), Duration.ofHours(config.getInteger(Const.Config.SaltsExpiredShutdownHours, 12)), Clock.systemUTC(), new ShutdownService()); + this.uidInstanceIdProvider = new UidInstanceIdProvider(config); String coreAttestUrl = this.config.getString(Const.Config.CoreAttestUrlProp); @@ -132,18 +147,54 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.fsOptOut = configureCloudOptOutStore(); } - String sitesMdPath = this.config.getString(Const.Config.SitesMetadataPathProp); - String keypairMdPath = this.config.getString(Const.Config.ClientSideKeypairsMetadataPathProp); - this.clientSideKeypairProvider = new RotatingClientSideKeypairStore(fsStores, new GlobalScope(new CloudPath(keypairMdPath))); - String clientsMdPath = this.config.getString(Const.Config.ClientsMetadataPathProp); - this.clientKeyProvider = new RotatingClientKeyProvider(fsStores, new GlobalScope(new CloudPath(clientsMdPath))); - String keysetKeysMdPath = this.config.getString(Const.Config.KeysetKeysMetadataPathProp); - this.keysetKeyStore = new RotatingKeysetKeyStore(fsStores, new GlobalScope(new CloudPath(keysetKeysMdPath))); - String keysetMdPath = this.config.getString(Const.Config.KeysetsMetadataPathProp); - this.keysetProvider = new RotatingKeysetProvider(fsStores, new GlobalScope(new CloudPath(keysetMdPath))); - String saltsMdPath = this.config.getString(Const.Config.SaltsMetadataPathProp); - this.saltProvider = new RotatingSaltProvider(fsStores, saltsMdPath); + if (this.encryptedCloudFilesEnabled) { + String cloudEncryptionKeyMdPath = this.config.getString(Const.Config.CloudEncryptionKeysMetadataPathProp); + this.cloudEncryptionKeyProvider = new RotatingCloudEncryptionKeyApiProvider(fsStores, + new GlobalScope(new CloudPath(cloudEncryptionKeyMdPath))); + + String keypairMdPath = this.config.getString(Const.Config.ClientSideKeypairsMetadataPathProp); + this.clientSideKeypairProvider = new RotatingClientSideKeypairStore(fsStores, + new GlobalScope(new CloudPath(keypairMdPath)), cloudEncryptionKeyProvider); + String clientsMdPath = this.config.getString(Const.Config.ClientsMetadataPathProp); + this.clientKeyProvider = new RotatingClientKeyProvider(fsStores, new GlobalScope(new CloudPath(clientsMdPath)), + cloudEncryptionKeyProvider); + String keysetKeysMdPath = this.config.getString(Const.Config.KeysetKeysMetadataPathProp); + this.keysetKeyStore = new RotatingKeysetKeyStore(fsStores, new GlobalScope(new CloudPath(keysetKeysMdPath)), + cloudEncryptionKeyProvider); + String keysetMdPath = this.config.getString(Const.Config.KeysetsMetadataPathProp); + this.keysetProvider = new RotatingKeysetProvider(fsStores, new GlobalScope(new CloudPath(keysetMdPath)), + cloudEncryptionKeyProvider); + String saltsMdPath = this.config.getString(Const.Config.SaltsMetadataPathProp); + this.saltProvider = new EncryptedRotatingSaltProvider(fsStores, cloudEncryptionKeyProvider, + new GlobalScope(new CloudPath(saltsMdPath))); + String sitesMdPath = this.config.getString(Const.Config.SitesMetadataPathProp); + this.siteProvider = clientSideTokenGenerate + ? new RotatingSiteStore(fsStores, new GlobalScope(new CloudPath(sitesMdPath)), + cloudEncryptionKeyProvider) + : null; + } else { + String keypairMdPath = this.config.getString(Const.Config.ClientSideKeypairsMetadataPathProp); + this.clientSideKeypairProvider = new RotatingClientSideKeypairStore(fsStores, new GlobalScope(new CloudPath(keypairMdPath))); + String clientsMdPath = this.config.getString(Const.Config.ClientsMetadataPathProp); + this.clientKeyProvider = new RotatingClientKeyProvider(fsStores, new GlobalScope(new CloudPath(clientsMdPath))); + String keysetKeysMdPath = this.config.getString(Const.Config.KeysetKeysMetadataPathProp); + this.keysetKeyStore = new RotatingKeysetKeyStore(fsStores, new GlobalScope(new CloudPath(keysetKeysMdPath))); + String keysetMdPath = this.config.getString(Const.Config.KeysetsMetadataPathProp); + this.keysetProvider = new RotatingKeysetProvider(fsStores, new GlobalScope(new CloudPath(keysetMdPath))); + String saltsMdPath = this.config.getString(Const.Config.SaltsMetadataPathProp); + this.saltProvider = new RotatingSaltProvider(fsStores, saltsMdPath); + String sitesMdPath = this.config.getString(Const.Config.SitesMetadataPathProp); + this.siteProvider = clientSideTokenGenerate ? new RotatingSiteStore(fsStores, new GlobalScope(new CloudPath(sitesMdPath))) : null; + } + this.optOutStore = new CloudSyncOptOutStore(vertx, fsLocal, this.config, operatorKey, Clock.systemUTC()); + + if (useRemoteConfig) { + String configMdPath = this.config.getString(Const.Config.RuntimeConfigMetadataPathProp); + this.configStore = new RuntimeConfigStore(fsStores, configMdPath); + } else { + this.configStore = new BootstrapConfigStore(this.config); + } if (this.validateServiceLinks) { String serviceMdPath = this.config.getString(Const.Config.ServiceMetadataPathProp); @@ -152,23 +203,29 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.serviceLinkProvider = new RotatingServiceLinkStore(fsStores, new GlobalScope(new CloudPath(serviceLinkMdPath))); } - this.siteProvider = clientSideTokenGenerate ? new RotatingSiteStore(fsStores, new GlobalScope(new CloudPath(sitesMdPath))) : null; - if (useStorageMock && coreAttestUrl == null) { if (clientSideTokenGenerate) { this.siteProvider.loadContent(); this.clientSideKeypairProvider.loadContent(); } - this.clientKeyProvider.loadContent(); - this.saltProvider.loadContent(); - this.keysetProvider.loadContent(); - this.keysetKeyStore.loadContent(); + if (useRemoteConfig) { + this.configStore.loadContent(); + } if (this.validateServiceLinks) { this.serviceProvider.loadContent(); this.serviceLinkProvider.loadContent(); } + if (this.encryptedCloudFilesEnabled) { + this.cloudEncryptionKeyProvider.loadContent(); + } + + this.clientKeyProvider.loadContent(); + this.saltProvider.loadContent(); + this.keysetProvider.loadContent(); + this.keysetKeyStore.loadContent(); + try { getKeyManager().getMasterKey(); } catch (KeyManager.NoActiveKeyException e) { @@ -193,17 +250,19 @@ public static void main(String[] args) throws Exception { final String vertxConfigPath = System.getProperty(Const.Config.VERTX_CONFIG_PATH_PROP); if (vertxConfigPath != null) { - System.out.format("Running CUSTOM CONFIG mode, config: %s\n", vertxConfigPath); + LOGGER.info("Running CUSTOM CONFIG mode, config: {}", vertxConfigPath); } else if (!Utils.isProductionEnvironment()) { - System.out.format("Running LOCAL DEBUG mode, config: %s\n", Const.Config.LOCAL_CONFIG_PATH); + LOGGER.info("Running LOCAL DEBUG mode, config: {}", Const.Config.LOCAL_CONFIG_PATH); System.setProperty(Const.Config.VERTX_CONFIG_PATH_PROP, Const.Config.LOCAL_CONFIG_PATH); } else { - System.out.format("Running PRODUCTION mode, config: %s\n", Const.Config.OVERRIDE_CONFIG_PATH); + LOGGER.info("Running PRODUCTION mode, config: {}", Const.Config.OVERRIDE_CONFIG_PATH); } Vertx vertx = createVertx(); - VertxUtils.createConfigRetriever(vertx).getConfig(ar -> { + ConfigRetriever configRetriever = VertxUtils.createConfigRetriever(vertx); + + configRetriever.getConfig(ar -> { if (ar.failed()) { LOGGER.error("Unable to read config: " + ar.cause().getMessage(), ar.cause()); return; @@ -265,8 +324,11 @@ private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) { } private void run() throws Exception { + this.createVertxInstancesMetric(); + this.createVertxEventLoopsMetric(); + Supplier operatorVerticleSupplier = () -> { - UIDOperatorVerticle verticle = new UIDOperatorVerticle(config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); + UIDOperatorVerticle verticle = new UIDOperatorVerticle(configStore, config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse, this.uidInstanceIdProvider); return verticle; }; @@ -285,19 +347,19 @@ private void run() throws Exception { }); compositePromise.future() - .compose(v -> { - metrics.setup(); - vertx.setPeriodic(60000, id -> metrics.update()); - - Promise promise = Promise.promise(); - vertx.deployVerticle(operatorVerticleSupplier, options, promise); - return promise.future(); - }) - .onFailure(t -> { - LOGGER.error("Failed to bootstrap operator: " + t.getMessage(), new Exception(t)); - vertx.close(); - System.exit(1); - }); + .compose(v -> { + metrics.setup(); + vertx.setPeriodic(60000, id -> metrics.update()); + + Promise promise = Promise.promise(); + vertx.deployVerticle(operatorVerticleSupplier, options, promise); + return promise.future(); + }) + .onFailure(t -> { + LOGGER.error("Failed to bootstrap operator: " + t.getMessage(), new Exception(t)); + vertx.close(); + System.exit(1); + }); } private Future createStoreVerticles() throws Exception { @@ -306,16 +368,21 @@ private Future createStoreVerticles() throws Exception { siteProvider.getMetadata(); clientSideKeypairProvider.getMetadata(); } - clientKeyProvider.getMetadata(); - keysetKeyStore.getMetadata(); - keysetProvider.getMetadata(); - saltProvider.getMetadata(); if (validateServiceLinks) { serviceProvider.getMetadata(); serviceLinkProvider.getMetadata(); } + if (encryptedCloudFilesEnabled) { + cloudEncryptionKeyProvider.getMetadata(); + } + + clientKeyProvider.getMetadata(); + keysetKeyStore.getMetadata(); + keysetProvider.getMetadata(); + saltProvider.getMetadata(); + // create cloud sync for optout store OptOutCloudSync optOutCloudSync = new OptOutCloudSync(config, false); this.optOutStore.registerCloudSync(optOutCloudSync); @@ -323,10 +390,24 @@ private Future createStoreVerticles() throws Exception { // create rotating store verticles to poll for updates Promise promise = Promise.promise(); List fs = new ArrayList<>(); + if (clientSideTokenGenerate) { fs.add(createAndDeployRotatingStoreVerticle("site", siteProvider, "site_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("client_side_keypairs", clientSideKeypairProvider, "client_side_keypairs_refresh_ms")); } + + if (validateServiceLinks) { + fs.add(createAndDeployRotatingStoreVerticle("service", serviceProvider, "service_refresh_ms")); + fs.add(createAndDeployRotatingStoreVerticle("service_link", serviceLinkProvider, "service_link_refresh_ms")); + } + + if (encryptedCloudFilesEnabled) { + fs.add(createAndDeployRotatingStoreVerticle("cloud_encryption_keys", cloudEncryptionKeyProvider, "cloud_encryption_keys_refresh_ms")); + } + + if (useRemoteConfig) { + fs.add(createAndDeployRotatingStoreVerticle("runtime_config", (RuntimeConfigStore) configStore, Const.Config.ConfigScanPeriodMsProp)); + } fs.add(createAndDeployRotatingStoreVerticle("auth", clientKeyProvider, "auth_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("keyset", keysetProvider, "keyset_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("keysetkey", keysetKeyStore, "keysetkey_refresh_ms")); @@ -337,10 +418,6 @@ private Future createStoreVerticles() throws Exception { else promise.complete(); }); - if (validateServiceLinks) { - fs.add(createAndDeployRotatingStoreVerticle("service", serviceProvider, "service_refresh_ms")); - fs.add(createAndDeployRotatingStoreVerticle("service_link", serviceLinkProvider, "service_link_refresh_ms")); - } return promise.future(); } @@ -349,26 +426,21 @@ private Future createAndDeployRotatingStoreVerticle(String name, IMetada final int intervalMs = config.getInteger(storeRefreshConfigMs, 10000); RotatingStoreVerticle rotatingStoreVerticle = new RotatingStoreVerticle(name, intervalMs, store); - Promise promise = Promise.promise(); - vertx.deployVerticle(rotatingStoreVerticle, promise); - return promise.future(); + return vertx.deployVerticle(rotatingStoreVerticle); } private Future createAndDeployCloudSyncStoreVerticle(String name, ICloudStorage fsCloud, ICloudSync cloudSync) { - Promise promise = Promise.promise(); CloudSyncVerticle cloudSyncVerticle = new CloudSyncVerticle(name, fsCloud, fsLocal, cloudSync, config); - vertx.deployVerticle(cloudSyncVerticle, promise); - return promise.future() + return vertx.deployVerticle(cloudSyncVerticle) .onComplete(v -> setupTimerEvent(cloudSyncVerticle.eventRefresh())); } private Future createAndDeployStatsCollector() { - Promise promise = Promise.promise(); StatsCollectorVerticle statsCollectorVerticle = new StatsCollectorVerticle(60000, config.getInteger(Const.Config.MaxInvalidPaths, 50), config.getInteger(Const.Config.MaxVersionBucketsPerSite, 50)); - vertx.deployVerticle(statsCollectorVerticle, promise); + Future result = vertx.deployVerticle(statsCollectorVerticle); _statsCollectorQueue = statsCollectorVerticle; - return promise.future(); + return result; } private void setupTimerEvent(String eventCloudRefresh) { @@ -385,7 +457,7 @@ private static Vertx createVertx() { MBeanServer server = ManagementFactory.getPlatformMBeanServer(); server.registerMBean(AdminApi.instance, objectName); } catch (InstanceAlreadyExistsException | MBeanRegistrationException | NotCompliantMBeanException | MalformedObjectNameException e) { - System.err.format("%s", e.getMessage()); + LOGGER.error("mBean initialisation failed {}", e.getMessage(), e); System.exit(-1); } @@ -414,7 +486,7 @@ private static Vertx createVertx() { } private static void setupMetrics(MicrometerMetricsOptions metricOptions) { - BackendRegistries.setupBackend(metricOptions); + BackendRegistries.setupBackend(metricOptions, null); MeterRegistry backendRegistry = BackendRegistries.getDefaultNow(); if (backendRegistry instanceof PrometheusMeterRegistry) { @@ -425,14 +497,8 @@ private static void setupMetrics(MicrometerMetricsOptions metricOptions) { prometheusRegistry.config() // providing common renaming for prometheus metric, e.g. "hello.world" to "hello_world" .meterFilter(new PrometheusRenameFilter()) - .meterFilter(MeterFilter.replaceTagValues(Label.HTTP_PATH.toString(), actualPath -> { - try { - String normalized = HttpUtils.normalizePath(actualPath).split("\\?")[0]; - return Endpoints.pathSet().contains(normalized) ? normalized : "/unknown"; - } catch (IllegalArgumentException e) { - return actualPath; - } - })) + .meterFilter(MeterFilter.replaceTagValues(Label.HTTP_PATH.toString(), + actualPath -> HTTPPathMetricFilter.filterPath(actualPath, Endpoints.pathSet()))) // Don't record metrics for 404s. .meterFilter(MeterFilter.deny(id -> id.getName().startsWith(MetricsDomain.HTTP_SERVER.getPrefix()) && @@ -461,20 +527,32 @@ public DistributionStatisticConfig configure(Meter.Id id, DistributionStatisticC // retrieve image version (will unify when uid2-common is used) final String version = Optional.ofNullable(System.getenv("IMAGE_VERSION")).orElse("unknown"); Gauge - .builder("app.status", () -> 1) + .builder("app_status", () -> 1) .description("application version and status") .tags("version", version) .register(globalRegistry); } - private Map.Entry createUidClients(Vertx vertx, String attestationUrl, String clientApiToken, Handler> responseWatcher) throws Exception { + private void createVertxInstancesMetric() { + Gauge.builder("uid2_vertx_service_instances", () -> config.getInteger("service_instances")) + .description("gauge for number of vertx service instances requested") + .register(Metrics.globalRegistry); + } + + private void createVertxEventLoopsMetric() { + Gauge.builder("uid2_vertx_event_loop_threads", () -> VertxOptions.DEFAULT_EVENT_LOOP_POOL_SIZE) + .description("gauge for number of vertx event loop threads") + .register(Metrics.globalRegistry); + } + + private Map.Entry createUidClients(Vertx vertx, String attestationUrl, String clientApiToken, Handler> responseWatcher) throws Exception { AttestationResponseHandler attestationResponseHandler = getAttestationTokenRetriever(vertx, attestationUrl, clientApiToken, responseWatcher); - UidCoreClient coreClient = new UidCoreClient(clientApiToken, CloudUtils.defaultProxy, attestationResponseHandler); - UidOptOutClient optOutClient = new UidOptOutClient(clientApiToken, CloudUtils.defaultProxy, attestationResponseHandler); + UidCoreClient coreClient = new UidCoreClient(clientApiToken, CloudUtils.defaultProxy, attestationResponseHandler, this.encryptedCloudFilesEnabled, this.uidInstanceIdProvider); + UidOptOutClient optOutClient = new UidOptOutClient(clientApiToken, CloudUtils.defaultProxy, attestationResponseHandler, this.uidInstanceIdProvider); return new AbstractMap.SimpleEntry<>(coreClient, optOutClient); } - private AttestationResponseHandler getAttestationTokenRetriever(Vertx vertx, String attestationUrl, String clientApiToken, Handler> responseWatcher) throws Exception { + private AttestationResponseHandler getAttestationTokenRetriever(Vertx vertx, String attestationUrl, String clientApiToken, Handler> responseWatcher) throws Exception { String enclavePlatform = this.config.getString(Const.Config.EnclavePlatformProp); String operatorType = this.config.getString(Const.Config.OperatorTypeProp, ""); @@ -506,7 +584,7 @@ private AttestationResponseHandler getAttestationTokenRetriever(Vertx vertx, Str throw new IllegalArgumentException(String.format("enclave_platform is providing the wrong value: %s", enclavePlatform)); } - return new AttestationResponseHandler(vertx, attestationUrl, clientApiToken, operatorType, this.appVersion, attestationProvider, responseWatcher, CloudUtils.defaultProxy); + return new AttestationResponseHandler(vertx, attestationUrl, clientApiToken, operatorType, this.appVersion, attestationProvider, responseWatcher, CloudUtils.defaultProxy, this.uidInstanceIdProvider); } private IOperatorKeyRetriever createOperatorKeyRetriever() throws Exception { diff --git a/src/main/java/com/uid2/operator/model/AdvertisingTokenInput.java b/src/main/java/com/uid2/operator/model/AdvertisingTokenInput.java deleted file mode 100644 index b5ffcb89a..000000000 --- a/src/main/java/com/uid2/operator/model/AdvertisingTokenInput.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.uid2.operator.model; - -import java.time.Instant; - -import com.uid2.operator.model.userIdentity.RawUidIdentity; -import com.uid2.shared.model.TokenVersion; - -public class AdvertisingTokenInput extends VersionedToken { - public final OperatorIdentity operatorIdentity; - public final SourcePublisher sourcePublisher; - public final RawUidIdentity rawUidIdentity; - - public AdvertisingTokenInput(TokenVersion version, Instant createdAt, Instant expiresAt, OperatorIdentity operatorIdentity, - SourcePublisher sourcePublisher, RawUidIdentity rawUidIdentity) { - super(version, createdAt, expiresAt); - this.operatorIdentity = operatorIdentity; - this.sourcePublisher = sourcePublisher; - this.rawUidIdentity = rawUidIdentity; - } -} - diff --git a/src/main/java/com/uid2/operator/model/AdvertisingTokenRequest.java b/src/main/java/com/uid2/operator/model/AdvertisingTokenRequest.java new file mode 100644 index 000000000..d63fa66a8 --- /dev/null +++ b/src/main/java/com/uid2/operator/model/AdvertisingTokenRequest.java @@ -0,0 +1,28 @@ +package com.uid2.operator.model; + +import java.time.Instant; + +import com.uid2.operator.model.identities.RawUid; +import com.uid2.operator.util.PrivacyBits; +import com.uid2.shared.model.TokenVersion; + +// class containing enough information to create a new uid token (aka advertising token) +public class AdvertisingTokenRequest extends VersionedTokenRequest { + public final OperatorIdentity operatorIdentity; + public final SourcePublisher sourcePublisher; + public final RawUid rawUid; + public final PrivacyBits privacyBits; + public final Instant establishedAt; + + public AdvertisingTokenRequest(TokenVersion version, Instant createdAt, Instant expiresAt, OperatorIdentity operatorIdentity, + SourcePublisher sourcePublisher, RawUid rawUid, PrivacyBits privacyBits, + Instant establishedAt) { + super(version, createdAt, expiresAt); + this.operatorIdentity = operatorIdentity; + this.sourcePublisher = sourcePublisher; + this.rawUid = rawUid; + this.privacyBits = privacyBits; + this.establishedAt = establishedAt; + } +} + diff --git a/src/main/java/com/uid2/operator/model/MapRequest.java b/src/main/java/com/uid2/operator/model/IdentityMapRequestItem.java similarity index 57% rename from src/main/java/com/uid2/operator/model/MapRequest.java rename to src/main/java/com/uid2/operator/model/IdentityMapRequestItem.java index 925296e44..079af8e76 100644 --- a/src/main/java/com/uid2/operator/model/MapRequest.java +++ b/src/main/java/com/uid2/operator/model/IdentityMapRequestItem.java @@ -1,20 +1,19 @@ package com.uid2.operator.model; -import com.uid2.operator.model.userIdentity.HashedDiiIdentity; +import com.uid2.operator.model.identities.HashedDii; import java.time.Instant; -public final class MapRequest { - public final HashedDiiIdentity hashedDiiIdentity; +public final class IdentityMapRequestItem { + public final HashedDii hashedDii; public final OptoutCheckPolicy optoutCheckPolicy; public final Instant asOf; - public MapRequest( - HashedDiiIdentity hashedDiiIdentity, + public IdentityMapRequestItem( + HashedDii hashedDii, OptoutCheckPolicy optoutCheckPolicy, - Instant asOf) - { - this.hashedDiiIdentity = hashedDiiIdentity; + Instant asOf) { + this.hashedDii = hashedDii; this.optoutCheckPolicy = optoutCheckPolicy; this.asOf = asOf; } diff --git a/src/main/java/com/uid2/operator/model/RawUidResponse.java b/src/main/java/com/uid2/operator/model/IdentityMapResponseItem.java similarity index 53% rename from src/main/java/com/uid2/operator/model/RawUidResponse.java rename to src/main/java/com/uid2/operator/model/IdentityMapResponseItem.java index 249bef4c5..0bdc70389 100644 --- a/src/main/java/com/uid2/operator/model/RawUidResponse.java +++ b/src/main/java/com/uid2/operator/model/IdentityMapResponseItem.java @@ -1,15 +1,19 @@ package com.uid2.operator.model; // Contains the computed raw UID and its bucket ID from identity/map request -public class RawUidResponse { - public static RawUidResponse OptoutIdentity = new RawUidResponse(new byte[33], ""); +public class IdentityMapResponseItem { + public static final IdentityMapResponseItem OptoutIdentity = new IdentityMapResponseItem(new byte[33], "", null, null); // The raw UID is also known as Advertising Id (historically) public final byte[] rawUid; public final String bucketId; + public final byte[] previousRawUid; + public final Long refreshFrom; - public RawUidResponse(byte[] rawUid, String bucketId) { + public IdentityMapResponseItem(byte[] rawUid, String bucketId, byte[] previousRawUid, Long refreshFrom) { this.rawUid = rawUid; this.bucketId = bucketId; + this.previousRawUid = previousRawUid; + this.refreshFrom = refreshFrom; } // historically Optout is known as Logout diff --git a/src/main/java/com/uid2/operator/model/IdentityMapResponseType.java b/src/main/java/com/uid2/operator/model/IdentityMapResponseType.java new file mode 100644 index 000000000..5c04a23a0 --- /dev/null +++ b/src/main/java/com/uid2/operator/model/IdentityMapResponseType.java @@ -0,0 +1,17 @@ +package com.uid2.operator.model; + +public enum IdentityMapResponseType { + OPTOUT("optout"), + INVALID_IDENTIFIER("invalid identifier"); + + private final String value; + + IdentityMapResponseType(String value) { + this.value = value; + } + + public String getValue() { + return value; + } +} + diff --git a/src/main/java/com/uid2/operator/model/IdentityMapV3Request.java b/src/main/java/com/uid2/operator/model/IdentityMapV3Request.java new file mode 100644 index 000000000..7e72ab9a0 --- /dev/null +++ b/src/main/java/com/uid2/operator/model/IdentityMapV3Request.java @@ -0,0 +1,18 @@ +package com.uid2.operator.model; + +import com.fasterxml.jackson.annotation.*; + +public record IdentityMapV3Request( + @JsonSetter(contentNulls = Nulls.FAIL) + @JsonProperty("email") String[] email, + + @JsonSetter(contentNulls = Nulls.FAIL) + @JsonProperty("email_hash") String[] email_hash, + + @JsonSetter(contentNulls = Nulls.FAIL) + @JsonProperty("phone") String[] phone, + + @JsonSetter(contentNulls = Nulls.FAIL) + @JsonProperty("phone_hash") String[] phone_hash +) { +} diff --git a/src/main/java/com/uid2/operator/model/IdentityRequest.java b/src/main/java/com/uid2/operator/model/IdentityRequest.java deleted file mode 100644 index e9a0c96cb..000000000 --- a/src/main/java/com/uid2/operator/model/IdentityRequest.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.uid2.operator.model; - -import com.uid2.operator.model.userIdentity.HashedDiiIdentity; - -public final class IdentityRequest { - public final SourcePublisher sourcePublisher; - public final HashedDiiIdentity hashedDiiIdentity; - public final OptoutCheckPolicy optoutCheckPolicy; - - public IdentityRequest( - SourcePublisher sourcePublisher, - HashedDiiIdentity hashedDiiIdentity, - OptoutCheckPolicy tokenGeneratePolicy) - { - this.sourcePublisher = sourcePublisher; - this.hashedDiiIdentity = hashedDiiIdentity; - this.optoutCheckPolicy = tokenGeneratePolicy; - } - - public boolean shouldCheckOptOut() { - return optoutCheckPolicy.equals(OptoutCheckPolicy.RespectOptOut); - } -} diff --git a/src/main/java/com/uid2/operator/model/RefreshResponse.java b/src/main/java/com/uid2/operator/model/RefreshResponse.java deleted file mode 100644 index 2a520fcc4..000000000 --- a/src/main/java/com/uid2/operator/model/RefreshResponse.java +++ /dev/null @@ -1,79 +0,0 @@ -package com.uid2.operator.model; - -import java.time.Duration; - -public class RefreshResponse { - - public static RefreshResponse Invalid = new RefreshResponse(Status.Invalid, IdentityResponse.OptOutIdentityResponse); - public static RefreshResponse Optout = new RefreshResponse(Status.Optout, IdentityResponse.OptOutIdentityResponse); - public static RefreshResponse Expired = new RefreshResponse(Status.Expired, IdentityResponse.OptOutIdentityResponse); - public static RefreshResponse Deprecated = new RefreshResponse(Status.Deprecated, IdentityResponse.OptOutIdentityResponse); - public static RefreshResponse NoActiveKey = new RefreshResponse(Status.NoActiveKey, IdentityResponse.OptOutIdentityResponse); - private final Status status; - private final IdentityResponse identityResponse; - private final Duration durationSinceLastRefresh; - private final boolean isCstg; - - private RefreshResponse(Status status, IdentityResponse identityResponse, Duration durationSinceLastRefresh, boolean isCstg) { - this.status = status; - this.identityResponse = identityResponse; - this.durationSinceLastRefresh = durationSinceLastRefresh; - this.isCstg = isCstg; - } - - private RefreshResponse(Status status, IdentityResponse identityResponse) { - this(status, identityResponse, null, false); - } - - public static RefreshResponse createRefreshedResponse(IdentityResponse identityResponse, Duration durationSinceLastRefresh, boolean isCstg) { - return new RefreshResponse(Status.Refreshed, identityResponse, durationSinceLastRefresh, isCstg); - } - - public Status getStatus() { - return status; - } - - public IdentityResponse getIdentityResponse() { - return identityResponse; - } - - public Duration getDurationSinceLastRefresh() { - return durationSinceLastRefresh; - } - - public boolean isCstg() { return isCstg;} - - public boolean isRefreshed() { - return Status.Refreshed.equals(this.status); - } - - public boolean isOptOut() { - return Status.Optout.equals(this.status); - } - - public boolean isInvalidToken() { - return Status.Invalid.equals(this.status); - } - - public boolean isDeprecated() { - return Status.Deprecated.equals(this.status); - } - - public boolean isExpired() { - return Status.Expired.equals(this.status); - } - - public boolean noActiveKey() { - return Status.NoActiveKey.equals(this.status); - } - - public enum Status { - Refreshed, - Invalid, - Optout, - Expired, - Deprecated, - NoActiveKey - } - -} diff --git a/src/main/java/com/uid2/operator/model/RefreshTokenInput.java b/src/main/java/com/uid2/operator/model/RefreshTokenInput.java deleted file mode 100644 index 15bef6a0c..000000000 --- a/src/main/java/com/uid2/operator/model/RefreshTokenInput.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.uid2.operator.model; - -import java.time.Instant; - -import com.uid2.operator.model.userIdentity.FirstLevelHashIdentity; -import com.uid2.shared.model.TokenVersion; - -public class RefreshTokenInput extends VersionedToken { - public final OperatorIdentity operatorIdentity; - public final SourcePublisher sourcePublisher; - public final FirstLevelHashIdentity firstLevelHashIdentity; - - public RefreshTokenInput(TokenVersion version, Instant createdAt, Instant expiresAt, OperatorIdentity operatorIdentity, - SourcePublisher sourcePublisher, FirstLevelHashIdentity firstLevelHashIdentity) { - super(version, createdAt, expiresAt); - this.operatorIdentity = operatorIdentity; - this.sourcePublisher = sourcePublisher; - this.firstLevelHashIdentity = firstLevelHashIdentity; - } -} diff --git a/src/main/java/com/uid2/operator/model/SourcePublisher.java b/src/main/java/com/uid2/operator/model/SourcePublisher.java index 4f13fd53e..bd19740a1 100644 --- a/src/main/java/com/uid2/operator/model/SourcePublisher.java +++ b/src/main/java/com/uid2/operator/model/SourcePublisher.java @@ -3,6 +3,10 @@ // The original publisher that requests to generate a UID token public class SourcePublisher { public final int siteId; + + // these 2 values are added into adverting/UID token and refresh token payload but + // are not really used for any real purposes currently so sometimes are set to 0 + // see the constructor below public final int clientKeyId; public final long publisherId; @@ -11,4 +15,10 @@ public SourcePublisher(int siteId, int clientKeyId, long publisherId) { this.clientKeyId = clientKeyId; this.publisherId = publisherId; } + + public SourcePublisher(int siteId) { + this.siteId = siteId; + this.clientKeyId = 0; + this.publisherId = 0; + } } diff --git a/src/main/java/com/uid2/operator/model/TokenGenerateRequest.java b/src/main/java/com/uid2/operator/model/TokenGenerateRequest.java new file mode 100644 index 000000000..39f3b56fc --- /dev/null +++ b/src/main/java/com/uid2/operator/model/TokenGenerateRequest.java @@ -0,0 +1,40 @@ +package com.uid2.operator.model; + +import com.uid2.operator.model.identities.HashedDii; +import com.uid2.operator.util.PrivacyBits; + +import java.time.Instant; + +public final class TokenGenerateRequest { + public final SourcePublisher sourcePublisher; + public final HashedDii hashedDii; + public final OptoutCheckPolicy optoutCheckPolicy; + + public final PrivacyBits privacyBits; + public final Instant establishedAt; + + public TokenGenerateRequest( + SourcePublisher sourcePublisher, + HashedDii hashedDii, + OptoutCheckPolicy tokenGeneratePolicy, + PrivacyBits privacyBits, + Instant establishedAt) { + this.sourcePublisher = sourcePublisher; + this.hashedDii = hashedDii; + this.optoutCheckPolicy = tokenGeneratePolicy; + this.privacyBits = privacyBits; + this.establishedAt = establishedAt; + } + + public TokenGenerateRequest( + SourcePublisher sourcePublisher, + HashedDii hashedDii, + OptoutCheckPolicy tokenGeneratePolicy) { + this(sourcePublisher, hashedDii, tokenGeneratePolicy, PrivacyBits.DEFAULT, Instant.now()); + + } + + public boolean shouldCheckOptOut() { + return optoutCheckPolicy.equals(OptoutCheckPolicy.RespectOptOut); + } +} diff --git a/src/main/java/com/uid2/operator/model/IdentityResponse.java b/src/main/java/com/uid2/operator/model/TokenGenerateResponse.java similarity index 57% rename from src/main/java/com/uid2/operator/model/IdentityResponse.java rename to src/main/java/com/uid2/operator/model/TokenGenerateResponse.java index fc9182650..21338f6f4 100644 --- a/src/main/java/com/uid2/operator/model/IdentityResponse.java +++ b/src/main/java/com/uid2/operator/model/TokenGenerateResponse.java @@ -1,22 +1,27 @@ package com.uid2.operator.model; import com.uid2.shared.model.TokenVersion; +import io.vertx.core.json.JsonObject; import java.time.Instant; // this defines all the fields for the response of the /token/generate and /client/generate endpoints before they are // jsonified -public class IdentityResponse { - public static IdentityResponse OptOutIdentityResponse = new IdentityResponse("", null, "", Instant.EPOCH, Instant.EPOCH, Instant.EPOCH); +// todo: can be converted to record later +public class TokenGenerateResponse { + public static final TokenGenerateResponse OptOutResponse = new TokenGenerateResponse("", null, "", Instant.EPOCH, Instant.EPOCH, Instant.EPOCH); + + //aka UID token private final String advertisingToken; private final TokenVersion advertisingTokenVersion; private final String refreshToken; + // when the advertising token/uid token expires private final Instant identityExpires; private final Instant refreshExpires; private final Instant refreshFrom; - public IdentityResponse(String advertisingToken, TokenVersion advertisingTokenVersion, String refreshToken, - Instant identityExpires, Instant refreshExpires, Instant refreshFrom) { + public TokenGenerateResponse(String advertisingToken, TokenVersion advertisingTokenVersion, String refreshToken, + Instant identityExpires, Instant refreshExpires, Instant refreshFrom) { this.advertisingToken = advertisingToken; this.advertisingTokenVersion = advertisingTokenVersion; this.refreshToken = refreshToken; @@ -52,4 +57,15 @@ public Instant getRefreshFrom() { public boolean isOptedOut() { return advertisingToken == null || advertisingToken.isEmpty(); } + + public JsonObject toTokenGenerateResponseJson() { + final JsonObject json = new JsonObject(); + json.put("advertising_token", this.getAdvertisingToken()); + json.put("refresh_token", this.getRefreshToken()); + json.put("identity_expires", this.getIdentityExpires().toEpochMilli()); + json.put("refresh_expires", this.getRefreshExpires().toEpochMilli()); + json.put("refresh_from", this.getRefreshFrom().toEpochMilli()); + return json; + } } + diff --git a/src/main/java/com/uid2/operator/model/TokenRefreshRequest.java b/src/main/java/com/uid2/operator/model/TokenRefreshRequest.java new file mode 100644 index 000000000..e2e51971a --- /dev/null +++ b/src/main/java/com/uid2/operator/model/TokenRefreshRequest.java @@ -0,0 +1,26 @@ +package com.uid2.operator.model; + +import java.time.Instant; + +import com.uid2.operator.model.identities.FirstLevelHash; +import com.uid2.operator.util.PrivacyBits; +import com.uid2.shared.model.TokenVersion; + +// class containing enough data to create a new refresh token +public class TokenRefreshRequest extends VersionedTokenRequest { + public final OperatorIdentity operatorIdentity; + public final SourcePublisher sourcePublisher; + public final FirstLevelHash firstLevelHash; + // by default, inherited from the previous refresh token's privacy bits + public final PrivacyBits privacyBits; + + + public TokenRefreshRequest(TokenVersion version, Instant createdAt, Instant expiresAt, OperatorIdentity operatorIdentity, + SourcePublisher sourcePublisher, FirstLevelHash firstLevelHash, PrivacyBits privacyBits) { + super(version, createdAt, expiresAt); + this.operatorIdentity = operatorIdentity; + this.sourcePublisher = sourcePublisher; + this.firstLevelHash = firstLevelHash; + this.privacyBits = privacyBits; + } +} diff --git a/src/main/java/com/uid2/operator/model/TokenRefreshResponse.java b/src/main/java/com/uid2/operator/model/TokenRefreshResponse.java new file mode 100644 index 000000000..40e5d73c9 --- /dev/null +++ b/src/main/java/com/uid2/operator/model/TokenRefreshResponse.java @@ -0,0 +1,80 @@ +package com.uid2.operator.model; + +import java.time.Duration; + +public class TokenRefreshResponse { + + public static final TokenRefreshResponse Invalid = new TokenRefreshResponse(Status.Invalid, + TokenGenerateResponse.OptOutResponse); + public static final TokenRefreshResponse Optout = new TokenRefreshResponse(Status.Optout, TokenGenerateResponse.OptOutResponse); + public static final TokenRefreshResponse Expired = new TokenRefreshResponse(Status.Expired, TokenGenerateResponse.OptOutResponse); + public static final TokenRefreshResponse Deprecated = new TokenRefreshResponse(Status.Deprecated, TokenGenerateResponse.OptOutResponse); + public static final TokenRefreshResponse NoActiveKey = new TokenRefreshResponse(Status.NoActiveKey, TokenGenerateResponse.OptOutResponse); + private final Status status; + private final TokenGenerateResponse tokenGenerateResponse; + private final Duration durationSinceLastRefresh; + private final boolean isCstg; + + private TokenRefreshResponse(Status status, TokenGenerateResponse tokenGenerateResponse, Duration durationSinceLastRefresh, boolean isCstg) { + this.status = status; + this.tokenGenerateResponse = tokenGenerateResponse; + this.durationSinceLastRefresh = durationSinceLastRefresh; + this.isCstg = isCstg; + } + + private TokenRefreshResponse(Status status, TokenGenerateResponse tokenGenerateResponse) { + this(status, tokenGenerateResponse, null, false); + } + + public static TokenRefreshResponse createRefreshedResponse(TokenGenerateResponse tokenGenerateResponse, Duration durationSinceLastRefresh, boolean isCstg) { + return new TokenRefreshResponse(Status.Refreshed, tokenGenerateResponse, durationSinceLastRefresh, isCstg); + } + + public Status getStatus() { + return status; + } + + public TokenGenerateResponse getIdentityResponse() { + return tokenGenerateResponse; + } + + public Duration getDurationSinceLastRefresh() { + return durationSinceLastRefresh; + } + + public boolean isCstg() { return isCstg;} + + public boolean isRefreshed() { + return Status.Refreshed.equals(this.status); + } + + public boolean isOptOut() { + return Status.Optout.equals(this.status); + } + + public boolean isInvalidToken() { + return Status.Invalid.equals(this.status); + } + + public boolean isDeprecated() { + return Status.Deprecated.equals(this.status); + } + + public boolean isExpired() { + return Status.Expired.equals(this.status); + } + + public boolean noActiveKey() { + return Status.NoActiveKey.equals(this.status); + } + + public enum Status { + Refreshed, + Invalid, + Optout, + Expired, + Deprecated, + NoActiveKey + } + +} diff --git a/src/main/java/com/uid2/operator/model/VersionedToken.java b/src/main/java/com/uid2/operator/model/VersionedTokenRequest.java similarity index 68% rename from src/main/java/com/uid2/operator/model/VersionedToken.java rename to src/main/java/com/uid2/operator/model/VersionedTokenRequest.java index 5be86b80e..5cc9c5335 100644 --- a/src/main/java/com/uid2/operator/model/VersionedToken.java +++ b/src/main/java/com/uid2/operator/model/VersionedTokenRequest.java @@ -1,16 +1,16 @@ package com.uid2.operator.model; import java.time.Instant; -import java.util.Objects; + import com.uid2.shared.model.TokenVersion; -public abstract class VersionedToken { +public abstract class VersionedTokenRequest { public final TokenVersion version; public final Instant createdAt; public final Instant expiresAt; - public VersionedToken(TokenVersion version, Instant createdAt, Instant expiresAt) { + public VersionedTokenRequest(TokenVersion version, Instant createdAt, Instant expiresAt) { this.version = version; this.createdAt = createdAt; this.expiresAt = expiresAt; diff --git a/src/main/java/com/uid2/operator/model/IdentityType.java b/src/main/java/com/uid2/operator/model/identities/DiiType.java similarity index 67% rename from src/main/java/com/uid2/operator/model/IdentityType.java rename to src/main/java/com/uid2/operator/model/identities/DiiType.java index b64817df5..062b55d35 100644 --- a/src/main/java/com/uid2/operator/model/IdentityType.java +++ b/src/main/java/com/uid2/operator/model/identities/DiiType.java @@ -1,15 +1,15 @@ -package com.uid2.operator.model; +package com.uid2.operator.model.identities; import com.uid2.operator.vertx.ClientInputValidationException; -public enum IdentityType { +public enum DiiType { Email(0), Phone(1); public final int value; - IdentityType(int value) { this.value = value; } + DiiType(int value) { this.value = value; } - public static IdentityType fromValue(int value) { + public static DiiType fromValue(int value) { switch (value) { case 0: return Email; case 1: return Phone; diff --git a/src/main/java/com/uid2/operator/model/identities/FirstLevelHash.java b/src/main/java/com/uid2/operator/model/identities/FirstLevelHash.java new file mode 100644 index 000000000..49b2728f4 --- /dev/null +++ b/src/main/java/com/uid2/operator/model/identities/FirstLevelHash.java @@ -0,0 +1,19 @@ +package com.uid2.operator.model.identities; + +import java.time.Instant; +import java.util.Arrays; + +/** + * Contains a first level salted hash computed from Hashed DII (email/phone number) + * @param establishedAt for brand new token generation, it should be the time it is generated if the first level hash is from token/refresh call, it will be when the raw UID was originally created in the earliest token generation + */ +public record FirstLevelHash(IdentityScope identityScope, DiiType diiType, byte[] firstLevelHash, + Instant establishedAt) { + + // explicitly not checking establishedAt - this is only for making sure the first level hash matches a new input + public boolean matches(FirstLevelHash that) { + return this.identityScope.equals(that.identityScope) && + this.diiType.equals(that.diiType) && + Arrays.equals(this.firstLevelHash, that.firstLevelHash); + } +} diff --git a/src/main/java/com/uid2/operator/model/identities/HashedDii.java b/src/main/java/com/uid2/operator/model/identities/HashedDii.java new file mode 100644 index 000000000..64c7bbf0f --- /dev/null +++ b/src/main/java/com/uid2/operator/model/identities/HashedDii.java @@ -0,0 +1,7 @@ +package com.uid2.operator.model.identities; + +// Contains a hash Directly Identifying Information (DII) (email or phone) see https://unifiedid.com/docs/ref-info/glossary-uid#gl-dii +// This hash can either be computed from a raw email/phone number DII input or provided by the UID Participant directly +// +public record HashedDii(IdentityScope identityScope, DiiType diiType, byte[] hashedDii) { +} diff --git a/src/main/java/com/uid2/operator/IdentityConst.java b/src/main/java/com/uid2/operator/model/identities/IdentityConst.java similarity index 82% rename from src/main/java/com/uid2/operator/IdentityConst.java rename to src/main/java/com/uid2/operator/model/identities/IdentityConst.java index 9362ade6e..63fa62f96 100644 --- a/src/main/java/com/uid2/operator/IdentityConst.java +++ b/src/main/java/com/uid2/operator/model/identities/IdentityConst.java @@ -1,4 +1,4 @@ -package com.uid2.operator; +package com.uid2.operator.model.identities; import com.uid2.operator.service.EncodingUtils; @@ -7,13 +7,13 @@ public class IdentityConst { public static final String OptOutTokenIdentityForEmail = "optout@unifiedid.com"; public static final String OptOutTokenIdentityForPhone = "+00000000001"; - // DIIs for for testing with token/validate endpoint, see https://unifiedid.com/docs/endpoints/post-token-validate + // DIIs for testing with token/validate endpoint, see https://unifiedid.com/docs/endpoints/post-token-validate public static final String ValidateIdentityForEmail = "validate@example.com"; public static final String ValidateIdentityForPhone = "+12345678901"; public static final byte[] ValidateIdentityForEmailHash = EncodingUtils.getSha256Bytes(IdentityConst.ValidateIdentityForEmail); public static final byte[] ValidateIdentityForPhoneHash = EncodingUtils.getSha256Bytes(IdentityConst.ValidateIdentityForPhone); - // DIIs to use when you want to generate a optout response in token generation or identity map + // DIIs to use when you want to generate an optout response in token generation or identity map public static final String OptOutIdentityForEmail = "optout@example.com"; public static final String OptOutIdentityForPhone = "+00000000000"; diff --git a/src/main/java/com/uid2/operator/model/IdentityScope.java b/src/main/java/com/uid2/operator/model/identities/IdentityScope.java similarity index 94% rename from src/main/java/com/uid2/operator/model/IdentityScope.java rename to src/main/java/com/uid2/operator/model/identities/IdentityScope.java index 0bff1edc1..3dc19a764 100644 --- a/src/main/java/com/uid2/operator/model/IdentityScope.java +++ b/src/main/java/com/uid2/operator/model/identities/IdentityScope.java @@ -1,4 +1,4 @@ -package com.uid2.operator.model; +package com.uid2.operator.model.identities; import com.uid2.operator.vertx.ClientInputValidationException; diff --git a/src/main/java/com/uid2/operator/model/identities/RawUid.java b/src/main/java/com/uid2/operator/model/identities/RawUid.java new file mode 100644 index 000000000..4ae619d00 --- /dev/null +++ b/src/main/java/com/uid2/operator/model/identities/RawUid.java @@ -0,0 +1,13 @@ +package com.uid2.operator.model.identities; + +import java.util.Arrays; + +// A raw UID is stored inside +public record RawUid(IdentityScope identityScope, DiiType diiType, byte[] rawUid) { + + public boolean matches(RawUid that) { + return this.identityScope.equals(that.identityScope) && + this.diiType.equals(that.diiType) && + Arrays.equals(this.rawUid, that.rawUid); + } +} diff --git a/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java deleted file mode 100644 index 64b8bcedd..000000000 --- a/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.uid2.operator.model.userIdentity; - -import com.uid2.operator.model.IdentityScope; -import com.uid2.operator.model.IdentityType; - -import java.time.Instant; -import java.util.Arrays; - -// Contains a first level salted hash computed from Hashed DII (email/phone number) -public class FirstLevelHashIdentity extends UserIdentity { - public final byte[] firstLevelHash; - - public FirstLevelHashIdentity(IdentityScope identityScope, IdentityType identityType, byte[] firstLevelHash, int privacyBits, - Instant establishedAt, Instant refreshedAt) { - super(identityScope, identityType, privacyBits, establishedAt, refreshedAt); - this.firstLevelHash = firstLevelHash; - } - - public boolean matches(FirstLevelHashIdentity that) { - return this.identityScope.equals(that.identityScope) && - this.identityType.equals(that.identityType) && - Arrays.equals(this.firstLevelHash, that.firstLevelHash); - } -} diff --git a/src/main/java/com/uid2/operator/model/userIdentity/HashedDiiIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/HashedDiiIdentity.java deleted file mode 100644 index dad862f21..000000000 --- a/src/main/java/com/uid2/operator/model/userIdentity/HashedDiiIdentity.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.uid2.operator.model.userIdentity; - -import com.uid2.operator.model.IdentityScope; -import com.uid2.operator.model.IdentityType; - -import java.time.Instant; - -// Contains a hash DII, -// This hash can either be computed from a raw email/phone number DII input or provided by the UID Participant directly -public class HashedDiiIdentity extends UserIdentity { - public final byte[] hashedDii; - - public HashedDiiIdentity(IdentityScope identityScope, IdentityType identityType, byte[] hashedDii, int privacyBits, - Instant establishedAt, Instant refreshedAt) { - super(identityScope, identityType, privacyBits, establishedAt, refreshedAt); - this.hashedDii = hashedDii; - } -} diff --git a/src/main/java/com/uid2/operator/model/userIdentity/RawUidIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/RawUidIdentity.java deleted file mode 100644 index 4e15c6ff0..000000000 --- a/src/main/java/com/uid2/operator/model/userIdentity/RawUidIdentity.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.uid2.operator.model.userIdentity; - -import com.uid2.operator.model.IdentityScope; -import com.uid2.operator.model.IdentityType; - -import java.time.Instant; -import java.util.Arrays; - -// A raw UID is stored inside -public class RawUidIdentity extends UserIdentity { - public final byte[] rawUid; - - public RawUidIdentity(IdentityScope identityScope, IdentityType identityType, byte[] rawUid, int privacyBits, - Instant establishedAt, Instant refreshedAt) { - super(identityScope, identityType, privacyBits, establishedAt, refreshedAt); - this.rawUid = rawUid; - } - - public boolean matches(RawUidIdentity that) { - return this.identityScope.equals(that.identityScope) && - this.identityType.equals(that.identityType) && - Arrays.equals(this.rawUid, that.rawUid); - } -} diff --git a/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java deleted file mode 100644 index 1391b7d75..000000000 --- a/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.uid2.operator.model.userIdentity; - -import com.uid2.operator.model.IdentityScope; -import com.uid2.operator.model.IdentityType; - -import java.time.Instant; - -//base class for all other HshedDii/FirstLevelHash/RawUIDIdentity class and define the basic common fields -public abstract class UserIdentity { - - public final IdentityScope identityScope; - public final IdentityType identityType; - public final int privacyBits; - public final Instant establishedAt; - public final Instant refreshedAt; - - public UserIdentity(IdentityScope identityScope, IdentityType identityType, int privacyBits, Instant establishedAt, Instant refreshedAt) { - this.identityScope = identityScope; - this.identityType = identityType; - this.privacyBits = privacyBits; - this.establishedAt = establishedAt; - this.refreshedAt = refreshedAt; - } -} diff --git a/src/main/java/com/uid2/operator/monitoring/OperatorMetrics.java b/src/main/java/com/uid2/operator/monitoring/OperatorMetrics.java index c04d610df..990fff02f 100644 --- a/src/main/java/com/uid2/operator/monitoring/OperatorMetrics.java +++ b/src/main/java/com/uid2/operator/monitoring/OperatorMetrics.java @@ -3,7 +3,7 @@ import com.uid2.operator.model.KeyManager; import com.uid2.shared.model.KeysetKey; import com.uid2.shared.model.SaltEntry; -import com.uid2.shared.store.ISaltProvider; +import com.uid2.shared.store.salt.ISaltProvider; import io.micrometer.core.instrument.Gauge; import java.time.Instant; @@ -26,13 +26,13 @@ public void setup() { Gauge .builder("uid2_second_level_salt_last_updated_max", () -> Arrays.stream(saltProvider.getSnapshot(Instant.now()).getAllRotatingSalts()) - .map(SaltEntry::getLastUpdated).max(Long::compare).orElse(null)) + .map(SaltEntry::lastUpdated).max(Long::compare).orElse(null)) .description("max last updated timestamp within currently effective second level salts") .register(globalRegistry); Gauge .builder("uid2_second_level_salt_last_updated_min", () -> Arrays.stream(saltProvider.getSnapshot(Instant.now()).getAllRotatingSalts()) - .map(SaltEntry::getLastUpdated).min(Long::compare).orElse(null)) + .map(SaltEntry::lastUpdated).min(Long::compare).orElse(null)) .description("max last updated timestamp within currently effective second level salts") .register(globalRegistry); diff --git a/src/main/java/com/uid2/operator/monitoring/StatsCollectorHandler.java b/src/main/java/com/uid2/operator/monitoring/StatsCollectorHandler.java index 04a36d9c1..ebeb304d5 100644 --- a/src/main/java/com/uid2/operator/monitoring/StatsCollectorHandler.java +++ b/src/main/java/com/uid2/operator/monitoring/StatsCollectorHandler.java @@ -20,7 +20,9 @@ public StatsCollectorHandler(IStatsCollectorQueue _statCollectorQueue, Vertx ver @Override public void handle(RoutingContext routingContext) { - assert routingContext != null; + if (routingContext == null) { + throw new NullPointerException(); + } //setAuthClient() has not yet been called, so getAuthClient() would return null. This is resolved by using addBodyEndHandler() routingContext.addBodyEndHandler(v -> addStatsMessageToQueue(routingContext)); diff --git a/src/main/java/com/uid2/operator/monitoring/StatsCollectorVerticle.java b/src/main/java/com/uid2/operator/monitoring/StatsCollectorVerticle.java index 8e49ec1f8..564821975 100644 --- a/src/main/java/com/uid2/operator/monitoring/StatsCollectorVerticle.java +++ b/src/main/java/com/uid2/operator/monitoring/StatsCollectorVerticle.java @@ -54,16 +54,16 @@ public StatsCollectorVerticle(long jsonIntervalMS, int maxInvalidPaths, int maxV this.maxInvalidPaths = maxInvalidPaths; logCycleSkipperCounter = Counter - .builder("uid2.api_usage_log_cycle_skipped") + .builder("uid2_api_usage_log_cycle_skipped_total") .description("counter for how many log cycles are skipped because the thread is still running") .register(Metrics.globalRegistry); domainMissedCounter = Counter - .builder("uid2.api_usage_domain_missed") + .builder("uid2_api_usage_domain_missed_total") .description("counter for how many domains are missed because the dictionary is full") .register(Metrics.globalRegistry); queueFullCounter = Counter - .builder("uid2.api_usage_queue_full") + .builder("uid2_api_usage_queue_full_total") .description("counter for how many usage messages are dropped because the queue is full") .register(Metrics.globalRegistry); @@ -88,8 +88,6 @@ public void handleMessage(Message message) { return; } - assert messageItem != null; - String path = messageItem.getPath(); String apiVersion = "v0"; String endpoint = path.substring(1); diff --git a/src/main/java/com/uid2/operator/monitoring/TokenResponseStatsCollector.java b/src/main/java/com/uid2/operator/monitoring/TokenResponseStatsCollector.java index fc28bba70..b5a76b8f1 100644 --- a/src/main/java/com/uid2/operator/monitoring/TokenResponseStatsCollector.java +++ b/src/main/java/com/uid2/operator/monitoring/TokenResponseStatsCollector.java @@ -1,6 +1,6 @@ package com.uid2.operator.monitoring; -import com.uid2.operator.model.RefreshResponse; +import com.uid2.operator.model.TokenRefreshResponse; import com.uid2.operator.vertx.UIDOperatorVerticle; import com.uid2.shared.model.TokenVersion; import com.uid2.shared.store.ISiteStore; @@ -9,11 +9,7 @@ public class TokenResponseStatsCollector { public enum Endpoint { - GenerateV0, - GenerateV1, GenerateV2, - RefreshV0, - RefreshV1, RefreshV2, //it's the first version but the endpoint is v2/token/client-generate so we will call it v2 ClientSideTokenGenerateV2, @@ -69,7 +65,7 @@ private static void recordInternal(ISiteStore siteStore, Integer siteId, Endpoin builder.register(Metrics.globalRegistry).increment(); } - public static void recordRefresh(ISiteStore siteStore, Integer siteId, Endpoint endpoint, RefreshResponse refreshResponse, PlatformType platformType) { + public static void recordRefresh(ISiteStore siteStore, Integer siteId, Endpoint endpoint, TokenRefreshResponse refreshResponse, PlatformType platformType) { if (!refreshResponse.isRefreshed()) { if (refreshResponse.isOptOut() || refreshResponse.isDeprecated()) { recordInternal(siteStore, siteId, endpoint, ResponseStatus.OptOut, refreshResponse.getIdentityResponse().getAdvertisingTokenVersion(), refreshResponse.isCstg(), platformType); diff --git a/src/main/java/com/uid2/operator/reader/ApiStoreReader.java b/src/main/java/com/uid2/operator/reader/ApiStoreReader.java new file mode 100644 index 000000000..7b39c9a83 --- /dev/null +++ b/src/main/java/com/uid2/operator/reader/ApiStoreReader.java @@ -0,0 +1,58 @@ +package com.uid2.operator.reader; + +import com.uid2.shared.cloud.DownloadCloudStorage; +import com.uid2.shared.store.ScopedStoreReader; +import com.uid2.shared.store.parser.Parser; +import com.uid2.shared.store.parser.ParsingResult; +import com.uid2.shared.store.scope.StoreScope; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +public class ApiStoreReader extends ScopedStoreReader { + private static final Logger LOGGER = LoggerFactory.getLogger(ApiStoreReader.class); + + public ApiStoreReader(DownloadCloudStorage fileStreamProvider, StoreScope scope, Parser parser, String dataTypeName) { + super(fileStreamProvider, scope, parser, dataTypeName); + } + + + public long loadContent(JsonObject contents) throws Exception { + return loadContent(contents, dataTypeName); + } + + @Override + public long loadContent(JsonObject contents, String dataType) throws IOException { + if (contents == null) { + throw new IllegalArgumentException(String.format("No contents provided for loading data type %s, cannot load content", dataType)); + } + + try { + JsonArray dataArray = contents.getJsonArray(dataType); + if (dataArray == null) { + throw new IllegalArgumentException(String.format("No array of type: %s, found in the contents", dataType)); + } + + String jsonString = dataArray.toString(); + InputStream inputStream = new ByteArrayInputStream(jsonString.getBytes(StandardCharsets.UTF_8)); + + ParsingResult parsed = parser.deserialize(inputStream); + latestSnapshot.set(parsed.getData()); + + final int count = parsed.getCount(); + latestEntryCount.set(count); + LOGGER.info(String.format("Loaded %d %s", count, dataType)); + return count; + } catch (Exception e) { + LOGGER.error(String.format("Unable to load %s", dataType)); + throw e; + } + } +} + diff --git a/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java b/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java new file mode 100644 index 000000000..37226f046 --- /dev/null +++ b/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java @@ -0,0 +1,27 @@ +package com.uid2.operator.reader; + +import com.uid2.shared.cloud.DownloadCloudStorage; +import com.uid2.shared.model.CloudEncryptionKey; +import com.uid2.shared.store.parser.CloudEncryptionKeyParser; +import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; +import com.uid2.shared.store.scope.StoreScope; +import io.vertx.core.json.JsonObject; + +import java.time.Instant; +import java.util.*; + +public class RotatingCloudEncryptionKeyApiProvider extends RotatingCloudEncryptionKeyProvider { + public RotatingCloudEncryptionKeyApiProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope) { + super(new ApiStoreReader<>(fileStreamProvider, scope, new CloudEncryptionKeyParser(), "cloud_encryption_keys")); + } + + public RotatingCloudEncryptionKeyApiProvider(ApiStoreReader> reader) { + super(reader); + } + + @Override + public long getVersion(JsonObject metadata) { + // Since we are pulling from an api not a data file, we use the epoch time we got the keys as the version + return Instant.now().getEpochSecond(); + } +} diff --git a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java index 1a74a794a..cb5f03ccc 100644 --- a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java +++ b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java @@ -1,8 +1,11 @@ package com.uid2.operator.service; import com.uid2.operator.model.*; -import com.uid2.operator.model.userIdentity.FirstLevelHashIdentity; -import com.uid2.operator.model.userIdentity.RawUidIdentity; +import com.uid2.operator.model.identities.DiiType; +import com.uid2.operator.model.identities.FirstLevelHash; +import com.uid2.operator.model.identities.IdentityScope; +import com.uid2.operator.model.identities.RawUid; +import com.uid2.operator.util.PrivacyBits; import com.uid2.operator.vertx.ClientInputValidationException; import com.uid2.shared.Const.Data; import com.uid2.shared.encryption.AesCbc; @@ -24,7 +27,7 @@ public EncryptedTokenEncoder(KeyManager keyManager) { this.keyManager = keyManager; } - public byte[] encodeIntoAdvertisingToken(AdvertisingTokenInput t, Instant asOf) { + public byte[] encodeIntoAdvertisingToken(AdvertisingTokenRequest t, Instant asOf) { final KeysetKey masterKey = this.keyManager.getMasterKey(asOf); final KeysetKey siteEncryptionKey = this.keyManager.getActiveKeyBySiteIdWithFallback(t.sourcePublisher.siteId, Data.AdvertisingTokenSiteId, asOf); @@ -33,7 +36,7 @@ public byte[] encodeIntoAdvertisingToken(AdvertisingTokenInput t, Instant asOf) : encodeIntoAdvertisingTokenV3(t, masterKey, siteEncryptionKey); //TokenVersion.V4 also calls encodeV3() since the byte array is identical between V3 and V4 } - private byte[] encodeIntoAdvertisingTokenV2(AdvertisingTokenInput t, KeysetKey masterKey, KeysetKey siteKey) { + private byte[] encodeIntoAdvertisingTokenV2(AdvertisingTokenRequest t, KeysetKey masterKey, KeysetKey siteKey) { final Buffer b = Buffer.buffer(); b.appendByte((byte) t.version.rawVersion); @@ -41,7 +44,7 @@ private byte[] encodeIntoAdvertisingTokenV2(AdvertisingTokenInput t, KeysetKey m Buffer b2 = Buffer.buffer(); b2.appendLong(t.expiresAt.toEpochMilli()); - encodeSiteIdentityV2(b2, t.sourcePublisher, t.rawUidIdentity, siteKey); + encodeSiteIdentityV2(b2, t.sourcePublisher, t.rawUid, siteKey, t.privacyBits, t.establishedAt); final byte[] encryptedId = AesCbc.encrypt(b2.getBytes(), masterKey).getPayload(); @@ -50,13 +53,15 @@ private byte[] encodeIntoAdvertisingTokenV2(AdvertisingTokenInput t, KeysetKey m return b.getBytes(); } - private byte[] encodeIntoAdvertisingTokenV3(AdvertisingTokenInput t, KeysetKey masterKey, KeysetKey siteKey) { + private byte[] encodeIntoAdvertisingTokenV3(AdvertisingTokenRequest t, KeysetKey masterKey, KeysetKey siteKey) { final Buffer sitePayload = Buffer.buffer(69); encodePublisherRequesterV3(sitePayload, t.sourcePublisher); - sitePayload.appendInt(t.rawUidIdentity.privacyBits); - sitePayload.appendLong(t.rawUidIdentity.establishedAt.toEpochMilli()); - sitePayload.appendLong(t.rawUidIdentity.refreshedAt.toEpochMilli()); - sitePayload.appendBytes(t.rawUidIdentity.rawUid); // 32 or 33 bytes + sitePayload.appendInt(t.privacyBits.getAsInt()); + sitePayload.appendLong(t.establishedAt.toEpochMilli()); + // this is the refreshedAt field in the spec - but effectively it is the time this advertising token is generated + // this is a redundant field as it is stored in master payload again, can consider dropping this field in future token version + sitePayload.appendLong(t.createdAt.toEpochMilli()); + sitePayload.appendBytes(t.rawUid.rawUid()); // 32 or 33 bytes final Buffer masterPayload = Buffer.buffer(130); masterPayload.appendLong(t.expiresAt.toEpochMilli()); @@ -66,7 +71,7 @@ private byte[] encodeIntoAdvertisingTokenV3(AdvertisingTokenInput t, KeysetKey m masterPayload.appendBytes(AesGcm.encrypt(sitePayload.getBytes(), siteKey).getPayload()); final Buffer b = Buffer.buffer(164); - b.appendByte(encodeIdentityTypeV3(t.rawUidIdentity.identityScope, t.rawUidIdentity.identityType)); + b.appendByte(encodeIdentityTypeV3(t.rawUid.identityScope(), t.rawUid.diiType())); b.appendByte((byte) t.version.rawVersion); b.appendInt(masterKey.getId()); b.appendBytes(AesGcm.encrypt(masterPayload.getBytes(), masterKey).getPayload()); @@ -75,7 +80,7 @@ private byte[] encodeIntoAdvertisingTokenV3(AdvertisingTokenInput t, KeysetKey m } @Override - public RefreshTokenInput decodeRefreshToken(String s) { + public TokenRefreshRequest decodeRefreshToken(String s) { if (s != null && !s.isEmpty()) { final byte[] bytes; try { @@ -94,7 +99,7 @@ public RefreshTokenInput decodeRefreshToken(String s) { throw new ClientInputValidationException("Invalid refresh token version"); } - private RefreshTokenInput decodeRefreshTokenV2(Buffer b) { + private TokenRefreshRequest decodeRefreshTokenV2(Buffer b) { final Instant createdAt = Instant.ofEpochMilli(b.getLong(1)); //final Instant expiresAt = Instant.ofEpochMilli(b.getLong(9)); final Instant validTill = Instant.ofEpochMilli(b.getLong(17)); @@ -119,18 +124,19 @@ private RefreshTokenInput decodeRefreshTokenV2(Buffer b) { throw new ClientInputValidationException("Failed to decode refreshTokenV2: Identity segment is not valid base64.", e); } - final int privacyBits = b2.getInt(8 + length); + final PrivacyBits privacyBits = PrivacyBits.fromInt(b2.getInt(8 + length)); final long establishedMillis = b2.getLong(8 + length + 4); - return new RefreshTokenInput( + return new TokenRefreshRequest( TokenVersion.V2, createdAt, validTill, new OperatorIdentity(0, OperatorType.Service, 0, 0), - new SourcePublisher(siteId, 0, 0), - new FirstLevelHashIdentity(IdentityScope.UID2, IdentityType.Email, identity, privacyBits, - Instant.ofEpochMilli(establishedMillis), null)); + new SourcePublisher(siteId), + new FirstLevelHash(IdentityScope.UID2, DiiType.Email, identity, + Instant.ofEpochMilli(establishedMillis)), + privacyBits); } - private RefreshTokenInput decodeRefreshTokenV3(Buffer b, byte[] bytes) { + private TokenRefreshRequest decodeRefreshTokenV3(Buffer b, byte[] bytes) { final int keyId = b.getInt(2); final KeysetKey key = this.keyManager.getKey(keyId); @@ -145,26 +151,27 @@ private RefreshTokenInput decodeRefreshTokenV3(Buffer b, byte[] bytes) { final Instant createdAt = Instant.ofEpochMilli(b2.getLong(8)); final OperatorIdentity operatorIdentity = decodeOperatorIdentityV3(b2, 16); final SourcePublisher sourcePublisher = decodeSourcePublisherV3(b2, 29); - final int privacyBits = b2.getInt(45); + final PrivacyBits privacyBits = PrivacyBits.fromInt(b2.getInt(45)); final Instant establishedAt = Instant.ofEpochMilli(b2.getLong(49)); final IdentityScope identityScope = decodeIdentityScopeV3(b2.getByte(57)); - final IdentityType identityType = decodeIdentityTypeV3(b2.getByte(57)); + final DiiType diiType = decodeIdentityTypeV3(b2.getByte(57)); final byte[] firstLevelHash = b2.getBytes(58, 90); if (identityScope != decodeIdentityScopeV3(b.getByte(0))) { throw new ClientInputValidationException("Failed to decode refreshTokenV3: Identity scope mismatch"); } - if (identityType != decodeIdentityTypeV3(b.getByte(0))) { + if (diiType != decodeIdentityTypeV3(b.getByte(0))) { throw new ClientInputValidationException("Failed to decode refreshTokenV3: Identity type mismatch"); } - return new RefreshTokenInput( + return new TokenRefreshRequest( TokenVersion.V3, createdAt, expiresAt, operatorIdentity, sourcePublisher, - new FirstLevelHashIdentity(identityScope, identityType, firstLevelHash, privacyBits, establishedAt, null)); + new FirstLevelHash(identityScope, diiType, firstLevelHash, establishedAt), + privacyBits); } @Override - public AdvertisingTokenInput decodeAdvertisingToken(String base64AdvertisingToken) { + public AdvertisingTokenRequest decodeAdvertisingToken(String base64AdvertisingToken) { //Logic and code copied from: https://github.com/IABTechLab/uid2-client-java/blob/0220ef43c1661ecf3b8f4ed2db524e2db31c06b5/src/main/java/com/uid2/client/Uid2Encryption.java#L37 if (base64AdvertisingToken.length() < 4) { throw new ClientInputValidationException("Advertising token is too short"); @@ -199,7 +206,7 @@ public AdvertisingTokenInput decodeAdvertisingToken(String base64AdvertisingToke return decodeAdvertisingTokenV3orV4(b, bytes, tokenVersion); } - public AdvertisingTokenInput decodeAdvertisingTokenV2(Buffer b) { + public AdvertisingTokenRequest decodeAdvertisingTokenV2(Buffer b) { try { final int masterKeyId = b.getInt(1); @@ -219,17 +226,18 @@ public AdvertisingTokenInput decodeAdvertisingTokenV2(Buffer b) { final byte[] rawUid = EncodingUtils.fromBase64(b3.slice(8, 8 + length).getBytes()); - final int privacyBits = b3.getInt(8 + length); + final PrivacyBits privacyBits = PrivacyBits.fromInt(b3.getInt(8 + length)); final long establishedMillis = b3.getLong(8 + length + 4); - return new AdvertisingTokenInput( + return new AdvertisingTokenRequest( TokenVersion.V2, Instant.ofEpochMilli(establishedMillis), Instant.ofEpochMilli(expiresMillis), new OperatorIdentity(0, OperatorType.Service, 0, masterKeyId), new SourcePublisher(siteId, siteKeyId, 0), - new RawUidIdentity(IdentityScope.UID2, IdentityType.Email, rawUid, privacyBits, - Instant.ofEpochMilli(establishedMillis), null) + new RawUid(IdentityScope.UID2, DiiType.Email, rawUid), + privacyBits, + Instant.ofEpochMilli(establishedMillis) ); } catch (Exception e) { @@ -238,7 +246,7 @@ public AdvertisingTokenInput decodeAdvertisingTokenV2(Buffer b) { } - public AdvertisingTokenInput decodeAdvertisingTokenV3orV4(Buffer b, byte[] bytes, TokenVersion tokenVersion) { + public AdvertisingTokenRequest decodeAdvertisingTokenV3orV4(Buffer b, byte[] bytes, TokenVersion tokenVersion) { final int masterKeyId = b.getInt(2); final byte[] masterPayloadBytes = AesGcm.decrypt(bytes, 6, this.keyManager.getKey(masterKeyId)); @@ -250,38 +258,40 @@ public AdvertisingTokenInput decodeAdvertisingTokenV3orV4(Buffer b, byte[] bytes final Buffer sitePayload = Buffer.buffer(AesGcm.decrypt(masterPayloadBytes, 33, this.keyManager.getKey(siteKeyId))); final SourcePublisher sourcePublisher = decodeSourcePublisherV3(sitePayload, 0); - final int privacyBits = sitePayload.getInt(16); + final PrivacyBits privacyBits = PrivacyBits.fromInt(sitePayload.getInt(16)); final Instant establishedAt = Instant.ofEpochMilli(sitePayload.getLong(20)); + // refreshedAt is currently not used final Instant refreshedAt = Instant.ofEpochMilli(sitePayload.getLong(28)); final byte[] rawUid = sitePayload.slice(36, sitePayload.length()).getBytes(); final IdentityScope identityScope = rawUid.length == 32 ? IdentityScope.UID2 : decodeIdentityScopeV3(rawUid[0]); - final IdentityType identityType = rawUid.length == 32 ? IdentityType.Email : decodeIdentityTypeV3(rawUid[0]); + final DiiType diiType = rawUid.length == 32 ? DiiType.Email : decodeIdentityTypeV3(rawUid[0]); if (rawUid.length > 32) { if (identityScope != decodeIdentityScopeV3(b.getByte(0))) { throw new ClientInputValidationException("Failed decoding advertisingTokenV3: Identity scope mismatch"); } - if (identityType != decodeIdentityTypeV3(b.getByte(0))) { + if (diiType != decodeIdentityTypeV3(b.getByte(0))) { throw new ClientInputValidationException("Failed decoding advertisingTokenV3: Identity type mismatch"); } } - return new AdvertisingTokenInput( + return new AdvertisingTokenRequest( tokenVersion, createdAt, expiresAt, operatorIdentity, sourcePublisher, - new RawUidIdentity(identityScope, identityType, rawUid, privacyBits, establishedAt, refreshedAt) + new RawUid(identityScope, diiType, rawUid), + privacyBits, establishedAt ); } private void recordRefreshTokenVersionCount(String siteId, TokenVersion tokenVersion) { - Counter.builder("uid2_refresh_token_served_count") + Counter.builder("uid2_refresh_token_served_count_total") .description(String.format("Counter for the amount of refresh token %s served", tokenVersion.toString().toLowerCase())) .tags("site_id", String.valueOf(siteId)) .tags("refresh_token_version", tokenVersion.toString().toLowerCase()) .register(Metrics.globalRegistry).increment(); } - public byte[] encodeIntoRefreshToken(RefreshTokenInput t, Instant asOf) { + public byte[] encodeIntoRefreshToken(TokenRefreshRequest t, Instant asOf) { final KeysetKey serviceKey = this.keyManager.getRefreshKey(asOf); switch (t.version) { @@ -296,7 +306,7 @@ public byte[] encodeIntoRefreshToken(RefreshTokenInput t, Instant asOf) { } } - public byte[] encodeIntoRefreshTokenV2(RefreshTokenInput t, KeysetKey serviceKey) { + public byte[] encodeIntoRefreshTokenV2(TokenRefreshRequest t, KeysetKey serviceKey) { final Buffer b = Buffer.buffer(); b.appendByte((byte) t.version.rawVersion); b.appendLong(t.createdAt.toEpochMilli()); @@ -304,24 +314,25 @@ public byte[] encodeIntoRefreshTokenV2(RefreshTokenInput t, KeysetKey serviceKey // give an extra minute for clients which are trying to refresh tokens close to or at the refresh expiry timestamp b.appendLong(t.expiresAt.plusSeconds(60).toEpochMilli()); b.appendInt(serviceKey.getId()); - final byte[] encryptedIdentity = encryptIdentityV2(t.sourcePublisher, t.firstLevelHashIdentity, serviceKey); + final byte[] encryptedIdentity = encryptIdentityV2(t.sourcePublisher, t.firstLevelHash, serviceKey, + t.privacyBits); b.appendBytes(encryptedIdentity); return b.getBytes(); } - public byte[] encodeIntoRefreshTokenV3(RefreshTokenInput t, KeysetKey serviceKey) { + public byte[] encodeIntoRefreshTokenV3(TokenRefreshRequest t, KeysetKey serviceKey) { final Buffer refreshPayload = Buffer.buffer(90); refreshPayload.appendLong(t.expiresAt.toEpochMilli()); refreshPayload.appendLong(t.createdAt.toEpochMilli()); encodeOperatorIdentityV3(refreshPayload, t.operatorIdentity); encodePublisherRequesterV3(refreshPayload, t.sourcePublisher); - refreshPayload.appendInt(t.firstLevelHashIdentity.privacyBits); - refreshPayload.appendLong(t.firstLevelHashIdentity.establishedAt.toEpochMilli()); - refreshPayload.appendByte(encodeIdentityTypeV3(t.firstLevelHashIdentity.identityScope, t.firstLevelHashIdentity.identityType)); - refreshPayload.appendBytes(t.firstLevelHashIdentity.firstLevelHash); + refreshPayload.appendInt(t.privacyBits.getAsInt()); + refreshPayload.appendLong(t.firstLevelHash.establishedAt().toEpochMilli()); + refreshPayload.appendByte(encodeIdentityTypeV3(t.firstLevelHash.identityScope(), t.firstLevelHash.diiType())); + refreshPayload.appendBytes(t.firstLevelHash.firstLevelHash()); final Buffer b = Buffer.buffer(124); - b.appendByte(encodeIdentityTypeV3(t.firstLevelHashIdentity.identityScope, t.firstLevelHashIdentity.identityType)); + b.appendByte(encodeIdentityTypeV3(t.firstLevelHash.identityScope(), t.firstLevelHash.diiType())); b.appendByte((byte) t.version.rawVersion); b.appendInt(serviceKey.getId()); b.appendBytes(AesGcm.encrypt(refreshPayload.getBytes(), serviceKey).getPayload()); @@ -329,10 +340,10 @@ public byte[] encodeIntoRefreshTokenV3(RefreshTokenInput t, KeysetKey serviceKey return b.getBytes(); } - private void encodeSiteIdentityV2(Buffer b, SourcePublisher sourcePublisher, RawUidIdentity rawUidIdentity, - KeysetKey siteEncryptionKey) { + private void encodeSiteIdentityV2(Buffer b, SourcePublisher sourcePublisher, RawUid rawUid, + KeysetKey siteEncryptionKey, PrivacyBits privacyBits, Instant establishedAt) { b.appendInt(siteEncryptionKey.getId()); - final byte[] encryptedIdentity = encryptIdentityV2(sourcePublisher, rawUidIdentity, siteEncryptionKey); + final byte[] encryptedIdentity = encryptIdentityV2(sourcePublisher, rawUid, siteEncryptionKey, privacyBits, establishedAt); b.appendBytes(encryptedIdentity); } @@ -342,41 +353,42 @@ public static String bytesToBase64Token(byte[] advertisingTokenBytes, TokenVersi } @Override - public IdentityResponse encodeIntoIdentityResponse(AdvertisingTokenInput advertisingTokenInput, RefreshTokenInput refreshTokenInput, Instant refreshFrom, Instant asOf) { - final String advertisingToken = generateAdvertisingTokenString(advertisingTokenInput, asOf); - final String refreshToken = generateRefreshTokenString(refreshTokenInput, asOf); - return new IdentityResponse( + public TokenGenerateResponse encodeIntoIdentityResponse(AdvertisingTokenRequest advertisingTokenRequest, TokenRefreshRequest tokenRefreshRequest, Instant refreshFrom, Instant asOf) { + final String advertisingToken = generateAdvertisingTokenString(advertisingTokenRequest, asOf); + final String refreshToken = generateRefreshTokenString(tokenRefreshRequest, asOf); + return new TokenGenerateResponse( advertisingToken, - advertisingTokenInput.version, + advertisingTokenRequest.version, refreshToken, - advertisingTokenInput.expiresAt, - refreshTokenInput.expiresAt, + advertisingTokenRequest.expiresAt, + tokenRefreshRequest.expiresAt, refreshFrom ); } - private String generateRefreshTokenString(RefreshTokenInput refreshTokenInput, Instant asOf) { - return EncodingUtils.toBase64String(encodeIntoRefreshToken(refreshTokenInput, asOf)); + private String generateRefreshTokenString(TokenRefreshRequest tokenRefreshRequest, Instant asOf) { + return EncodingUtils.toBase64String(encodeIntoRefreshToken(tokenRefreshRequest, asOf)); } - private String generateAdvertisingTokenString(AdvertisingTokenInput advertisingTokenInput, Instant asOf) { - final byte[] advertisingTokenBytes = encodeIntoAdvertisingToken(advertisingTokenInput, asOf); - return bytesToBase64Token(advertisingTokenBytes, advertisingTokenInput.version); + private String generateAdvertisingTokenString(AdvertisingTokenRequest advertisingTokenRequest, Instant asOf) { + final byte[] advertisingTokenBytes = encodeIntoAdvertisingToken(advertisingTokenRequest, asOf); + return bytesToBase64Token(advertisingTokenBytes, advertisingTokenRequest.version); } - private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, FirstLevelHashIdentity firstLevelHashIdentity, KeysetKey key) { - return encryptIdentityV2(sourcePublisher, firstLevelHashIdentity.firstLevelHash, firstLevelHashIdentity.privacyBits, - firstLevelHashIdentity.establishedAt, key); + private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, FirstLevelHash firstLevelHash, + KeysetKey key, PrivacyBits privacyBits) { + return encryptIdentityV2(sourcePublisher, firstLevelHash.firstLevelHash(), privacyBits, + firstLevelHash.establishedAt(), key); } - private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, RawUidIdentity rawUidIdentity, - KeysetKey key) { - return encryptIdentityV2(sourcePublisher, rawUidIdentity.rawUid, rawUidIdentity.privacyBits, - rawUidIdentity.establishedAt, key); + private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, RawUid rawUid, + KeysetKey key, PrivacyBits privacyBits, Instant establishedAt) { + return encryptIdentityV2(sourcePublisher, rawUid.rawUid(), privacyBits, + establishedAt, key); } - private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, byte[] id, int privacyBits, + private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, byte[] id, PrivacyBits privacyBits, Instant establishedAt, KeysetKey key) { Buffer b = Buffer.buffer(); try { @@ -384,7 +396,7 @@ private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, byte[] id, int final byte[] identityBytes = EncodingUtils.toBase64(id); b.appendInt(identityBytes.length); b.appendBytes(identityBytes); - b.appendInt(privacyBits); + b.appendInt(privacyBits.getAsInt()); b.appendLong(establishedAt.toEpochMilli()); return AesCbc.encrypt(b.getBytes(), key).getPayload(); } catch (Exception e) { @@ -392,8 +404,8 @@ private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, byte[] id, int } } - static private byte encodeIdentityTypeV3(IdentityScope identityScope, IdentityType identityType) { - return (byte) (TokenUtils.encodeIdentityScope(identityScope) | (identityType.value << 2) | 3); + static private byte encodeIdentityTypeV3(IdentityScope identityScope, DiiType diiType) { + return (byte) (TokenUtils.encodeIdentityScope(identityScope) | (diiType.value << 2) | 3); // "| 3" is used so that the 2nd char matches the version when V3 or higher. Eg "3" for V3 and "4" for V4 } @@ -401,8 +413,8 @@ static private IdentityScope decodeIdentityScopeV3(byte value) { return IdentityScope.fromValue((value & 0x10) >> 4); } - static private IdentityType decodeIdentityTypeV3(byte value) { - return IdentityType.fromValue((value & 0xf) >> 2); + static private DiiType decodeIdentityTypeV3(byte value) { + return DiiType.fromValue((value & 0xf) >> 2); } static void encodePublisherRequesterV3(Buffer b, SourcePublisher sourcePublisher) { diff --git a/src/main/java/com/uid2/operator/service/ITokenEncoder.java b/src/main/java/com/uid2/operator/service/ITokenEncoder.java index 9380dc8c2..73fe2e70c 100644 --- a/src/main/java/com/uid2/operator/service/ITokenEncoder.java +++ b/src/main/java/com/uid2/operator/service/ITokenEncoder.java @@ -1,15 +1,15 @@ package com.uid2.operator.service; -import com.uid2.operator.model.AdvertisingTokenInput; -import com.uid2.operator.model.IdentityResponse; -import com.uid2.operator.model.RefreshTokenInput; +import com.uid2.operator.model.AdvertisingTokenRequest; +import com.uid2.operator.model.TokenGenerateResponse; +import com.uid2.operator.model.TokenRefreshRequest; import java.time.Instant; public interface ITokenEncoder { - IdentityResponse encodeIntoIdentityResponse(AdvertisingTokenInput advertisingTokenInput, RefreshTokenInput refreshTokenInput, Instant refreshFrom, Instant asOf); + TokenGenerateResponse encodeIntoIdentityResponse(AdvertisingTokenRequest advertisingTokenRequest, TokenRefreshRequest tokenRefreshRequest, Instant refreshFrom, Instant asOf); - AdvertisingTokenInput decodeAdvertisingToken(String base64String); + AdvertisingTokenRequest decodeAdvertisingToken(String base64String); - RefreshTokenInput decodeRefreshToken(String base64String); + TokenRefreshRequest decodeRefreshToken(String base64String); } diff --git a/src/main/java/com/uid2/operator/service/IUIDOperatorService.java b/src/main/java/com/uid2/operator/service/IUIDOperatorService.java index 38624848d..d877940b7 100644 --- a/src/main/java/com/uid2/operator/service/IUIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/IUIDOperatorService.java @@ -1,7 +1,7 @@ package com.uid2.operator.service; import com.uid2.operator.model.*; -import com.uid2.operator.model.userIdentity.HashedDiiIdentity; +import com.uid2.operator.model.identities.HashedDii; import com.uid2.shared.model.SaltEntry; import io.vertx.core.AsyncResult; import io.vertx.core.Handler; @@ -12,22 +12,20 @@ public interface IUIDOperatorService { - IdentityResponse generateIdentity(IdentityRequest request); + TokenGenerateResponse generateIdentity(TokenGenerateRequest request, Duration refreshIdentityAfter, Duration refreshExpiresAfter, Duration identityExpiresAfter); - RefreshResponse refreshIdentity(RefreshTokenInput refreshTokenInput); + TokenRefreshResponse refreshIdentity(TokenRefreshRequest input, Duration refreshIdentityAfter, Duration refreshExpiresAfter, Duration identityExpiresAfter); - RawUidResponse mapIdentity(MapRequest request); + IdentityMapResponseItem mapHashedDii(IdentityMapRequestItem request); @Deprecated - RawUidResponse map(HashedDiiIdentity hashedDiiIdentity, Instant asOf); + IdentityMapResponseItem map(HashedDii hashedDii, Instant asOf); List getModifiedBuckets(Instant sinceTimestamp); - void invalidateTokensAsync(HashedDiiIdentity hashedDiiIdentity, Instant asOf, Handler> handler); + void invalidateTokensAsync(HashedDii hashedDii, Instant asOf, String uidTraceId, Handler> handler); - boolean advertisingTokenMatches(String advertisingToken, HashedDiiIdentity hashedDiiIdentity, Instant asOf); + boolean advertisingTokenMatches(String advertisingToken, HashedDii hashedDii, Instant asOf); - Instant getLatestOptoutEntry(HashedDiiIdentity hashedDiiIdentity, Instant asOf); - - Duration getIdentityExpiryDuration(); + Instant getLatestOptoutEntry(HashedDii hashedDii, Instant asOf); } diff --git a/src/main/java/com/uid2/operator/service/InputUtil.java b/src/main/java/com/uid2/operator/service/InputUtil.java index ff9b3647b..81ed4c6d3 100644 --- a/src/main/java/com/uid2/operator/service/InputUtil.java +++ b/src/main/java/com/uid2/operator/service/InputUtil.java @@ -1,10 +1,8 @@ package com.uid2.operator.service; -import com.uid2.operator.model.IdentityScope; -import com.uid2.operator.model.IdentityType; -import com.uid2.operator.model.userIdentity.HashedDiiIdentity; - -import java.time.Instant; +import com.uid2.operator.model.identities.IdentityScope; +import com.uid2.operator.model.identities.DiiType; +import com.uid2.operator.model.identities.HashedDii; public class InputUtil { @@ -169,7 +167,7 @@ public static String normalizeEmailString(String email) { return addressPartToUse.append('@').append(domainPart).toString(); } - public enum IdentityInputType { + public enum DiiInputType { Raw, Hash } @@ -185,62 +183,63 @@ private static enum EmailParsingState { public static class InputVal { private final String provided; private final String normalized; - private final IdentityType identityType; - private final IdentityInputType inputType; + //Directly Identifying Information (DII) (email or phone) see https://unifiedid.com/docs/ref-info/glossary-uid#gl-dii + private final DiiType diiType; + private final DiiInputType inputType; private final boolean valid; - private final byte[] identityInput; + private final byte[] diiInput; - public InputVal(String provided, String normalized, IdentityType identityType, IdentityInputType inputType, boolean valid) { + public InputVal(String provided, String normalized, DiiType diiType, DiiInputType inputType, boolean valid) { this.provided = provided; this.normalized = normalized; - this.identityType = identityType; + this.diiType = diiType; this.inputType = inputType; this.valid = valid; if (valid) { - if (this.inputType == IdentityInputType.Raw) { - this.identityInput = TokenUtils.getIdentityHash(this.normalized); + if (this.inputType == DiiInputType.Raw) { + this.diiInput = TokenUtils.getHashedDii(this.normalized); } else { - this.identityInput = EncodingUtils.fromBase64(this.normalized); + this.diiInput = EncodingUtils.fromBase64(this.normalized); } } else { - this.identityInput = null; + this.diiInput = null; } } public static InputVal validEmail(String input, String normalized) { - return new InputVal(input, normalized, IdentityType.Email, IdentityInputType.Raw, true); + return new InputVal(input, normalized, DiiType.Email, DiiInputType.Raw, true); } public static InputVal invalidEmail(String input) { - return new InputVal(input, null, IdentityType.Email, IdentityInputType.Raw, false); + return new InputVal(input, null, DiiType.Email, DiiInputType.Raw, false); } public static InputVal validEmailHash(String input, String normalized) { - return new InputVal(input, normalized, IdentityType.Email, IdentityInputType.Hash, true); + return new InputVal(input, normalized, DiiType.Email, DiiInputType.Hash, true); } public static InputVal invalidEmailHash(String input) { - return new InputVal(input, null, IdentityType.Email, IdentityInputType.Hash, false); + return new InputVal(input, null, DiiType.Email, DiiInputType.Hash, false); } public static InputVal validPhone(String input, String normalized) { - return new InputVal(input, normalized, IdentityType.Phone, IdentityInputType.Raw, true); + return new InputVal(input, normalized, DiiType.Phone, DiiInputType.Raw, true); } public static InputVal invalidPhone(String input) { - return new InputVal(input, null, IdentityType.Phone, IdentityInputType.Raw, false); + return new InputVal(input, null, DiiType.Phone, DiiInputType.Raw, false); } public static InputVal validPhoneHash(String input, String normalized) { - return new InputVal(input, normalized, IdentityType.Phone, IdentityInputType.Hash, true); + return new InputVal(input, normalized, DiiType.Phone, DiiInputType.Hash, true); } public static InputVal invalidPhoneHash(String input) { - return new InputVal(input, null, IdentityType.Phone, IdentityInputType.Hash, false); + return new InputVal(input, null, DiiType.Phone, DiiInputType.Hash, false); } - public byte[] getIdentityInput() { - return this.identityInput; + public byte[] getHashedDiiInput() { + return this.diiInput; } public String getProvided() { @@ -251,24 +250,21 @@ public String getNormalized() { return normalized; } - public IdentityType getIdentityType() { - return identityType; + public DiiType getDiiType() { + return diiType; } - public IdentityInputType getInputType() { return inputType; } + public DiiInputType getInputType() { return inputType; } public boolean isValid() { return valid; } - public HashedDiiIdentity toHashedDiiIdentity(IdentityScope identityScope, int privacyBits, Instant establishedAt) { - return new HashedDiiIdentity( + public HashedDii toHashedDii(IdentityScope identityScope) { + return new HashedDii( identityScope, - this.identityType, - getIdentityInput(), - privacyBits, - establishedAt, - establishedAt); + this.diiType, + getHashedDiiInput()); } } diff --git a/src/main/java/com/uid2/operator/service/JsonParseUtils.java b/src/main/java/com/uid2/operator/service/JsonParseUtils.java index 8860c6fd9..4255b799f 100644 --- a/src/main/java/com/uid2/operator/service/JsonParseUtils.java +++ b/src/main/java/com/uid2/operator/service/JsonParseUtils.java @@ -10,7 +10,7 @@ public static JsonArray parseArray(JsonObject object, String key, RoutingContext try { outArray = object.getJsonArray(key); } catch (ClassCastException e) { - ResponseUtil.ClientError(rc, String.format("%s must be an array", key)); + ResponseUtil.LogInfoAndSend400Response(rc, String.format("%s must be an array", key)); return null; } return outArray; diff --git a/src/main/java/com/uid2/operator/service/ResponseUtil.java b/src/main/java/com/uid2/operator/service/ResponseUtil.java index 5f59eab96..a1842c275 100644 --- a/src/main/java/com/uid2/operator/service/ResponseUtil.java +++ b/src/main/java/com/uid2/operator/service/ResponseUtil.java @@ -1,7 +1,6 @@ package com.uid2.operator.service; import com.uid2.operator.monitoring.TokenResponseStatsCollector; -import com.uid2.operator.vertx.UIDOperatorVerticle; import com.uid2.shared.model.TokenVersion; import com.uid2.shared.store.ISiteStore; import io.vertx.core.http.HttpHeaders; @@ -64,19 +63,28 @@ public static void SuccessV2(RoutingContext rc, Object body) { rc.data().put("response", json); } - public static void ClientError(RoutingContext rc, String message) { - Warning(ResponseStatus.ClientError, 400, rc, message); + public static void LogInfoAndSend400Response(RoutingContext rc, String message) { + LogInfoAndSendResponse(ResponseStatus.ClientError, 400, rc, message); } public static void SendClientErrorResponseAndRecordStats(String errorStatus, int statusCode, RoutingContext rc, String message, Integer siteId, TokenResponseStatsCollector.Endpoint endpoint, TokenResponseStatsCollector.ResponseStatus responseStatus, ISiteStore siteProvider, TokenResponseStatsCollector.PlatformType platformType) { - Warning(errorStatus, statusCode, rc, message); + if (ResponseStatus.ClientError.equals(errorStatus) || + ResponseStatus.InvalidAppName.equals(errorStatus) || + ResponseStatus.InvalidHttpOrigin.equals(errorStatus)) + { + LogInfoAndSendResponse(errorStatus, statusCode, rc, message); + } + else { + LogWarningAndSendResponse(errorStatus, statusCode, rc, message); + } + recordTokenResponseStats(siteId, endpoint, responseStatus, siteProvider, null, platformType); } public static void SendServerErrorResponseAndRecordStats(RoutingContext rc, String message, Integer siteId, TokenResponseStatsCollector.Endpoint endpoint, TokenResponseStatsCollector.ResponseStatus responseStatus, ISiteStore siteProvider, Exception exception, TokenResponseStatsCollector.PlatformType platformType) { - Error(ResponseStatus.UnknownError, 500, rc, message, exception); + LogErrorAndSendResponse(ResponseStatus.UnknownError, 500, rc, message, exception); rc.fail(500); recordTokenResponseStats(siteId, endpoint, responseStatus, siteProvider, null, platformType); } @@ -97,62 +105,40 @@ public static JsonObject Response(String status, String message) { return json; } - public static void Error(String errorStatus, int statusCode, RoutingContext rc, String message) { - logError(errorStatus, statusCode, message, new RoutingContextReader(rc), rc.request().remoteAddress().hostAddress()); + public static void LogErrorAndSendResponse(String errorStatus, int statusCode, RoutingContext rc, String message) { + String msg = ComposeMessage(errorStatus, statusCode, message, new RoutingContextReader(rc), rc.request().remoteAddress().hostAddress()); + LOGGER.error(msg); final JsonObject json = Response(errorStatus, message); rc.response().setStatusCode(statusCode).putHeader(HttpHeaders.CONTENT_TYPE, "application/json") .end(json.encode()); } - public static void Error(String errorStatus, int statusCode, RoutingContext rc, String message, Exception exception) { - logError(errorStatus, statusCode, message, new RoutingContextReader(rc), rc.request().remoteAddress().hostAddress(), exception); + public static void LogErrorAndSendResponse(String errorStatus, int statusCode, RoutingContext rc, String message, Exception exception) { + String msg = ComposeMessage(errorStatus, statusCode, message, new RoutingContextReader(rc), rc.request().remoteAddress().hostAddress()); + LOGGER.error(msg, exception); final JsonObject json = Response(errorStatus, message); rc.response().setStatusCode(statusCode).putHeader(HttpHeaders.CONTENT_TYPE, "application/json") .end(json.encode()); } - public static void Warning(String status, int statusCode, RoutingContext rc, String message) { - logWarning(status, statusCode, message, new RoutingContextReader(rc), rc.request().remoteAddress().hostAddress()); + public static void LogInfoAndSendResponse(String status, int statusCode, RoutingContext rc, String message) { + String msg = ComposeMessage(status, statusCode, message, new RoutingContextReader(rc), rc.request().remoteAddress().hostAddress()); + LOGGER.info(msg); final JsonObject json = Response(status, message); rc.response().setStatusCode(statusCode).putHeader(HttpHeaders.CONTENT_TYPE, "application/json") .end(json.encode()); } - private static void logError(String errorStatus, int statusCode, String message, RoutingContextReader contextReader, String clientAddress) { - JsonObject errorJsonObj = JsonObject.of( - "errorStatus", errorStatus, - "contact", contextReader.getContact(), - "siteId", contextReader.getSiteId(), - "statusCode", statusCode, - "clientAddress", clientAddress, - "message", message - ); - final String linkName = contextReader.getLinkName(); - if (!linkName.isBlank()) { - errorJsonObj.put(SecureLinkValidatorService.SERVICE_LINK_NAME, linkName); - } - final String serviceName = contextReader.getServiceName(); - if (!serviceName.isBlank()) { - errorJsonObj.put(SecureLinkValidatorService.SERVICE_NAME, serviceName); - } - LOGGER.error("Error response to http request. " + errorJsonObj.encode()); - } - - private static void logError(String errorStatus, int statusCode, String message, RoutingContextReader contextReader, String clientAddress, Exception exception) { - String errorMessage = "Error response to http request. " + JsonObject.of( - "errorStatus", errorStatus, - "contact", contextReader.getContact(), - "siteId", contextReader.getSiteId(), - "path", contextReader.getPath(), - "statusCode", statusCode, - "clientAddress", clientAddress, - "message", message - ).encode(); - LOGGER.error(errorMessage, exception); + public static void LogWarningAndSendResponse(String status, int statusCode, RoutingContext rc, String message) { + String msg = ComposeMessage(status, statusCode, message, new RoutingContextReader(rc), rc.request().remoteAddress().hostAddress()); + LOGGER.warn(msg); + final JsonObject json = Response(status, message); + rc.response().setStatusCode(statusCode).putHeader(HttpHeaders.CONTENT_TYPE, "application/json") + .end(json.encode()); } - private static void logWarning(String status, int statusCode, String message, RoutingContextReader contextReader, String clientAddress) { - JsonObject warnMessageJsonObject = JsonObject.of( + private static String ComposeMessage(String status, int statusCode, String message, RoutingContextReader contextReader, String clientAddress) { + JsonObject msgJsonObject = JsonObject.of( "errorStatus", status, "contact", contextReader.getContact(), "siteId", contextReader.getSiteId(), @@ -165,14 +151,22 @@ private static void logWarning(String status, int statusCode, String message, Ro final String origin = contextReader.getOrigin(); if (statusCode >= 400 && statusCode < 500) { if (referer != null) { - warnMessageJsonObject.put("referer", referer); + msgJsonObject.put("referer", referer); } if (origin != null) { - warnMessageJsonObject.put("origin", origin); + msgJsonObject.put("origin", origin); } } - String warnMessage = "Warning response to http request. " + warnMessageJsonObject.encode(); - LOGGER.warn(warnMessage); + + final String linkName = contextReader.getLinkName(); + if (!linkName.isBlank()) { + msgJsonObject.put(SecureLinkValidatorService.SERVICE_LINK_NAME, linkName); + } + final String serviceName = contextReader.getServiceName(); + if (!serviceName.isBlank()) { + msgJsonObject.put(SecureLinkValidatorService.SERVICE_NAME, serviceName); + } + return "Response to http request. " + msgJsonObject.encode(); } public static class ResponseStatus { @@ -183,6 +177,7 @@ public static class ResponseStatus { public static final String InvalidToken = "invalid_token"; public static final String ExpiredToken = "expired_token"; public static final String GenericError = "error"; + public static final String InvalidClient = "invalid_client"; public static final String UnknownError = "unknown"; public static final String InsufficientUserConsent = "insufficient_user_consent"; public static final String InvalidHttpOrigin = "invalid_http_origin"; diff --git a/src/main/java/com/uid2/operator/service/SecureLinkValidatorService.java b/src/main/java/com/uid2/operator/service/SecureLinkValidatorService.java index 3a630dfe5..9e1d8b85c 100644 --- a/src/main/java/com/uid2/operator/service/SecureLinkValidatorService.java +++ b/src/main/java/com/uid2/operator/service/SecureLinkValidatorService.java @@ -46,9 +46,12 @@ public boolean validateRequest(RoutingContext rc, JsonObject requestJsonObject, LOGGER.warn("Path: {} , ClientKey has ServiceId set, but LinkId in request was not authorized. ServiceId: {}, LinkId in request: {}", rc.normalizedPath(), clientKey.getServiceId(), linkId); return false; } + if (serviceLink.isDisabled()) { + LOGGER.warn("Path: {} , ServiceLink {} is disabled", rc.normalizedPath(), linkId); + return false; + } if (!serviceLink.getRoles().contains(role)) { LOGGER.warn("Path: {} , ServiceLink {} does not have role {}", rc.normalizedPath(), linkId, role); - return false; } Service service = rotatingServiceStore.getService(clientKey.getServiceId()); diff --git a/src/main/java/com/uid2/operator/service/TokenUtils.java b/src/main/java/com/uid2/operator/service/TokenUtils.java index ef532e578..57d6d01dd 100644 --- a/src/main/java/com/uid2/operator/service/TokenUtils.java +++ b/src/main/java/com/uid2/operator/service/TokenUtils.java @@ -1,45 +1,45 @@ package com.uid2.operator.service; -import com.uid2.operator.model.IdentityScope; -import com.uid2.operator.model.IdentityType; +import com.uid2.operator.model.identities.IdentityScope; +import com.uid2.operator.model.identities.DiiType; import java.util.HashSet; import java.util.Set; public class TokenUtils { - public static byte[] getIdentityHash(String identityString) { - return EncodingUtils.getSha256Bytes(identityString); + public static byte[] getHashedDii(String rawDii) { + return EncodingUtils.getSha256Bytes(rawDii); } - public static String getIdentityHashString(String identityString) { - return EncodingUtils.toBase64String(getIdentityHash(identityString)); + public static String getHashedDiiString(String rawDii) { + return EncodingUtils.toBase64String(getHashedDii(rawDii)); } - public static byte[] getFirstLevelHash(byte[] identityHash, String firstLevelSalt) { - return getFirstLevelHashFromIdentityHash(EncodingUtils.toBase64String(identityHash), firstLevelSalt); + public static byte[] getFirstLevelHashFromHashedDii(byte[] hashedDii, String firstLevelSalt) { + return getFirstLevelHashFromHashedDii(EncodingUtils.toBase64String(hashedDii), firstLevelSalt); } - public static byte[] getFirstLevelHashFromIdentity(String identityString, String firstLevelSalt) { - return getFirstLevelHash(getIdentityHash(identityString), firstLevelSalt); + public static byte[] getFirstLevelHashFromRawDii(String rawDii, String firstLevelSalt) { + return getFirstLevelHashFromHashedDii(getHashedDii(rawDii), firstLevelSalt); } - public static byte[] getFirstLevelHashFromIdentityHash(String identityHash, String firstLevelSalt) { - return EncodingUtils.getSha256Bytes(identityHash, firstLevelSalt); + public static byte[] getFirstLevelHashFromHashedDii(String hashedDii, String firstLevelSalt) { + return EncodingUtils.getSha256Bytes(hashedDii, firstLevelSalt); } public static byte[] getRawUidV2(byte[] firstLevelHash, String rotatingSalt) { return EncodingUtils.getSha256Bytes(EncodingUtils.toBase64String(firstLevelHash), rotatingSalt); } - public static byte[] getRawUidV2FromIdentity(String identityString, String firstLevelSalt, String rotatingSalt) { - return getRawUidV2(getFirstLevelHashFromIdentity(identityString, firstLevelSalt), rotatingSalt); + public static byte[] getRawUidV2FromRawDii(String rawDii, String firstLevelSalt, String rotatingSalt) { + return getRawUidV2(getFirstLevelHashFromRawDii(rawDii, firstLevelSalt), rotatingSalt); } - public static byte[] getRawUidV2FromIdentityHash(String identityString, String firstLevelSalt, String rotatingSalt) { - return getRawUidV2(getFirstLevelHashFromIdentityHash(identityString, firstLevelSalt), rotatingSalt); + public static byte[] getRawUidV2FromHashedDii(String hashedDii, String firstLevelSalt, String rotatingSalt) { + return getRawUidV2(getFirstLevelHashFromHashedDii(hashedDii, firstLevelSalt), rotatingSalt); } - public static byte[] getRawUidV3(IdentityScope scope, IdentityType type, byte[] firstLevelHash, String rotatingSalt) { + public static byte[] getRawUidV3(IdentityScope scope, DiiType type, byte[] firstLevelHash, String rotatingSalt) { final byte[] sha = EncodingUtils.getSha256Bytes(EncodingUtils.toBase64String(firstLevelHash), rotatingSalt); final byte[] rawUid = new byte[33]; rawUid[0] = (byte)(encodeIdentityScope(scope) | encodeIdentityType(type)); @@ -47,36 +47,19 @@ public static byte[] getRawUidV3(IdentityScope scope, IdentityType type, byte[] return rawUid; } - public static byte[] getRawUidV3FromIdentity(IdentityScope scope, IdentityType type, String identityString, String firstLevelSalt, String rotatingSalt) { - return getRawUidV3(scope, type, getFirstLevelHashFromIdentity(identityString, firstLevelSalt), rotatingSalt); + public static byte[] getRawUidV3FromRawDii(IdentityScope scope, DiiType type, String rawDii, String firstLevelSalt, String rotatingSalt) { + return getRawUidV3(scope, type, getFirstLevelHashFromRawDii(rawDii, firstLevelSalt), rotatingSalt); } - public static byte[] getRawUidV3FromIdentityHash(IdentityScope scope, IdentityType type, String identityString, String firstLevelSalt, String rotatingSalt) { - return getRawUidV3(scope, type, getFirstLevelHashFromIdentityHash(identityString, firstLevelSalt), rotatingSalt); + public static byte[] getRawUidV3FromHashedDii(IdentityScope scope, DiiType type, String hashedDii, String firstLevelSalt, String rotatingSalt) { + return getRawUidV3(scope, type, getFirstLevelHashFromHashedDii(hashedDii, firstLevelSalt), rotatingSalt); } public static byte encodeIdentityScope(IdentityScope identityScope) { return (byte) (identityScope.value << 4); } - public static byte encodeIdentityType(IdentityType identityType) { - return (byte) (identityType.value << 2); - } - - public static Set getSiteIdsUsingV4Tokens(String siteIdsUsingV4TokensInString) { - String[] siteIdsV4TokensList = siteIdsUsingV4TokensInString.split(","); - - Set siteIdsV4TokensSet = new HashSet<>(); - try { - for (String siteId : siteIdsV4TokensList) { - String siteIdTrimmed = siteId.trim(); - if (!siteIdTrimmed.isEmpty()) { - siteIdsV4TokensSet.add(Integer.parseInt(siteIdTrimmed)); - } - } - } catch (NumberFormatException ex) { - throw new IllegalArgumentException(String.format("Invalid integer format found in site_ids_using_v4_tokens: %s", siteIdsUsingV4TokensInString)); - } - return siteIdsV4TokensSet; + public static byte encodeIdentityType(DiiType diiType) { + return (byte) (diiType.value << 2); } } diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index 440c7fc8e..a41d95a9d 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -1,18 +1,16 @@ package com.uid2.operator.service; import com.uid2.operator.model.*; -import com.uid2.operator.model.userIdentity.FirstLevelHashIdentity; -import com.uid2.operator.model.userIdentity.HashedDiiIdentity; -import com.uid2.operator.model.userIdentity.RawUidIdentity; +import com.uid2.operator.model.identities.*; import com.uid2.operator.util.PrivacyBits; +import com.uid2.shared.audit.UidInstanceIdProvider; import com.uid2.shared.model.SaltEntry; import com.uid2.operator.store.IOptOutStore; -import com.uid2.shared.store.ISaltProvider; +import com.uid2.shared.store.salt.ISaltProvider; import com.uid2.shared.model.TokenVersion; import io.vertx.core.AsyncResult; import io.vertx.core.Future; import io.vertx.core.Handler; -import io.vertx.core.json.JsonObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,9 +21,8 @@ import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.*; - -import static com.uid2.operator.IdentityConst.*; -import static com.uid2.operator.service.TokenUtils.getSiteIdsUsingV4Tokens; +import static com.uid2.operator.model.identities.IdentityConst.*; +import static java.time.temporal.ChronoUnit.DAYS; public class UIDOperatorService implements IUIDOperatorService { public static final String IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS = "identity_token_expires_after_seconds"; @@ -33,147 +30,143 @@ public class UIDOperatorService implements IUIDOperatorService { public static final String REFRESH_IDENTITY_TOKEN_AFTER_SECONDS = "refresh_identity_token_after_seconds"; private static final Logger LOGGER = LoggerFactory.getLogger(UIDOperatorService.class); - private static final Instant RefreshCutoff = LocalDateTime.parse("2021-03-08T17:00:00", DateTimeFormatter.ISO_LOCAL_DATE_TIME).toInstant(ZoneOffset.UTC); + private static final Instant REFRESH_CUTOFF = LocalDateTime.parse("2021-03-08T17:00:00", DateTimeFormatter.ISO_LOCAL_DATE_TIME).toInstant(ZoneOffset.UTC); + private static final long DAY_IN_MS = Duration.ofDays(1).toMillis(); + private final ISaltProvider saltProvider; private final IOptOutStore optOutStore; - private final ITokenEncoder encoder; + private final EncryptedTokenEncoder encoder; private final Clock clock; private final IdentityScope identityScope; - private final FirstLevelHashIdentity testOptOutIdentityForEmail; - private final FirstLevelHashIdentity testOptOutIdentityForPhone; - private final FirstLevelHashIdentity testValidateIdentityForEmail; - private final FirstLevelHashIdentity testValidateIdentityForPhone; - private final FirstLevelHashIdentity testRefreshOptOutIdentityForEmail; - private final FirstLevelHashIdentity testRefreshOptOutIdentityForPhone; - private final Duration identityExpiresAfter; - private final Duration refreshExpiresAfter; - private final Duration refreshIdentityAfter; + + private final FirstLevelHash testOptOutIdentityForEmail; + private final FirstLevelHash testOptOutIdentityForPhone; + private final FirstLevelHash testValidateIdentityForEmail; + private final FirstLevelHash testValidateIdentityForPhone; + private final FirstLevelHash testRefreshOptOutIdentityForEmail; + private final FirstLevelHash testRefreshOptOutIdentityForPhone; private final OperatorIdentity operatorIdentity; - private final TokenVersion tokenVersionToUseIfNotV4; - private final int advertisingTokenV4Percentage; - private final Set siteIdsUsingV4Tokens; private final TokenVersion refreshTokenVersion; - private final boolean identityV3Enabled; + // if we use Raw UID v3 format for the raw UID2/EUIDs generated in this operator + private final boolean rawUidV3Enabled; private final Handler saltRetrievalResponseHandler; + private final UidInstanceIdProvider uidInstanceIdProvider; - public UIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProvider saltProvider, ITokenEncoder encoder, Clock clock, - IdentityScope identityScope, Handler saltRetrievalResponseHandler) { + public UIDOperatorService(IOptOutStore optOutStore, ISaltProvider saltProvider, EncryptedTokenEncoder encoder, Clock clock, + IdentityScope identityScope, Handler saltRetrievalResponseHandler, boolean identityV3Enabled, UidInstanceIdProvider uidInstanceIdProvider) { this.saltProvider = saltProvider; this.encoder = encoder; this.optOutStore = optOutStore; this.clock = clock; this.identityScope = identityScope; this.saltRetrievalResponseHandler = saltRetrievalResponseHandler; - - this.testOptOutIdentityForEmail = getFirstLevelHashIdentity(identityScope, IdentityType.Email, - InputUtil.normalizeEmail(OptOutIdentityForEmail).getIdentityInput(), Instant.now()); - this.testOptOutIdentityForPhone = getFirstLevelHashIdentity(identityScope, IdentityType.Phone, - InputUtil.normalizePhone(OptOutIdentityForPhone).getIdentityInput(), Instant.now()); - this.testValidateIdentityForEmail = getFirstLevelHashIdentity(identityScope, IdentityType.Email, - InputUtil.normalizeEmail(ValidateIdentityForEmail).getIdentityInput(), Instant.now()); - this.testValidateIdentityForPhone = getFirstLevelHashIdentity(identityScope, IdentityType.Phone, - InputUtil.normalizePhone(ValidateIdentityForPhone).getIdentityInput(), Instant.now()); - this.testRefreshOptOutIdentityForEmail = getFirstLevelHashIdentity(identityScope, IdentityType.Email, - InputUtil.normalizeEmail(RefreshOptOutIdentityForEmail).getIdentityInput(), Instant.now()); - this.testRefreshOptOutIdentityForPhone = getFirstLevelHashIdentity(identityScope, IdentityType.Phone, - InputUtil.normalizePhone(RefreshOptOutIdentityForPhone).getIdentityInput(), Instant.now()); + this.uidInstanceIdProvider = uidInstanceIdProvider; + + this.testOptOutIdentityForEmail = getFirstLevelHashIdentity(identityScope, DiiType.Email, + InputUtil.normalizeEmail(OptOutIdentityForEmail).getHashedDiiInput(), Instant.now()); + this.testOptOutIdentityForPhone = getFirstLevelHashIdentity(identityScope, DiiType.Phone, + InputUtil.normalizePhone(OptOutIdentityForPhone).getHashedDiiInput(), Instant.now()); + this.testValidateIdentityForEmail = getFirstLevelHashIdentity(identityScope, DiiType.Email, + InputUtil.normalizeEmail(ValidateIdentityForEmail).getHashedDiiInput(), Instant.now()); + this.testValidateIdentityForPhone = getFirstLevelHashIdentity(identityScope, DiiType.Phone, + InputUtil.normalizePhone(ValidateIdentityForPhone).getHashedDiiInput(), Instant.now()); + this.testRefreshOptOutIdentityForEmail = getFirstLevelHashIdentity(identityScope, DiiType.Email, + InputUtil.normalizeEmail(RefreshOptOutIdentityForEmail).getHashedDiiInput(), Instant.now()); + this.testRefreshOptOutIdentityForPhone = getFirstLevelHashIdentity(identityScope, DiiType.Phone, + InputUtil.normalizePhone(RefreshOptOutIdentityForPhone).getHashedDiiInput(), Instant.now()); this.operatorIdentity = new OperatorIdentity(0, OperatorType.Service, 0, 0); - this.identityExpiresAfter = Duration.ofSeconds(config.getInteger(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); - this.refreshExpiresAfter = Duration.ofSeconds(config.getInteger(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS)); - this.refreshIdentityAfter = Duration.ofSeconds(config.getInteger(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS)); + this.refreshTokenVersion = TokenVersion.V3; + this.rawUidV3Enabled = identityV3Enabled; + } - if (this.identityExpiresAfter.compareTo(this.refreshExpiresAfter) > 0) { - throw new IllegalStateException(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); + private void validateTokenDurations(Duration refreshIdentityAfter, Duration refreshExpiresAfter, Duration identityExpiresAfter) { + if (identityExpiresAfter.compareTo(refreshExpiresAfter) > 0) { + throw new IllegalStateException(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " (" + refreshExpiresAfter.toSeconds() + ") < " + IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS + " (" + identityExpiresAfter.toSeconds() + ")"); } - if (this.refreshIdentityAfter.compareTo(this.identityExpiresAfter) > 0) { - throw new IllegalStateException(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); + if (refreshIdentityAfter.compareTo(identityExpiresAfter) > 0) { + throw new IllegalStateException(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS + " (" + identityExpiresAfter.toSeconds() + ") < " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS + " (" + refreshIdentityAfter.toSeconds() + ")"); } - if (this.refreshIdentityAfter.compareTo(this.refreshExpiresAfter) > 0) { - throw new IllegalStateException(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); + if (refreshIdentityAfter.compareTo(refreshExpiresAfter) > 0) { + throw new IllegalStateException(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " (" + refreshExpiresAfter.toSeconds() + ") < " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS + " (" + refreshIdentityAfter.toSeconds() + ")"); } - - this.advertisingTokenV4Percentage = config.getInteger("advertising_token_v4_percentage", 0); //0 indicates token v4 will not be used - this.siteIdsUsingV4Tokens = getSiteIdsUsingV4Tokens(config.getString("site_ids_using_v4_tokens", "")); - this.tokenVersionToUseIfNotV4 = config.getBoolean("advertising_token_v3", false) ? TokenVersion.V3 : TokenVersion.V2; - - this.refreshTokenVersion = TokenVersion.V3; - this.identityV3Enabled = config.getBoolean("identity_v3", false); } @Override - public IdentityResponse generateIdentity(IdentityRequest request) { + public TokenGenerateResponse generateIdentity(TokenGenerateRequest request, Duration refreshIdentityAfter, Duration refreshExpiresAfter, Duration identityExpiresAfter) { + this.validateTokenDurations(refreshIdentityAfter, refreshExpiresAfter, identityExpiresAfter); final Instant now = EncodingUtils.NowUTCMillis(this.clock); - final byte[] firstLevelHash = getFirstLevelHash(request.hashedDiiIdentity.hashedDii, now); - final FirstLevelHashIdentity firstLevelHashIdentity = new FirstLevelHashIdentity( - request.hashedDiiIdentity.identityScope, request.hashedDiiIdentity.identityType, firstLevelHash, request.hashedDiiIdentity.privacyBits, - request.hashedDiiIdentity.establishedAt, request.hashedDiiIdentity.refreshedAt); + final byte[] firstLevelHash = getFirstLevelHash(request.hashedDii.hashedDii(), now); + final FirstLevelHash firstLevelHashIdentity = new FirstLevelHash( + request.hashedDii.identityScope(), request.hashedDii.diiType(), firstLevelHash, + request.establishedAt); if (request.shouldCheckOptOut() && getGlobalOptOutResult(firstLevelHashIdentity, false).isOptedOut()) { - return IdentityResponse.OptOutIdentityResponse; + return TokenGenerateResponse.OptOutResponse; } else { - return generateIdentity(request.sourcePublisher, firstLevelHashIdentity); + return this.generateIdentity(request.sourcePublisher, firstLevelHashIdentity, request.privacyBits, refreshIdentityAfter, refreshExpiresAfter, identityExpiresAfter); } } @Override - public RefreshResponse refreshIdentity(RefreshTokenInput token) { + public TokenRefreshResponse refreshIdentity(TokenRefreshRequest input, Duration refreshIdentityAfter, Duration refreshExpiresAfter, Duration identityExpiresAfter) { + this.validateTokenDurations(refreshIdentityAfter, refreshExpiresAfter, identityExpiresAfter); // should not be possible as different scopes should be using different keys, but just in case - if (token.firstLevelHashIdentity.identityScope != this.identityScope) { - return RefreshResponse.Invalid; + if (input.firstLevelHash.identityScope() != this.identityScope) { + return TokenRefreshResponse.Invalid; } - if (token.firstLevelHashIdentity.establishedAt.isBefore(RefreshCutoff)) { - return RefreshResponse.Deprecated; + if (input.firstLevelHash.establishedAt().isBefore(REFRESH_CUTOFF)) { + return TokenRefreshResponse.Deprecated; } final Instant now = clock.instant(); - if (token.expiresAt.isBefore(now)) { - return RefreshResponse.Expired; + if (input.expiresAt.isBefore(now)) { + return TokenRefreshResponse.Expired; } - final PrivacyBits privacyBits = PrivacyBits.fromInt(token.firstLevelHashIdentity.privacyBits); - final boolean isCstg = privacyBits.isClientSideTokenGenerated(); + final boolean isCstg = input.privacyBits.isClientSideTokenGenerated(); try { - final GlobalOptoutResult logoutEntry = getGlobalOptOutResult(token.firstLevelHashIdentity, true); + final GlobalOptoutResult logoutEntry = getGlobalOptOutResult(input.firstLevelHash, true); final boolean optedOut = logoutEntry.isOptedOut(); - final Duration durationSinceLastRefresh = Duration.between(token.createdAt, now); + final Duration durationSinceLastRefresh = Duration.between(input.createdAt, now); if (!optedOut) { - IdentityResponse identityResponse = this.generateIdentity(token.sourcePublisher, token.firstLevelHashIdentity); - - return RefreshResponse.createRefreshedResponse(identityResponse, durationSinceLastRefresh, isCstg); + TokenGenerateResponse tokenGenerateResponse = this.generateIdentity(input.sourcePublisher, + input.firstLevelHash, + input.privacyBits, refreshIdentityAfter, refreshExpiresAfter, identityExpiresAfter); + return TokenRefreshResponse.createRefreshedResponse(tokenGenerateResponse, durationSinceLastRefresh, isCstg); } else { - return RefreshResponse.Optout; + return TokenRefreshResponse.Optout; } } catch (KeyManager.NoActiveKeyException e) { - return RefreshResponse.NoActiveKey; + return TokenRefreshResponse.NoActiveKey; } catch (Exception ex) { - return RefreshResponse.Invalid; + return TokenRefreshResponse.Invalid; } } @Override - public RawUidResponse mapIdentity(MapRequest request) { - final FirstLevelHashIdentity firstLevelHashIdentity = getFirstLevelHashIdentity(request.hashedDiiIdentity, + public IdentityMapResponseItem mapHashedDii(IdentityMapRequestItem request) { + final FirstLevelHash firstLevelHash = getFirstLevelHashIdentity(request.hashedDii, request.asOf); - if (request.shouldCheckOptOut() && getGlobalOptOutResult(firstLevelHashIdentity, false).isOptedOut()) { - return RawUidResponse.OptoutIdentity; + if (request.shouldCheckOptOut() && getGlobalOptOutResult(firstLevelHash, false).isOptedOut()) { + return IdentityMapResponseItem.OptoutIdentity; } else { - return generateRawUid(firstLevelHashIdentity, request.asOf); + return generateRawUid(firstLevelHash, request.asOf); } } @Override - public RawUidResponse map(HashedDiiIdentity diiIdentity, Instant asOf) { - final FirstLevelHashIdentity firstLevelHashIdentity = getFirstLevelHashIdentity(diiIdentity, asOf); - return generateRawUid(firstLevelHashIdentity, asOf); + public IdentityMapResponseItem map(HashedDii hashedDii, Instant asOf) { + final FirstLevelHash firstLevelHash = getFirstLevelHashIdentity(hashedDii, asOf); + return generateRawUid(firstLevelHash, asOf); } @Override @@ -192,11 +185,11 @@ private ISaltProvider.ISaltSnapshot getSaltProviderSnapshot(Instant asOf) { } @Override - public void invalidateTokensAsync(HashedDiiIdentity diiIdentity, Instant asOf, Handler> handler) { - final FirstLevelHashIdentity firstLevelHashIdentity = getFirstLevelHashIdentity(diiIdentity, asOf); - final RawUidResponse rawUidResponse = generateRawUid(firstLevelHashIdentity, asOf); + public void invalidateTokensAsync(HashedDii diiIdentity, Instant asOf, String uidTraceId, Handler> handler) { + final FirstLevelHash firstLevelHash = getFirstLevelHashIdentity(diiIdentity, asOf); + final IdentityMapResponseItem identityMapResponseItem = generateRawUid(firstLevelHash, asOf); - this.optOutStore.addEntry(firstLevelHashIdentity, rawUidResponse.rawUid, r -> { + this.optOutStore.addEntry(firstLevelHash, identityMapResponseItem.rawUid, uidTraceId, this.uidInstanceIdProvider.getInstanceId(), r -> { if (r.succeeded()) { handler.handle(Future.succeededFuture(r.result())); } else { @@ -206,92 +199,111 @@ public void invalidateTokensAsync(HashedDiiIdentity diiIdentity, Instant asOf, H } @Override - public boolean advertisingTokenMatches(String advertisingToken, HashedDiiIdentity diiIdentity, Instant asOf) { - final FirstLevelHashIdentity firstLevelHashIdentity = getFirstLevelHashIdentity(diiIdentity, asOf); - final RawUidResponse rawUidResponse = generateRawUid(firstLevelHashIdentity, asOf); - - final AdvertisingTokenInput token = this.encoder.decodeAdvertisingToken(advertisingToken); - return Arrays.equals(rawUidResponse.rawUid, token.rawUidIdentity.rawUid); + public boolean advertisingTokenMatches(String advertisingToken, HashedDii diiIdentity, Instant asOf) { + final FirstLevelHash firstLevelHash = getFirstLevelHashIdentity(diiIdentity, asOf); + final IdentityMapResponseItem identityMapResponseItem = generateRawUid(firstLevelHash, asOf); + final AdvertisingTokenRequest token = this.encoder.decodeAdvertisingToken(advertisingToken); + return Arrays.equals(identityMapResponseItem.rawUid, token.rawUid.rawUid()); } @Override - public Instant getLatestOptoutEntry(HashedDiiIdentity hashedDiiIdentity, Instant asOf) { - final FirstLevelHashIdentity firstLevelHashIdentity = getFirstLevelHashIdentity(hashedDiiIdentity, asOf); - return this.optOutStore.getLatestEntry(firstLevelHashIdentity); + public Instant getLatestOptoutEntry(HashedDii hashedDii, Instant asOf) { + final FirstLevelHash firstLevelHash = getFirstLevelHashIdentity(hashedDii, asOf); + return this.optOutStore.getLatestEntry(firstLevelHash); } - @Override - public Duration getIdentityExpiryDuration() { - return this.identityExpiresAfter; + private FirstLevelHash getFirstLevelHashIdentity(HashedDii hashedDii, Instant asOf) { + return getFirstLevelHashIdentity(hashedDii.identityScope(), hashedDii.diiType(), hashedDii.hashedDii(), asOf); + } + + private FirstLevelHash getFirstLevelHashIdentity(IdentityScope identityScope, DiiType diiType, byte[] hashedDii, Instant asOf) { + final byte[] firstLevelHash = getFirstLevelHash(hashedDii, asOf); + return new FirstLevelHash(identityScope, diiType, firstLevelHash, null); } - private FirstLevelHashIdentity getFirstLevelHashIdentity(HashedDiiIdentity hashedDiiIdentity, Instant asOf) { - return getFirstLevelHashIdentity(hashedDiiIdentity.identityScope, hashedDiiIdentity.identityType, hashedDiiIdentity.hashedDii, asOf); + private byte[] getFirstLevelHash(byte[] hashedDii, Instant asOf) { + return TokenUtils.getFirstLevelHashFromHashedDii(hashedDii, getSaltProviderSnapshot(asOf).getFirstLevelSalt()); } - private FirstLevelHashIdentity getFirstLevelHashIdentity(IdentityScope identityScope, IdentityType identityType, byte[] identityHash, Instant asOf) { - final byte[] firstLevelHash = getFirstLevelHash(identityHash, asOf); - return new FirstLevelHashIdentity(identityScope, identityType, firstLevelHash, 0, null, null); + + private IdentityMapResponseItem generateRawUid(FirstLevelHash firstLevelHash, Instant asOf) { + final SaltEntry rotatingSalt = getSaltProviderSnapshot(asOf).getRotatingSalt(firstLevelHash.firstLevelHash()); + final byte[] advertisingId = getAdvertisingId(firstLevelHash, rotatingSalt.currentSalt()); + final byte[] previousAdvertisingId = getPreviousAdvertisingId(firstLevelHash, rotatingSalt, asOf); + final long refreshFrom = getRefreshFrom(rotatingSalt, asOf); + + return new IdentityMapResponseItem( + advertisingId, + rotatingSalt.hashedId(), + previousAdvertisingId, + refreshFrom); } - private byte[] getFirstLevelHash(byte[] identityHash, Instant asOf) { - return TokenUtils.getFirstLevelHash(identityHash, getSaltProviderSnapshot(asOf).getFirstLevelSalt()); + private byte[] getAdvertisingId(FirstLevelHash firstLevelHash, String salt) { + return rawUidV3Enabled + ? TokenUtils.getRawUidV3(firstLevelHash.identityScope(), firstLevelHash.diiType(), + firstLevelHash.firstLevelHash(), salt) + : TokenUtils.getRawUidV2(firstLevelHash.firstLevelHash(), salt); } - private RawUidResponse generateRawUid(FirstLevelHashIdentity firstLevelHashIdentity, Instant asOf) { - final SaltEntry rotatingSalt = getSaltProviderSnapshot(asOf).getRotatingSalt(firstLevelHashIdentity.firstLevelHash); + private byte[] getPreviousAdvertisingId(FirstLevelHash firstLevelHash, SaltEntry rotatingSalt, Instant asOf) { + long age = asOf.toEpochMilli() - rotatingSalt.lastUpdated(); + if (age / DAY_IN_MS < 90) { + if (rotatingSalt.previousSalt() == null || rotatingSalt.previousSalt().isBlank()) { + return null; + } + return getAdvertisingId(firstLevelHash, rotatingSalt.previousSalt()); + } + return null; + } - return new RawUidResponse( - this.identityV3Enabled - ? TokenUtils.getRawUidV3(firstLevelHashIdentity.identityScope, - firstLevelHashIdentity.identityType, firstLevelHashIdentity.firstLevelHash, rotatingSalt.getSalt()) - : TokenUtils.getRawUidV2(firstLevelHashIdentity.firstLevelHash, rotatingSalt.getSalt()), - rotatingSalt.getHashedId()); + private long getRefreshFrom(SaltEntry rotatingSalt, Instant asOf) { + Long refreshFrom = rotatingSalt.refreshFrom(); + if (refreshFrom == null || refreshFrom < asOf.toEpochMilli()) { + return asOf.truncatedTo(DAYS).plus(1, DAYS).toEpochMilli(); + } + return refreshFrom; } - private IdentityResponse generateIdentity(SourcePublisher sourcePublisher, FirstLevelHashIdentity firstLevelHashIdentity) { + private TokenGenerateResponse generateIdentity(SourcePublisher sourcePublisher, + FirstLevelHash firstLevelHash, PrivacyBits privacyBits, + Duration refreshIdentityAfter, Duration refreshExpiresAfter, Duration identityExpiresAfter) { final Instant nowUtc = EncodingUtils.NowUTCMillis(this.clock); - final RawUidResponse rawUidResponse = generateRawUid(firstLevelHashIdentity, nowUtc); - final RawUidIdentity rawUidIdentity = new RawUidIdentity(firstLevelHashIdentity.identityScope, - firstLevelHashIdentity.identityType, - rawUidResponse.rawUid, firstLevelHashIdentity.privacyBits, firstLevelHashIdentity.establishedAt, nowUtc); + final IdentityMapResponseItem identityMapResponseItem = generateRawUid(firstLevelHash, nowUtc); + final RawUid rawUid = new RawUid(firstLevelHash.identityScope(), + firstLevelHash.diiType(), + identityMapResponseItem.rawUid); return this.encoder.encodeIntoIdentityResponse( - this.createAdvertisingTokenInput(sourcePublisher, rawUidIdentity, nowUtc), - this.createRefreshTokenInput(sourcePublisher, firstLevelHashIdentity, nowUtc), + this.createAdvertisingTokenRequest(sourcePublisher, rawUid, nowUtc, privacyBits, + firstLevelHash.establishedAt(), identityExpiresAfter), + this.createTokenRefreshRequest(sourcePublisher, firstLevelHash, nowUtc, privacyBits, refreshExpiresAfter), nowUtc.plusMillis(refreshIdentityAfter.toMillis()), nowUtc ); } - private RefreshTokenInput createRefreshTokenInput(SourcePublisher sourcePublisher, FirstLevelHashIdentity firstLevelHashIdentity, - Instant now) { - return new RefreshTokenInput( + private TokenRefreshRequest createTokenRefreshRequest(SourcePublisher sourcePublisher, + FirstLevelHash firstLevelHash, + Instant now, + PrivacyBits privacyBits, Duration refreshExpiresAfter) { + return new TokenRefreshRequest( this.refreshTokenVersion, now, now.plusMillis(refreshExpiresAfter.toMillis()), this.operatorIdentity, sourcePublisher, - firstLevelHashIdentity); + firstLevelHash, + privacyBits); } - private AdvertisingTokenInput createAdvertisingTokenInput(SourcePublisher sourcePublisher, RawUidIdentity rawUidIdentity, - Instant now) { - TokenVersion tokenVersion; - if (siteIdsUsingV4Tokens.contains(sourcePublisher.siteId)) { - tokenVersion = TokenVersion.V4; - } else { - int pseudoRandomNumber = 1; - final var rawUid = rawUidIdentity.rawUid; - if (rawUid.length > 2) - { - int hash = ((rawUid[0] & 0xFF) << 12) | ((rawUid[1] & 0xFF) << 4) | ((rawUid[2] & 0xFF) & 0xF); //using same logic as ModBasedSaltEntryIndexer.getIndex() in uid2-shared - pseudoRandomNumber = (hash % 100) + 1; //1 to 100 - } - tokenVersion = (pseudoRandomNumber <= this.advertisingTokenV4Percentage) ? TokenVersion.V4 : this.tokenVersionToUseIfNotV4; - } - return new AdvertisingTokenInput(tokenVersion, now, now.plusMillis(identityExpiresAfter.toMillis()), this.operatorIdentity, sourcePublisher, rawUidIdentity); + private AdvertisingTokenRequest createAdvertisingTokenRequest(SourcePublisher sourcePublisher, RawUid rawUidIdentity, + Instant now, PrivacyBits privacyBits, Instant establishedAt, Duration identityExpiresAfter) { + + return new AdvertisingTokenRequest(TokenVersion.V4, now, now.plusMillis(identityExpiresAfter.toMillis()), + this.operatorIdentity, sourcePublisher, rawUidIdentity, + privacyBits, establishedAt); } static protected class GlobalOptoutResult { @@ -315,24 +327,16 @@ public Instant getTime() { } } - private GlobalOptoutResult getGlobalOptOutResult(FirstLevelHashIdentity firstLevelHashIdentity, boolean forRefresh) { - if (forRefresh && (firstLevelHashIdentity.matches(testRefreshOptOutIdentityForEmail) || firstLevelHashIdentity.matches(testRefreshOptOutIdentityForPhone))) { + private GlobalOptoutResult getGlobalOptOutResult(FirstLevelHash firstLevelHash, boolean forRefresh) { + if (forRefresh && (firstLevelHash.matches(testRefreshOptOutIdentityForEmail) || firstLevelHash.matches(testRefreshOptOutIdentityForPhone))) { return new GlobalOptoutResult(Instant.now()); - } else if (firstLevelHashIdentity.matches(testValidateIdentityForEmail) || firstLevelHashIdentity.matches(testValidateIdentityForPhone) - || firstLevelHashIdentity.matches(testRefreshOptOutIdentityForEmail) || firstLevelHashIdentity.matches(testRefreshOptOutIdentityForPhone)) { + } else if (firstLevelHash.matches(testValidateIdentityForEmail) || firstLevelHash.matches(testValidateIdentityForPhone) + || firstLevelHash.matches(testRefreshOptOutIdentityForEmail) || firstLevelHash.matches(testRefreshOptOutIdentityForPhone)) { return new GlobalOptoutResult(null); - } else if (firstLevelHashIdentity.matches(testOptOutIdentityForEmail) || firstLevelHashIdentity.matches(testOptOutIdentityForPhone)) { + } else if (firstLevelHash.matches(testOptOutIdentityForEmail) || firstLevelHash.matches(testOptOutIdentityForPhone)) { return new GlobalOptoutResult(Instant.now()); } - Instant result = this.optOutStore.getLatestEntry(firstLevelHashIdentity); + Instant result = this.optOutStore.getLatestEntry(firstLevelHash); return new GlobalOptoutResult(result); } - - public TokenVersion getAdvertisingTokenVersionForTests(int siteId) { - assert this.advertisingTokenV4Percentage == 0 || this.advertisingTokenV4Percentage == 100; //we want tests to be deterministic - if (this.siteIdsUsingV4Tokens.contains(siteId)) { - return TokenVersion.V4; - } - return this.advertisingTokenV4Percentage == 100 ? TokenVersion.V4 : this.tokenVersionToUseIfNotV4; - } } diff --git a/src/main/java/com/uid2/operator/service/V2RequestUtil.java b/src/main/java/com/uid2/operator/service/V2RequestUtil.java index d9737323b..77f1cd1f0 100644 --- a/src/main/java/com/uid2/operator/service/V2RequestUtil.java +++ b/src/main/java/com/uid2/operator/service/V2RequestUtil.java @@ -1,8 +1,8 @@ package com.uid2.operator.service; -import com.uid2.operator.model.IdentityScope; +import com.uid2.operator.model.identities.IdentityScope; import com.uid2.operator.model.KeyManager; -import com.uid2.operator.vertx.ClientInputValidationException; +import com.uid2.operator.util.HttpMediaType; import com.uid2.shared.IClock; import com.uid2.shared.Utils; import com.uid2.shared.auth.ClientKey; @@ -10,12 +10,13 @@ import com.uid2.shared.encryption.Random; import com.uid2.shared.model.KeysetKey; import io.vertx.core.buffer.Buffer; +import io.vertx.core.http.HttpHeaders; import io.vertx.core.json.JsonObject; +import io.vertx.ext.web.RoutingContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.nio.charset.StandardCharsets; -import java.time.Clock; import java.time.Duration; import java.time.Instant; @@ -55,22 +56,43 @@ public boolean isValid() { private static final Logger LOGGER = LoggerFactory.getLogger(V2RequestUtil.class); + public static V2Request parseRequest(RoutingContext rc, ClientKey ck, IClock clock) { + if (rc.request().headers().contains(HttpHeaders.CONTENT_TYPE, HttpMediaType.APPLICATION_OCTET_STREAM.getType(), true)) { + return V2RequestUtil.parseRequestAsBuffer(rc.body().buffer(), ck, clock); + } else { + return V2RequestUtil.parseRequestAsString(rc.body().asString(), ck, clock); + } + } + + public static V2Request parseRequestAsBuffer(Buffer bodyBuffer, ClientKey ck, IClock clock) { + if (bodyBuffer == null) { + return new V2Request("Invalid body: Body is missing."); + } + return parseRequestCommon(bodyBuffer.getBytes(), ck, clock); + } + // clock is passed in to test V2_REQUEST_TIMESTAMP_DRIFT_THRESHOLD_IN_MINUTES in unit tests - public static V2Request parseRequest(String bodyString, ClientKey ck, IClock clock) { + public static V2Request parseRequestAsString(String bodyString, ClientKey ck, IClock clock) { if (bodyString == null) { return new V2Request("Invalid body: Body is missing."); } - byte[] bodyBytes; try { - // Payload envelop format: - // byte 0: version - // byte 1-12: GCM IV - // byte 13-end: encrypted payload + GCM AUTH TAG bodyBytes = Utils.decodeBase64String(bodyString); } catch (IllegalArgumentException ex) { return new V2Request("Invalid body: Body is not valid base64."); } + return parseRequestCommon(bodyBytes, ck, clock); + } + + private static V2Request parseRequestCommon(byte[] bodyBytes, ClientKey ck, IClock clock) { + // Payload envelop format: + // byte 0: version + // byte 1-12: GCM IV + // byte 13-end: encrypted payload + GCM AUTH TAG + if (bodyBytes == null || bodyBytes.length == 0) { + return new V2Request("Invalid body: Body is missing."); + } if (bodyBytes.length < MIN_PAYLOAD_LENGTH) { return new V2Request("Invalid body: Body too short. Check encryption method."); @@ -172,7 +194,12 @@ public static void handleRefreshTokenInResponseBody(JsonObject bodyJson, KeyMana .appendInt(refreshKey.getId()) .appendBytes(encrypted) .getBytes()); - assert modifiedToken.length() == V2_REFRESH_PAYLOAD_LENGTH; + if (modifiedToken.length() != V2_REFRESH_PAYLOAD_LENGTH) { + final String errorMsg = "Generated refresh token's length=" + modifiedToken.length() + + " is not equal to=" + V2_REFRESH_PAYLOAD_LENGTH; + LOGGER.error(errorMsg); + throw new IllegalArgumentException(errorMsg); + } bodyJson.put("refresh_token", modifiedToken); bodyJson.put("refresh_response_key", refreshResponseKey); diff --git a/src/main/java/com/uid2/operator/store/BootstrapConfigStore.java b/src/main/java/com/uid2/operator/store/BootstrapConfigStore.java new file mode 100644 index 000000000..04b36c4ba --- /dev/null +++ b/src/main/java/com/uid2/operator/store/BootstrapConfigStore.java @@ -0,0 +1,26 @@ +package com.uid2.operator.store; + +import io.vertx.core.json.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BootstrapConfigStore implements IConfigStore { + private static final Logger logger = LoggerFactory.getLogger(BootstrapConfigStore.class); + private final RuntimeConfig config; + + public BootstrapConfigStore(JsonObject config) { + this.config = config.mapTo(RuntimeConfig.class); + logger.info("Successfully loaded bootstrap config"); + } + + @Override + public RuntimeConfig getConfig() { + return config; + } + + @Override + public void loadContent() throws Exception { + logger.info("Remote Config FF is not enabled, bootstrap config was loaded."); + return; + } +} diff --git a/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java b/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java index e33106949..cc727ac03 100644 --- a/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java +++ b/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java @@ -4,9 +4,10 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.uid2.operator.Const; -import com.uid2.operator.model.userIdentity.FirstLevelHashIdentity; +import com.uid2.operator.model.identities.FirstLevelHash; import com.uid2.operator.service.EncodingUtils; import com.uid2.shared.Utils; +import com.uid2.shared.audit.Audit; import com.uid2.shared.cloud.CloudStorageException; import com.uid2.shared.cloud.DownloadCloudStorage; import com.uid2.shared.cloud.ICloudStorage; @@ -19,6 +20,7 @@ import io.vertx.core.Handler; import io.vertx.core.Vertx; import io.vertx.core.json.JsonObject; +import io.vertx.ext.web.client.HttpRequest; import io.vertx.ext.web.client.HttpResponse; import io.vertx.ext.web.client.WebClient; import io.vertx.ext.web.codec.BodyCodec; @@ -74,8 +76,8 @@ public CloudSyncOptOutStore(Vertx vertx, ICloudStorage fsLocal, JsonObject jsonC } @Override - public Instant getLatestEntry(FirstLevelHashIdentity firstLevelHashIdentity) { - long epochSecond = this.snapshot.get().getOptOutTimestamp(firstLevelHashIdentity.firstLevelHash); + public Instant getLatestEntry(FirstLevelHash firstLevelHash) { + long epochSecond = this.snapshot.get().getOptOutTimestamp(firstLevelHash.firstLevelHash()); Instant instant = epochSecond > 0 ? Instant.ofEpochSecond(epochSecond) : null; return instant; } @@ -86,32 +88,38 @@ public long getOptOutTimestampByAdId(String adId) { } @Override - public void addEntry(FirstLevelHashIdentity firstLevelHashIdentity, byte[] advertisingId, Handler> handler) { + public void addEntry(FirstLevelHash firstLevelHash, byte[] advertisingId, String uidTraceId, String uidInstanceId, Handler> handler) { if (remoteApiHost == null) { handler.handle(Future.failedFuture("remote api not set")); return; } - this.webClient.get(remoteApiPort, remoteApiHost, remoteApiPath). - addQueryParam("identity_hash", EncodingUtils.toBase64String(firstLevelHashIdentity.firstLevelHash)) + HttpRequest request =this.webClient.get(remoteApiPort, remoteApiHost, remoteApiPath). + addQueryParam("identity_hash", EncodingUtils.toBase64String(firstLevelHash.firstLevelHash())) .addQueryParam("advertising_id", EncodingUtils.toBase64String(advertisingId)) // advertising id aka raw UID .putHeader("Authorization", remoteApiBearerToken) - .as(BodyCodec.string()) - .send(ar -> { - Exception failure = null; - if (ar.failed()) { - failure = new Exception(ar.cause()); - } else if (ar.result().statusCode() != 200) { - failure = new Exception("optout api http status: " + String.valueOf(ar.result().statusCode())); - } + .putHeader(Audit.UID_INSTANCE_ID_HEADER, uidInstanceId) + .as(BodyCodec.string()); - if (failure == null) { - handler.handle(Future.succeededFuture(Utils.nowUTCMillis())); - } else { - LOGGER.error("CloudSyncOptOutStore.addEntry remote web request failed", failure); - handler.handle(Future.failedFuture(failure)); - } - }); + if (uidTraceId != null) { + request = request.putHeader(Audit.UID_TRACE_ID_HEADER, uidTraceId); + } + + request.send(ar -> { + Exception failure = null; + if (ar.failed()) { + failure = new Exception(ar.cause()); + } else if (ar.result().statusCode() != 200) { + failure = new Exception("optout api http status: " + String.valueOf(ar.result().statusCode())); + } + + if (failure == null) { + handler.handle(Future.succeededFuture(Utils.nowUTCMillis())); + } else { + LOGGER.error("CloudSyncOptOutStore.addEntry remote web request failed", failure); + handler.handle(Future.failedFuture(failure)); + } + }); } public void registerCloudSync(OptOutCloudSync cloudSync) { @@ -298,7 +306,7 @@ public Collection getLoadedPartitions() { public static class OptOutStoreSnapshot { private static final Logger LOGGER = LoggerFactory.getLogger(OptOutStoreSnapshot.class); - private static final String METRIC_NAME_PREFIX = "uid2.optout."; + private static final String METRIC_NAME_PREFIX = "uid2_optout_"; // Metrics for processing deltas and partitions. private static final String OPT_OUT_PROCESSING_METRIC_NAME = METRIC_NAME_PREFIX + "processing"; @@ -525,7 +533,11 @@ private IndexUpdateMessage getIndexUpdateMessage(Instant now, Collection ium.addDeltaFile(f); else if (OptOutUtils.isPartitionFile(f)) ium.addPartitionFile(f); - else assert false; + else { + final String errorMsg = "File to index " + f + " is not of type delta or partition"; + LOGGER.error(errorMsg); + throw new IllegalStateException(errorMsg); + } } Collection indexedNonSynthetic = indexedFiles.stream() @@ -538,7 +550,12 @@ else if (OptOutUtils.isPartitionFile(f)) Instant tsOld = OptOutUtils.lastPartitionTimestamp(indexedNonSynthetic); Instant tsNew = OptOutUtils.lastPartitionTimestamp(newNonSynthetic); - assert tsOld == Instant.EPOCH || tsNew == Instant.EPOCH || tsOld.isBefore(tsNew); + if (tsOld != Instant.EPOCH && tsNew != Instant.EPOCH && !tsOld.isBefore(tsNew)) { + final String errorMsg = "Last partition timestamp of indexed files " + tsOld.getEpochSecond() + + " is after last partition of non-indexed files " + tsNew.getEpochSecond(); + // Leaving this as a warning until issue is fixed permanently + LOGGER.warn(errorMsg); + } // if there are new partitions in this update, let index delete some in-mem delta caches that is old if (tsNew != Instant.EPOCH) { tsNew = tsNew.minusSeconds(fileUtils.lookbackGracePeriod()); @@ -594,15 +611,21 @@ private OptOutStoreSnapshot updateIndexInternal(IndexUpdateContext iuc) { try { if (numPartitions == 0) { // if update doesn't have a new partition, simply update heap with new log data - assert iuc.getDeltasToRemove().size() == 0; + if (!iuc.getDeltasToRemove().isEmpty()) { + final String errorMsg = "Invalid number of Deltas to remove=" + iuc.getDeltasToRemove().size() + + " when there are 0 new partitions to index"; + LOGGER.error(errorMsg); + throw new IllegalStateException(errorMsg); + } return this.processDeltas(iuc); } else if (numPartitions > 1) { - // should not load more than 1 partition at a time, unless during service bootstrap - assert this.iteration == 0; + if (this.iteration != 0) { + final String errorMsg = "Should not load more than 1 partition at a time, unless during service bootstrap. Current iteration " + this.iteration; + // Leaving this as a warning as this condition is true in production + LOGGER.warn(errorMsg); + } return this.processPartitions(iuc); } else { - // array size cannot be a negative value - assert numPartitions == 1; return this.processPartitions(iuc); } } finally { @@ -628,7 +651,11 @@ private OptOutStoreSnapshot processDeltasImpl(IndexUpdateContext iuc) { // this is thread-safe, as heap is not being used // and bloomfilter can tolerate false positive for (byte[] data : loadedData) { - assert data.length != 0; + if (data.length == 0) { + final String errorMsg = "Loaded delta file has 0 size"; + LOGGER.error(errorMsg); + throw new IllegalStateException(errorMsg); + } OptOutCollection newLog = new OptOutCollection(data); this.heap.add(newLog); @@ -679,7 +706,11 @@ private OptOutStoreSnapshot processPartitionsImpl(IndexUpdateContext iuc) { } for (String key : sortedPartitionFiles) { byte[] data = iuc.loadedPartitions.get(key); - assert data.length != 0; + if (data.length == 0) { + final String errorMsg = "Loaded partition file has 0 size"; + LOGGER.error(errorMsg); + throw new IllegalStateException(errorMsg); + } newPartitions[snapIndex++] = new OptOutPartition(data); } diff --git a/src/main/java/com/uid2/operator/store/IConfigStore.java b/src/main/java/com/uid2/operator/store/IConfigStore.java new file mode 100644 index 000000000..6a41bff90 --- /dev/null +++ b/src/main/java/com/uid2/operator/store/IConfigStore.java @@ -0,0 +1,8 @@ +package com.uid2.operator.store; + +import io.vertx.core.json.JsonObject; + +public interface IConfigStore { + RuntimeConfig getConfig(); + void loadContent() throws Exception; +} diff --git a/src/main/java/com/uid2/operator/store/IOptOutStore.java b/src/main/java/com/uid2/operator/store/IOptOutStore.java index 995939c70..0ed8a95c7 100644 --- a/src/main/java/com/uid2/operator/store/IOptOutStore.java +++ b/src/main/java/com/uid2/operator/store/IOptOutStore.java @@ -1,6 +1,6 @@ package com.uid2.operator.store; -import com.uid2.operator.model.userIdentity.FirstLevelHashIdentity; +import com.uid2.operator.model.identities.FirstLevelHash; import io.vertx.core.AsyncResult; import io.vertx.core.Handler; @@ -11,12 +11,12 @@ public interface IOptOutStore { /** * Get latest opt-out record * - * @param firstLevelHashIdentity The first level hash of a DII Hash + * @param firstLevelHash The first level hash of a DII Hash * @return The timestamp of latest opt-out record. NULL if no record. */ - Instant getLatestEntry(FirstLevelHashIdentity firstLevelHashIdentity); + Instant getLatestEntry(FirstLevelHash firstLevelHash); long getOptOutTimestampByAdId(String adId); - void addEntry(FirstLevelHashIdentity firstLevelHashIdentity, byte[] advertisingId, Handler> handler); + void addEntry(FirstLevelHash firstLevelHash, byte[] advertisingId, String uidTraceId, String uidInstanceId, Handler> handler); } diff --git a/src/main/java/com/uid2/operator/store/RuntimeConfig.java b/src/main/java/com/uid2/operator/store/RuntimeConfig.java new file mode 100644 index 000000000..dfaf69339 --- /dev/null +++ b/src/main/java/com/uid2/operator/store/RuntimeConfig.java @@ -0,0 +1,159 @@ +package com.uid2.operator.store; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; + +@JsonDeserialize(builder = RuntimeConfig.Builder.class) +public class RuntimeConfig { + private final Integer identityTokenExpiresAfterSeconds; + private final Integer refreshTokenExpiresAfterSeconds; + private final Integer refreshIdentityTokenAfterSeconds; + private final Integer sharingTokenExpirySeconds; + private final Integer maxBidstreamLifetimeSeconds; + private final Integer maxSharingLifetimeSeconds; + + public Integer getIdentityTokenExpiresAfterSeconds() { + return identityTokenExpiresAfterSeconds; + } + + public Integer getRefreshTokenExpiresAfterSeconds() { + return refreshTokenExpiresAfterSeconds; + } + + public Integer getRefreshIdentityTokenAfterSeconds() { + return refreshIdentityTokenAfterSeconds; + } + + public Integer getMaxBidstreamLifetimeSeconds() { + if (maxBidstreamLifetimeSeconds != null) { + return maxBidstreamLifetimeSeconds; + } else { + return identityTokenExpiresAfterSeconds; + } + } + + public Integer getMaxSharingLifetimeSeconds() { + if (maxSharingLifetimeSeconds != null) { + return maxSharingLifetimeSeconds; + } else { + return sharingTokenExpirySeconds; + } + } + + public Integer getSharingTokenExpirySeconds() { + return sharingTokenExpirySeconds; + } + + private RuntimeConfig(Builder builder) { + this.identityTokenExpiresAfterSeconds = builder.identityTokenExpiresAfterSeconds; + this.refreshTokenExpiresAfterSeconds = builder.refreshTokenExpiresAfterSeconds; + this.refreshIdentityTokenAfterSeconds = builder.refreshIdentityTokenAfterSeconds; + this.sharingTokenExpirySeconds = builder.sharingTokenExpirySeconds; + this.maxBidstreamLifetimeSeconds = builder.maxBidstreamLifetimeSeconds; + this.maxSharingLifetimeSeconds = builder.maxSharingLifetimeSeconds; + + validateIdentityRefreshTokens(); + validateBidstreamLifetime(); + validateSharingTokenExpiry(); + } + + public RuntimeConfig.Builder toBuilder() { + return new Builder() + .withIdentityTokenExpiresAfterSeconds(this.identityTokenExpiresAfterSeconds) + .withRefreshTokenExpiresAfterSeconds(this.refreshTokenExpiresAfterSeconds) + .withRefreshIdentityTokenAfterSeconds(this.refreshIdentityTokenAfterSeconds) + .withSharingTokenExpirySeconds(this.sharingTokenExpirySeconds) + .withMaxBidstreamLifetimeSeconds(this.maxBidstreamLifetimeSeconds) + .withMaxSharingLifetimeSeconds(this.maxSharingLifetimeSeconds); + } + + private void validateIdentityRefreshTokens() { + if (this.identityTokenExpiresAfterSeconds == null) { + throw new IllegalArgumentException("identity_token_expires_after_seconds is required"); + } + + if (this.refreshTokenExpiresAfterSeconds == null) { + throw new IllegalArgumentException("refresh_token_expires_after_seconds is required"); + } + + if (this.refreshIdentityTokenAfterSeconds == null) { + throw new IllegalArgumentException("refresh_identity_token_after_seconds is required"); + } + + if (this.refreshTokenExpiresAfterSeconds < this.identityTokenExpiresAfterSeconds) { + throw new IllegalArgumentException(String.format("refresh_token_expires_after_seconds (%d) must be >= identity_token_expires_after_seconds (%d)", refreshTokenExpiresAfterSeconds, identityTokenExpiresAfterSeconds)); + } + + if (this.identityTokenExpiresAfterSeconds < this.refreshIdentityTokenAfterSeconds) { + throw new IllegalArgumentException(String.format("identity_token_expires_after_seconds (%d) must be >= refresh_identity_token_after_seconds (%d)", identityTokenExpiresAfterSeconds, refreshIdentityTokenAfterSeconds)); + } + } + + private void validateBidstreamLifetime() { + if (this.maxBidstreamLifetimeSeconds != null && this.maxBidstreamLifetimeSeconds < this.identityTokenExpiresAfterSeconds) { + throw new IllegalArgumentException(String.format("max_bidstream_lifetime_seconds (%d) must be >= identity_token_expires_after_seconds (%d)", maxBidstreamLifetimeSeconds, identityTokenExpiresAfterSeconds)); + } + } + + private void validateSharingTokenExpiry() { + if (this.sharingTokenExpirySeconds == null) { + throw new IllegalArgumentException("sharing_token_expiry_seconds is required"); + } + } + + @JsonPOJOBuilder + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) + public static class Builder { + @JsonProperty + private Integer identityTokenExpiresAfterSeconds; + @JsonProperty + private Integer refreshTokenExpiresAfterSeconds; + @JsonProperty + private Integer refreshIdentityTokenAfterSeconds; + @JsonProperty + private Integer sharingTokenExpirySeconds; + @JsonProperty + private Integer maxBidstreamLifetimeSeconds; + @JsonProperty + private Integer maxSharingLifetimeSeconds; + + public Builder withIdentityTokenExpiresAfterSeconds(Integer identityTokenExpiresAfterSeconds) { + this.identityTokenExpiresAfterSeconds = identityTokenExpiresAfterSeconds; + return this; + } + + public Builder withRefreshTokenExpiresAfterSeconds(Integer refreshTokenExpiresAfterSeconds) { + this.refreshTokenExpiresAfterSeconds = refreshTokenExpiresAfterSeconds; + return this; + } + + public Builder withRefreshIdentityTokenAfterSeconds(Integer refreshIdentityTokenAfterSeconds) { + this.refreshIdentityTokenAfterSeconds = refreshIdentityTokenAfterSeconds; + return this; + } + + public Builder withSharingTokenExpirySeconds(Integer sharingTokenExpirySeconds) { + this.sharingTokenExpirySeconds = sharingTokenExpirySeconds; + return this; + } + + public Builder withMaxBidstreamLifetimeSeconds(Integer maxBidstreamLifetimeSeconds) { + this.maxBidstreamLifetimeSeconds = maxBidstreamLifetimeSeconds; + return this; + } + + public Builder withMaxSharingLifetimeSeconds(Integer maxSharingLifetimeSeconds) { + this.maxSharingLifetimeSeconds = maxSharingLifetimeSeconds; + return this; + } + + public RuntimeConfig build() { + return new RuntimeConfig(this); + } + } +} diff --git a/src/main/java/com/uid2/operator/store/RuntimeConfigStore.java b/src/main/java/com/uid2/operator/store/RuntimeConfigStore.java new file mode 100644 index 000000000..728b07572 --- /dev/null +++ b/src/main/java/com/uid2/operator/store/RuntimeConfigStore.java @@ -0,0 +1,65 @@ +package com.uid2.operator.store; + +import com.uid2.shared.Utils; +import com.uid2.shared.cloud.DownloadCloudStorage; +import com.uid2.shared.store.reader.IMetadataVersionedStore; +import io.vertx.core.json.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.InputStream; +import java.util.concurrent.atomic.AtomicReference; + +public class RuntimeConfigStore implements IConfigStore, IMetadataVersionedStore { + private static final Logger logger = LoggerFactory.getLogger(RuntimeConfigStore.class); + private final DownloadCloudStorage fileStreamProvider; + private final String configMetadataPath; + private final AtomicReference config = new AtomicReference<>(); + + public RuntimeConfigStore(DownloadCloudStorage fileStreamProvider, String configMetadataPath) { + this.fileStreamProvider = fileStreamProvider; + this.configMetadataPath = configMetadataPath; + } + + @Override + public JsonObject getMetadata() throws Exception { + try (InputStream s = this.fileStreamProvider.download(configMetadataPath)) { + return Utils.toJsonObject(s); + } + } + + @Override + public long getVersion(JsonObject metadata) { + return metadata.getLong("version"); + } + + @Override + public long loadContent(JsonObject metadata) throws Exception { + if (metadata == null) { + throw new RuntimeException("Metadata is null"); + } + + // The config is returned as part of the metadata itself. + JsonObject runtimeConfig = metadata.getJsonObject("runtime_config"); + + logger.info("Received new config {}", runtimeConfig == null ? null : runtimeConfig.toString()); + if (runtimeConfig == null) { + throw new RuntimeException("Runtime config is null"); + } + + RuntimeConfig newRuntimeConfig = runtimeConfig.mapTo(RuntimeConfig.class); + this.config.set(newRuntimeConfig); + logger.info("Successfully updated runtime config"); + return 1; + } + + @Override + public void loadContent() throws Exception { + this.loadContent(this.getMetadata()); + } + + @Override + public RuntimeConfig getConfig() { + return this.config.get(); + } +} diff --git a/src/main/java/com/uid2/operator/util/HttpMediaType.java b/src/main/java/com/uid2/operator/util/HttpMediaType.java new file mode 100644 index 000000000..d632b0836 --- /dev/null +++ b/src/main/java/com/uid2/operator/util/HttpMediaType.java @@ -0,0 +1,17 @@ +package com.uid2.operator.util; + +public enum HttpMediaType { + TEXT_PLAIN("text/plain"), + APPLICATION_JSON("application/json"), + APPLICATION_OCTET_STREAM("application/octet-stream"); + + private final String type; + + HttpMediaType(String type) { + this.type = type; + } + + public String getType() { + return type; + } +} \ No newline at end of file diff --git a/src/main/java/com/uid2/operator/util/PrivacyBits.java b/src/main/java/com/uid2/operator/util/PrivacyBits.java index f8b7edf6a..0df69d7fd 100644 --- a/src/main/java/com/uid2/operator/util/PrivacyBits.java +++ b/src/main/java/com/uid2/operator/util/PrivacyBits.java @@ -3,6 +3,9 @@ public class PrivacyBits { + // For historical reason this bit is set + public static final PrivacyBits DEFAULT = PrivacyBits.fromInt(1); + private static final int BIT_LEGACY = 0; private static final int BIT_CSTG = 1; private static final int BIT_CSTG_OPTOUT = 2; @@ -16,6 +19,24 @@ public class PrivacyBits { public PrivacyBits() { } + public PrivacyBits(PrivacyBits pb) { + bits = pb.bits; + } + + @Override + public boolean equals(Object obj) { + if (obj == null || !obj.getClass().equals(this.getClass())) { + return false; + } + PrivacyBits other = (PrivacyBits)obj; + return this.bits == other.bits; + } + + @Override + public int hashCode() { + return this.bits; + } + public PrivacyBits(int bits) { this.bits = bits; } @@ -37,6 +58,9 @@ public boolean isClientSideTokenOptedOut() { public void setLegacyBit() { setBit(BIT_LEGACY);//unknown why this bit is set in https://github.com/IABTechLab/uid2-operator/blob/dbab58346e367c9d4122ad541ff9632dc37bd410/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java#L534 } + public boolean isLegacyBitSet() { + return isBitSet(BIT_LEGACY); + } private void setBit(int position) { bits |= (1 << position); diff --git a/src/main/java/com/uid2/operator/util/RoutingContextUtil.java b/src/main/java/com/uid2/operator/util/RoutingContextUtil.java new file mode 100644 index 000000000..e6f69e51a --- /dev/null +++ b/src/main/java/com/uid2/operator/util/RoutingContextUtil.java @@ -0,0 +1,65 @@ +package com.uid2.operator.util; + +import com.uid2.shared.auth.ClientKey; +import com.uid2.shared.auth.IAuthorizable; +import com.uid2.shared.auth.IAuthorizableProvider; +import com.uid2.shared.middleware.AuthMiddleware; +import io.vertx.ext.web.RoutingContext; + +import java.net.URI; +import java.net.URISyntaxException; + +public final class RoutingContextUtil { + private static final String BEARER_TOKEN_PREFIX = "bearer "; + private static final String UNKNOWN = "unknown"; + + private RoutingContextUtil() { + } + + public static String getApiContact(RoutingContext rc, IAuthorizableProvider authKeyStore) { + try { + final String authHeaderValue = rc.request().getHeader("Authorization"); + final String authKey = extractBearerToken(authHeaderValue); + final IAuthorizable profile = authKeyStore.get(authKey); + String apiContact = profile.getContact(); + return apiContact == null ? UNKNOWN : apiContact; + } catch (Exception ex) { + return UNKNOWN; + } + } + + public static Integer getSiteId(RoutingContext rc) { + return AuthMiddleware.getAuthClient(ClientKey.class, rc).getSiteId(); + } + + public static String getPath(RoutingContext rc) { + try { + // If the current route is a known path, extract the full path from the request URI + if (rc.currentRoute().getPath() != null) { + return new URI(rc.request().absoluteURI()).getPath(); + } + } catch (NullPointerException | URISyntaxException ex) { + // RoutingContextImplBase has a bug: context.currentRoute() throws with NullPointerException when called from bodyEndHandler for StaticHandlerImpl.sendFile() + } + + return UNKNOWN; + } + + public static String extractBearerToken(final String headerValue) { + if (headerValue == null) { + return null; + } + + final String v = headerValue.trim(); + if (v.length() < BEARER_TOKEN_PREFIX.length()) { + return null; + } + + final String givenPrefix = v.substring(0, BEARER_TOKEN_PREFIX.length()); + + if (!BEARER_TOKEN_PREFIX.equalsIgnoreCase(givenPrefix)) { + return null; + } + return v.substring(BEARER_TOKEN_PREFIX.length()); + } +} diff --git a/src/main/java/com/uid2/operator/util/Tuple.java b/src/main/java/com/uid2/operator/util/Tuple.java index 491413de4..f2cf39ff0 100644 --- a/src/main/java/com/uid2/operator/util/Tuple.java +++ b/src/main/java/com/uid2/operator/util/Tuple.java @@ -1,13 +1,15 @@ package com.uid2.operator.util; +import java.util.Objects; + public class Tuple { public static class Tuple2 { private final T1 item1; private final T2 item2; public Tuple2(T1 item1, T2 item2) { - assert item1 != null; - assert item2 != null; + Objects.requireNonNull(item1); + Objects.requireNonNull(item2); this.item1 = item1; this.item2 = item2; @@ -34,9 +36,9 @@ public static class Tuple3 { private final T3 item3; public Tuple3(T1 item1, T2 item2, T3 item3) { - assert item1 != null; - assert item2 != null; - assert item3 != null; + Objects.requireNonNull(item1); + Objects.requireNonNull(item2); + Objects.requireNonNull(item3); this.item1 = item1; this.item2 = item2; diff --git a/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java b/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java new file mode 100644 index 000000000..7378022e0 --- /dev/null +++ b/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java @@ -0,0 +1,66 @@ +package com.uid2.operator.vertx; + +import com.uid2.operator.util.RoutingContextUtil; +import com.uid2.operator.util.Tuple; +import com.uid2.shared.Const; +import com.uid2.shared.auth.IAuthorizableProvider; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Metrics; +import io.vertx.core.Handler; +import io.vertx.ext.web.RoutingContext; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class ClientVersionCapturingHandler implements Handler { + private static final Map, Counter> CLIENT_VERSION_COUNTERS = new HashMap<>(); + private static final Set VERSIONS = new HashSet<>(); + + private final IAuthorizableProvider authKeyStore; + + public ClientVersionCapturingHandler(String dir, String whitelistGlob, IAuthorizableProvider authKeyStore) throws IOException { + this.authKeyStore = authKeyStore; + + try (final DirectoryStream dirStream = Files.newDirectoryStream(Paths.get(dir), whitelistGlob)) { + dirStream.forEach(path -> { + final String version = getFileNameWithoutExtension(path); + VERSIONS.add(version); + }); + } + } + + @Override + public void handle(RoutingContext rc) { + String clientVersion = rc.request().headers().get(Const.Http.ClientVersionHeader); + if (clientVersion == null) { + clientVersion = !rc.queryParam("client").isEmpty() ? rc.queryParam("client").getFirst() : null; + } + + String apiContact = RoutingContextUtil.getApiContact(rc, authKeyStore); + String path = RoutingContextUtil.getPath(rc); + + if (clientVersion != null && VERSIONS.contains(clientVersion)) { + CLIENT_VERSION_COUNTERS.computeIfAbsent( + new Tuple.Tuple2<>(apiContact, clientVersion), + tuple -> Counter + .builder("uid2_client_sdk_versions_total") + .description("counter for how many http requests are processed per each client sdk version") + .tags("site_id", "unknown", "api_contact", tuple.getItem1(), "client_version", tuple.getItem2(), "path", path) + .register(Metrics.globalRegistry) + ).increment(); + } + rc.next(); + } + + private static String getFileNameWithoutExtension(Path path) { + final String fileName = path.getFileName().toString(); + return fileName.indexOf(".") > 0 ? fileName.substring(0, fileName.lastIndexOf(".")) : fileName; + } +} diff --git a/src/main/java/com/uid2/operator/vertx/Endpoints.java b/src/main/java/com/uid2/operator/vertx/Endpoints.java index 2643f943b..be8fa0b05 100644 --- a/src/main/java/com/uid2/operator/vertx/Endpoints.java +++ b/src/main/java/com/uid2/operator/vertx/Endpoints.java @@ -6,21 +6,6 @@ public enum Endpoints { OPS_HEALTHCHECK("/ops/healthcheck"), - - V0_KEY_LATEST("/key/latest"), - V0_TOKEN_GENERATE("/token/generate"), - V0_TOKEN_REFRESH("/token/refresh"), - V0_TOKEN_VALIDATE("/token/validate"), - V0_IDENTITY_MAP("/identity/map"), - V0_TOKEN_LOGOUT("/token/logout"), - - V1_TOKEN_GENERATE("/v1/token/generate"), - V1_TOKEN_VALIDATE("/v1/token/validate"), - V1_TOKEN_REFRESH("/v1/token/refresh"), - V1_IDENTITY_BUCKETS("/v1/identity/buckets"), - V1_IDENTITY_MAP("/v1/identity/map"), - V1_KEY_LATEST("/v1/key/latest"), - V2_TOKEN_GENERATE("/v2/token/generate"), V2_TOKEN_REFRESH("/v2/token/refresh"), V2_TOKEN_VALIDATE("/v2/token/validate"), @@ -33,6 +18,8 @@ public enum Endpoints { V2_OPTOUT_STATUS("/v2/optout/status"), V2_TOKEN_CLIENTGENERATE("/v2/token/client-generate"), + V3_IDENTITY_MAP("/v3/identity/map"), + EUID_SDK_1_0_0("/static/js/euid-sdk-1.0.0.js"), OPENID_SDK_1_0("/static/js/openid-sdk-1.0.js"), UID2_ESP_0_0_1A("/static/js/uid2-esp-0.0.1a.js"), diff --git a/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java b/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java index 113c14d3e..84075fb03 100644 --- a/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java +++ b/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java @@ -1,6 +1,8 @@ package com.uid2.operator.vertx; import com.uid2.operator.service.ShutdownService; +import com.uid2.shared.attest.AttestationResponseCode; +import lombok.extern.java.Log; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.utils.Pair; @@ -52,12 +54,12 @@ public void logSaltFailureAtInterval() { } } - public void handleAttestResponse(Pair response) { - if (response.left() == 401) { - LOGGER.error("core attestation failed with 401, shutting down operator, core response: " + response.right()); + public void handleAttestResponse(Pair response) { + if (response.left() == AttestationResponseCode.AttestationFailure) { + LOGGER.error("core attestation failed with AttestationFailure, shutting down operator, core response: {}", response.right()); this.shutdownService.Shutdown(1); } - if (response.left() == 200) { + if (response.left() == AttestationResponseCode.Success) { attestFailureStartTime.set(null); } else { Instant t = attestFailureStartTime.get(); diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 6a012222f..fb4066681 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -1,10 +1,13 @@ package com.uid2.operator.vertx; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.uid2.operator.Const; import com.uid2.operator.model.*; -import com.uid2.operator.model.IdentityResponse; -import com.uid2.operator.model.IdentityScope; -import com.uid2.operator.model.userIdentity.HashedDiiIdentity; +import com.uid2.operator.model.TokenGenerateResponse; +import com.uid2.operator.model.identities.IdentityScope; +import com.uid2.operator.model.identities.DiiType; +import com.uid2.operator.model.identities.HashedDii; import com.uid2.operator.monitoring.IStatsCollectorQueue; import com.uid2.operator.monitoring.StatsCollectorHandler; import com.uid2.operator.monitoring.TokenResponseStatsCollector; @@ -14,11 +17,12 @@ import com.uid2.operator.privacy.tcf.TransparentConsentSpecialFeature; import com.uid2.operator.service.*; import com.uid2.operator.store.*; -import com.uid2.operator.util.DomainNameCheckUtil; -import com.uid2.operator.util.PrivacyBits; -import com.uid2.operator.util.Tuple; +import com.uid2.operator.store.IConfigStore; +import com.uid2.operator.util.*; import com.uid2.shared.Const.Data; import com.uid2.shared.Utils; +import com.uid2.shared.audit.Audit; +import com.uid2.shared.audit.UidInstanceIdProvider; import com.uid2.shared.auth.*; import com.uid2.shared.encryption.AesGcm; import com.uid2.shared.health.HealthComponent; @@ -29,7 +33,8 @@ import com.uid2.shared.store.ACLMode.MissingAclMode; import com.uid2.shared.store.IClientKeyProvider; import com.uid2.shared.store.IClientSideKeypairStore; -import com.uid2.shared.store.ISaltProvider; +import com.uid2.shared.store.salt.ISaltProvider; +import com.uid2.shared.util.Mapper; import com.uid2.shared.vertx.RequestCapturingHandler; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.DistributionSummary; @@ -42,6 +47,7 @@ import io.vertx.core.Promise; import io.vertx.core.buffer.Buffer; import io.vertx.core.http.HttpHeaders; +import io.vertx.core.http.HttpServerOptions; import io.vertx.core.http.HttpServerResponse; import io.vertx.core.json.DecodeException; import io.vertx.core.json.JsonArray; @@ -68,8 +74,8 @@ import java.util.*; import java.util.function.Supplier; import java.util.stream.Collectors; - -import static com.uid2.operator.IdentityConst.*; +import static com.uid2.operator.model.identities.IdentityConst.*; +import static com.uid2.operator.Const.Config.*; import static com.uid2.operator.service.ResponseUtil.*; import static com.uid2.operator.vertx.Endpoints.*; @@ -81,32 +87,39 @@ public class UIDOperatorVerticle extends AbstractVerticle { * is slightly longer than it should be. When validating token lifetimes, we add a small buffer to account for this. */ public static final Duration TOKEN_LIFETIME_TOLERANCE = Duration.ofSeconds(10); - private static final DateTimeFormatter APIDateTimeFormatter = DateTimeFormatter.ISO_LOCAL_DATE_TIME.withZone(ZoneId.of("UTC")); + private static final long SECOND_IN_MILLIS = 1000; + // Use a formatter that always prints three-digit millisecond precision (e.g. 2024-07-02T14:15:16.000) + private static final DateTimeFormatter API_DATE_TIME_FORMATTER = + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS").withZone(ZoneOffset.UTC); private static final String REQUEST = "request"; private final HealthComponent healthComponent = HealthManager.instance.registerComponent("http-server"); private final Cipher aesGcm; - private final JsonObject config; + private final IConfigStore configStore; private final boolean clientSideTokenGenerate; private final AuthMiddleware auth; private final ISiteStore siteProvider; private final IClientSideKeypairStore clientSideKeypairProvider; - private final ITokenEncoder encoder; + private final EncryptedTokenEncoder encoder; private final ISaltProvider saltProvider; private final IOptOutStore optOutStore; private final IClientKeyProvider clientKeyProvider; private final Clock clock; + private final boolean identityV3Enabled; + private final boolean disableOptoutToken; + private final UidInstanceIdProvider uidInstanceIdProvider; protected IUIDOperatorService idService; private final Map _identityMapMetricSummaries = new HashMap<>(); private final Map, DistributionSummary> _refreshDurationMetricSummaries = new HashMap<>(); private final Map, Counter> _advertisingTokenExpiryStatus = new HashMap<>(); private final Map, Counter> _tokenGeneratePolicyCounters = new HashMap<>(); + private final Map _tokenGenerateTCFUsage = new HashMap<>(); private final Map> _identityMapUnmappedIdentifiers = new HashMap<>(); private final Map _identityMapRequestWithUnmapped = new HashMap<>(); private final Map optOutStatusCounters = new HashMap<>(); private final IdentityScope identityScope; - private final V2PayloadHandler v2PayloadHandler; + private final V2PayloadHandler encryptedPayloadHandler; private final boolean phoneSupport; private final int tcfVendorId; private final IStatsCollectorQueue _statsCollectorQueue; @@ -117,9 +130,7 @@ public class UIDOperatorVerticle extends AbstractVerticle { public final static int MASTER_KEYSET_ID_FOR_SDKS = 9999999; //this is because SDKs have an issue where they assume keyset ids are always positive; that will be fixed. public final static long OPT_OUT_CHECK_CUTOFF_DATE = Instant.parse("2023-09-01T00:00:00.00Z").getEpochSecond(); private final Handler saltRetrievalResponseHandler; - private final int maxBidstreamLifetimeSeconds; private final int allowClockSkewSeconds; - protected int maxSharingLifetimeSeconds; protected Map> siteIdToInvalidOriginsAndAppNames = new HashMap<>(); protected boolean keySharingEndpointProvideAppNames; protected Instant lastInvalidOriginProcessTime = Instant.now(); @@ -127,12 +138,22 @@ public class UIDOperatorVerticle extends AbstractVerticle { private final int optOutStatusMaxRequestSize; private final boolean optOutStatusApiEnabled; + private final static ObjectMapper OBJECT_MAPPER = Mapper.getApiInstance(); + //"Android" is from https://github.com/IABTechLab/uid2-android-sdk/blob/ff93ebf597f5de7d440a84f7015a334ba4138ede/sdk/src/main/java/com/uid2/UID2Client.kt#L46 //"ios"/"tvos" is from https://github.com/IABTechLab/uid2-ios-sdk/blob/91c290d29a7093cfc209eca493d1fee80c17e16a/Sources/UID2/UID2Client.swift#L36-L38 private final static List SUPPORTED_IN_APP = Arrays.asList("Android", "ios", "tvos"); + + private static final String ERROR_INVALID_INPUT_WITH_PHONE_SUPPORT = "Required Parameter Missing: exactly one of [email, email_hash, phone, phone_hash] must be specified"; + private static final String ERROR_INVALID_INPUT_EMAIL_MISSING = "Required Parameter Missing: exactly one of email or email_hash must be specified"; + private static final String ERROR_INVALID_MIXED_INPUT_WITH_PHONE_SUPPORT = "Required Parameter Missing: one or more of [email, email_hash, phone, phone_hash] must be specified"; + private static final String ERROR_INVALID_MIXED_INPUT_EMAIL_MISSING = "Required Parameter Missing: one or more of [email, email_hash] must be specified"; + private static final String ERROR_INVALID_INPUT_EMAIL_TWICE = "Only one of email or email_hash can be specified"; + private static final String RC_CONFIG_KEY = "remote-config"; public final static String ORIGIN_HEADER = "Origin"; - public UIDOperatorVerticle(JsonObject config, + public UIDOperatorVerticle(IConfigStore configStore, + JsonObject config, boolean clientSideTokenGenerate, ISiteStore siteProvider, IClientKeyProvider clientKeyProvider, @@ -143,7 +164,8 @@ public UIDOperatorVerticle(JsonObject config, Clock clock, IStatsCollectorQueue statsCollectorQueue, SecureLinkValidatorService secureLinkValidatorService, - Handler saltRetrievalResponseHandler) { + Handler saltRetrievalResponseHandler, + UidInstanceIdProvider uidInstanceIdProvider) { this.keyManager = keyManager; this.secureLinkValidatorService = secureLinkValidatorService; try { @@ -151,7 +173,7 @@ public UIDOperatorVerticle(JsonObject config, } catch (NoSuchAlgorithmException | NoSuchPaddingException e) { throw new RuntimeException(e); } - this.config = config; + this.configStore = configStore; this.clientSideTokenGenerate = clientSideTokenGenerate; this.healthComponent.setHealthStatus(false, "not started"); this.auth = new AuthMiddleware(clientKeyProvider); @@ -162,7 +184,7 @@ public UIDOperatorVerticle(JsonObject config, this.optOutStore = optOutStore; this.clock = clock; this.identityScope = IdentityScope.fromString(config.getString("identity_scope", "uid2")); - this.v2PayloadHandler = new V2PayloadHandler(keyManager, config.getBoolean("enable_v2_encryption", true), this.identityScope, siteProvider); + this.encryptedPayloadHandler = new V2PayloadHandler(keyManager, config.getBoolean("enable_v2_encryption", true), this.identityScope, siteProvider); this.phoneSupport = config.getBoolean("enable_phone_support", true); this.tcfVendorId = config.getInteger("tcf_vendor_id", 21); this.cstgDoDomainNameCheck = config.getBoolean("client_side_token_generate_domain_name_check_enabled", true); @@ -170,35 +192,32 @@ public UIDOperatorVerticle(JsonObject config, this._statsCollectorQueue = statsCollectorQueue; this.clientKeyProvider = clientKeyProvider; this.clientSideTokenGenerateLogInvalidHttpOrigin = config.getBoolean("client_side_token_generate_log_invalid_http_origins", false); - final Integer identityTokenExpiresAfterSeconds = config.getInteger(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); - this.maxBidstreamLifetimeSeconds = config.getInteger(Const.Config.MaxBidstreamLifetimeSecondsProp, identityTokenExpiresAfterSeconds); - if (this.maxBidstreamLifetimeSeconds < identityTokenExpiresAfterSeconds) { - LOGGER.error("Max bidstream lifetime seconds ({} seconds) is less than identity token lifetime ({} seconds)", maxBidstreamLifetimeSeconds, identityTokenExpiresAfterSeconds); - throw new RuntimeException("Max bidstream lifetime seconds is less than identity token lifetime seconds"); - } this.allowClockSkewSeconds = config.getInteger(Const.Config.AllowClockSkewSecondsProp, 1800); - this.maxSharingLifetimeSeconds = config.getInteger(Const.Config.MaxSharingLifetimeProp, config.getInteger(Const.Config.SharingTokenExpiryProp)); this.saltRetrievalResponseHandler = saltRetrievalResponseHandler; this.optOutStatusApiEnabled = config.getBoolean(Const.Config.OptOutStatusApiEnabled, true); this.optOutStatusMaxRequestSize = config.getInteger(Const.Config.OptOutStatusMaxRequestSize, 5000); + this.identityV3Enabled = config.getBoolean(IdentityV3Prop, false); + this.disableOptoutToken = config.getBoolean(DisableOptoutTokenProp, false); + this.uidInstanceIdProvider = uidInstanceIdProvider; } @Override public void start(Promise startPromise) throws Exception { this.healthComponent.setHealthStatus(false, "still starting"); this.idService = new UIDOperatorService( - this.config, this.optOutStore, this.saltProvider, this.encoder, this.clock, this.identityScope, - this.saltRetrievalResponseHandler + this.saltRetrievalResponseHandler, + this.identityV3Enabled, + this.uidInstanceIdProvider ); final Router router = createRoutesSetup(); final int port = Const.Port.ServicePortForOperator + Utils.getPortOffset(); - vertx.createHttpServer() + vertx.createHttpServer(new HttpServerOptions().setMaxFormBufferedBytes((int) MAX_REQUEST_BODY_SIZE)) .requestHandler(router) .listen(port, result -> { if (result.succeeded()) { @@ -218,7 +237,8 @@ private Router createRoutesSetup() throws IOException { final Router router = Router.router(vertx); router.allowForward(AllowForwardHeaders.X_FORWARD); - router.route().handler(new RequestCapturingHandler()); + router.route().handler(new RequestCapturingHandler(siteProvider)); + router.route().handler(new ClientVersionCapturingHandler("static/js", "*.js", clientKeyProvider)); router.route().handler(CorsHandler.create() .addRelativeOrigin(".*.") .allowedMethod(io.vertx.core.http.HttpMethod.GET) @@ -232,75 +252,55 @@ private Router createRoutesSetup() throws IOException { .allowedHeader("Content-Type")); router.route().handler(new StatsCollectorHandler(_statsCollectorQueue, vertx)); router.route("/static/*").handler(StaticHandler.create("static")); + router.route().handler(ctx -> { + RuntimeConfig curConfig = configStore.getConfig(); + ctx.put(RC_CONFIG_KEY, curConfig); + ctx.next(); + }); router.route().failureHandler(new GenericFailureHandler()); final BodyHandler bodyHandler = BodyHandler.create().setHandleFileUploads(false).setBodyLimit(MAX_REQUEST_BODY_SIZE); - setupV2Routes(router, bodyHandler); + setUpEncryptedRoutes(router, bodyHandler); // Static and health check router.get(OPS_HEALTHCHECK.toString()).handler(this::handleHealthCheck); - if (this.config.getBoolean(Const.Config.AllowLegacyAPIProp, true)) { - // V1 APIs - router.get(V1_TOKEN_GENERATE.toString()).handler(auth.handleV1(this::handleTokenGenerateV1, Role.GENERATOR)); - router.get(V1_TOKEN_VALIDATE.toString()).handler(this::handleTokenValidateV1); - router.get(V1_TOKEN_REFRESH.toString()).handler(auth.handleWithOptionalAuth(this::handleTokenRefreshV1)); - router.get(V1_IDENTITY_BUCKETS.toString()).handler(auth.handle(this::handleBucketsV1, Role.MAPPER)); - router.get(V1_IDENTITY_MAP.toString()).handler(auth.handle(this::handleIdentityMapV1, Role.MAPPER)); - router.post(V1_IDENTITY_MAP.toString()).handler(bodyHandler).handler(auth.handle(this::handleIdentityMapBatchV1, Role.MAPPER)); - router.get(V1_KEY_LATEST.toString()).handler(auth.handle(this::handleKeysRequestV1, Role.ID_READER)); - - // Deprecated APIs - router.get(V0_KEY_LATEST.toString()).handler(auth.handle(this::handleKeysRequest, Role.ID_READER)); - router.get(V0_TOKEN_GENERATE.toString()).handler(auth.handle(this::handleTokenGenerate, Role.GENERATOR)); - router.get(V0_TOKEN_REFRESH.toString()).handler(this::handleTokenRefresh); - router.get(V0_TOKEN_VALIDATE.toString()).handler(this::handleValidate); - router.get(V0_IDENTITY_MAP.toString()).handler(auth.handle(this::handleIdentityMap, Role.MAPPER)); - router.post(V0_IDENTITY_MAP.toString()).handler(bodyHandler).handler(auth.handle(this::handleIdentityMapBatch, Role.MAPPER)); - - // internal API to handle user optout of UID - router.get(V0_TOKEN_LOGOUT.toString()).handler(auth.handle(this::handleLogoutAsync, Role.OPTOUT)); - - // only uncomment to do local testing - //router.get("/internal/optout/get").handler(auth.loopbackOnly(this::handleOptOutGet)); - - } - return router; } - private void setupV2Routes(Router mainRouter, BodyHandler bodyHandler) { + private void setUpEncryptedRoutes(Router mainRouter, BodyHandler bodyHandler) { mainRouter.post(V2_TOKEN_GENERATE.toString()).handler(bodyHandler).handler(auth.handleV1( - rc -> v2PayloadHandler.handleTokenGenerate(rc, this::handleTokenGenerateV2), Role.GENERATOR)); + rc -> encryptedPayloadHandler.handleTokenGenerate(rc, this::handleTokenGenerateV2), Role.GENERATOR)); mainRouter.post(V2_TOKEN_REFRESH.toString()).handler(bodyHandler).handler(auth.handleWithOptionalAuth( - rc -> v2PayloadHandler.handleTokenRefresh(rc, this::handleTokenRefreshV2))); + rc -> encryptedPayloadHandler.handleTokenRefresh(rc, this::handleTokenRefreshV2))); mainRouter.post(V2_TOKEN_VALIDATE.toString()).handler(bodyHandler).handler(auth.handleV1( - rc -> v2PayloadHandler.handle(rc, this::handleTokenValidateV2), Role.GENERATOR)); + rc -> encryptedPayloadHandler.handle(rc, this::handleTokenValidateV2), Role.GENERATOR)); mainRouter.post(V2_IDENTITY_BUCKETS.toString()).handler(bodyHandler).handler(auth.handleV1( - rc -> v2PayloadHandler.handle(rc, this::handleBucketsV2), Role.MAPPER)); + rc -> encryptedPayloadHandler.handle(rc, this::handleBucketsV2), Role.MAPPER)); mainRouter.post(V2_IDENTITY_MAP.toString()).handler(bodyHandler).handler(auth.handleV1( - rc -> v2PayloadHandler.handle(rc, this::handleIdentityMapV2), Role.MAPPER)); + rc -> encryptedPayloadHandler.handle(rc, this::handleIdentityMapV2), Role.MAPPER)); mainRouter.post(V2_KEY_LATEST.toString()).handler(bodyHandler).handler(auth.handleV1( - rc -> v2PayloadHandler.handle(rc, this::handleKeysRequestV2), Role.ID_READER)); + rc -> encryptedPayloadHandler.handle(rc, this::handleKeysRequestV2), Role.ID_READER)); mainRouter.post(V2_KEY_SHARING.toString()).handler(bodyHandler).handler(auth.handleV1( - rc -> v2PayloadHandler.handle(rc, this::handleKeysSharing), Role.SHARER, Role.ID_READER)); + rc -> encryptedPayloadHandler.handle(rc, this::handleKeysSharing), Role.SHARER, Role.ID_READER)); mainRouter.post(V2_KEY_BIDSTREAM.toString()).handler(bodyHandler).handler(auth.handleV1( - rc -> v2PayloadHandler.handle(rc, this::handleKeysBidstream), Role.ID_READER)); - // internal API to handle user optout of UID + rc -> encryptedPayloadHandler.handle(rc, this::handleKeysBidstream), Role.ID_READER)); mainRouter.post(V2_TOKEN_LOGOUT.toString()).handler(bodyHandler).handler(auth.handleV1( - rc -> v2PayloadHandler.handleAsync(rc, this::handleLogoutAsyncV2), Role.OPTOUT)); + rc -> encryptedPayloadHandler.handleAsync(rc, this::handleLogoutAsyncV2), Role.OPTOUT)); if (this.optOutStatusApiEnabled) { mainRouter.post(V2_OPTOUT_STATUS.toString()).handler(bodyHandler).handler(auth.handleV1( - rc -> v2PayloadHandler.handle(rc, this::handleOptoutStatus), + rc -> encryptedPayloadHandler.handle(rc, this::handleOptoutStatus), Role.MAPPER, Role.SHARER, Role.ID_READER)); } if (this.clientSideTokenGenerate) mainRouter.post(V2_TOKEN_CLIENTGENERATE.toString()).handler(bodyHandler).handler(this::handleClientSideTokenGenerate); - } + mainRouter.post(V3_IDENTITY_MAP.toString()).handler(bodyHandler).handler(auth.handleV1( + rc -> encryptedPayloadHandler.handle(rc, this::handleIdentityMapV3), Role.MAPPER)); + } private void handleClientSideTokenGenerate(RoutingContext rc) { try { @@ -310,6 +310,9 @@ private void handleClientSideTokenGenerate(RoutingContext rc) { } } + private RuntimeConfig getConfigFromRc(RoutingContext rc) { + return rc.get(RC_CONFIG_KEY); + } private Set getDomainNameListForClientSideTokenGenerate(ClientSideKeypair keypair) { Site s = siteProvider.getSite(keypair.getSiteId()); @@ -328,8 +331,22 @@ private Set getAppNames(ClientSideKeypair keypair) { return site.getAppNames(); } + private void logIfApiKey(String key) { + ClientKey clientKey = this.clientKeyProvider.getClientKey(key); + if (clientKey != null) { + LOGGER.error("Client side key is an api key with api_key_id={} for site_id={}", clientKey.getKeyId(), clientKey.getSiteId()); + } + } + private void handleClientSideTokenGenerateImpl(RoutingContext rc) throws NoSuchAlgorithmException, InvalidKeyException { final JsonObject body; + + RuntimeConfig config = this.getConfigFromRc(rc); + + Duration refreshIdentityAfter = Duration.ofSeconds(config.getRefreshIdentityTokenAfterSeconds()); + Duration refreshExpiresAfter = Duration.ofSeconds(config.getRefreshTokenExpiresAfterSeconds()); + Duration identityExpiresAfter = Duration.ofSeconds(config.getIdentityTokenExpiresAfterSeconds()); + TokenResponseStatsCollector.PlatformType platformType = TokenResponseStatsCollector.PlatformType.Other; try { body = rc.body().asJsonObject(); @@ -350,14 +367,16 @@ private void handleClientSideTokenGenerateImpl(RoutingContext rc) throws NoSuchA final ClientSideKeypair clientSideKeypair = this.clientSideKeypairProvider.getSnapshot().getKeypair(request.getSubscriptionId()); if (clientSideKeypair == null) { + logIfApiKey(request.getSubscriptionId()); SendClientErrorResponseAndRecordStats(ResponseStatus.ClientError, 400, rc, "bad subscription_id", null, TokenResponseStatsCollector.Endpoint.ClientSideTokenGenerateV2, TokenResponseStatsCollector.ResponseStatus.BadSubscriptionId, siteProvider, platformType); return; } + rc.put(com.uid2.shared.Const.RoutingContextData.SiteId, clientSideKeypair.getSiteId()); if(clientSideKeypair.isDisabled()) { SendClientErrorResponseAndRecordStats(ResponseStatus.Unauthorized, 401, rc, "Unauthorized", - clientSideKeypair.getSiteId(), TokenResponseStatsCollector.Endpoint.ClientSideTokenGenerateV2, TokenResponseStatsCollector.ResponseStatus.Unauthorized, siteProvider, platformType); + clientSideKeypair.getSiteId(), TokenResponseStatsCollector.Endpoint.ClientSideTokenGenerateV2, TokenResponseStatsCollector.ResponseStatus.Unauthorized, siteProvider, platformType); return; } @@ -378,6 +397,7 @@ private void handleClientSideTokenGenerateImpl(RoutingContext rc) throws NoSuchA final X509EncodedKeySpec pkSpec = new X509EncodedKeySpec(clientPublicKeyBytes); clientPublicKey = kf.generatePublic(pkSpec); } catch (Exception e) { + logIfApiKey(request.getPublicKey()); SendClientErrorResponseAndRecordStats(ResponseStatus.ClientError, 400, rc, "bad public key", clientSideKeypair.getSiteId(), TokenResponseStatsCollector.Endpoint.ClientSideTokenGenerateV2, TokenResponseStatsCollector.ResponseStatus.BadPublicKey, siteProvider, platformType); return; } @@ -452,7 +472,7 @@ else if(emailHash != null) { input = InputUtil.normalizePhoneHash(phoneHash); } - if (!checkForInvalidTokenInput(input, rc)) { + if (!isTokenInputValid(input, rc)) { return; } @@ -460,13 +480,16 @@ else if(emailHash != null) { privacyBits.setLegacyBit(); privacyBits.setClientSideTokenGenerate(); - IdentityResponse identityResponse; + TokenGenerateResponse tokenGenerateResponse; try { - identityResponse = this.idService.generateIdentity( - new IdentityRequest( - new SourcePublisher(clientSideKeypair.getSiteId(), 0, 0), - input.toHashedDiiIdentity(this.identityScope, privacyBits.getAsInt(), Instant.now()), - OptoutCheckPolicy.RespectOptOut)); + tokenGenerateResponse = this.idService.generateIdentity( + new TokenGenerateRequest( + new SourcePublisher(clientSideKeypair.getSiteId()), + input.toHashedDii(this.identityScope), + OptoutCheckPolicy.RespectOptOut, privacyBits, Instant.now()), + refreshIdentityAfter, + refreshExpiresAfter, + identityExpiresAfter); } catch (KeyManager.NoActiveKeyException e){ SendServerErrorResponseAndRecordStats(rc, "No active encryption key available", clientSideKeypair.getSiteId(), TokenResponseStatsCollector.Endpoint.ClientSideTokenGenerateV2, TokenResponseStatsCollector.ResponseStatus.NoActiveKey, siteProvider, e, platformType); return; @@ -474,12 +497,12 @@ else if(emailHash != null) { JsonObject response; TokenResponseStatsCollector.ResponseStatus responseStatus = TokenResponseStatsCollector.ResponseStatus.Success; - if (identityResponse.isOptedOut()) { + if (tokenGenerateResponse.isOptedOut()) { response = ResponseUtil.SuccessNoBodyV2(ResponseStatus.OptOut); responseStatus = TokenResponseStatsCollector.ResponseStatus.OptOut; } else { //user not opted out and already generated valid identity token - response = ResponseUtil.SuccessV2(toJsonV1(identityResponse)); + response = ResponseUtil.SuccessV2(tokenGenerateResponse.toTokenGenerateResponseJson()); } //if returning an optout token or a successful identity token created originally if (responseStatus == TokenResponseStatsCollector.ResponseStatus.Success) { @@ -487,7 +510,7 @@ else if(emailHash != null) { } final byte[] encryptedResponse = AesGcm.encrypt(response.toBuffer().getBytes(), sharedSecret); rc.response().setStatusCode(200).end(Buffer.buffer(Unpooled.wrappedBuffer(Base64.getEncoder().encode(encryptedResponse)))); - recordTokenResponseStats(clientSideKeypair.getSiteId(), TokenResponseStatsCollector.Endpoint.ClientSideTokenGenerateV2, responseStatus, siteProvider, identityResponse.getAdvertisingTokenVersion(), platformType); + recordTokenResponseStats(clientSideKeypair.getSiteId(), TokenResponseStatsCollector.Endpoint.ClientSideTokenGenerateV2, responseStatus, siteProvider, tokenGenerateResponse.getAdvertisingTokenVersion(), platformType); } private boolean hasValidOriginOrAppName(RoutingContext rc, CstgRequest request, ClientSideKeypair keypair, TokenResponseStatsCollector.PlatformType platformType) { @@ -565,7 +588,7 @@ private void handleKeysRequestCommon(RoutingContext rc, Handler onSuc final ClientKey clientKey = AuthMiddleware.getAuthClient(ClientKey.class, rc); final int clientSiteId = clientKey.getSiteId(); if (!clientKey.hasValidSiteId()) { - ResponseUtil.Warning("invalid_client", 401, rc, "Unexpected client site id " + Integer.toString(clientSiteId)); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.InvalidClient, 401, rc, "Unexpected client site id " + Integer.toString(clientSiteId)); return; } @@ -573,15 +596,6 @@ private void handleKeysRequestCommon(RoutingContext rc, Handler onSuc onSuccess.handle(getAccessibleKeysAsJson(keys, clientKey)); } - public void handleKeysRequestV1(RoutingContext rc) { - try { - handleKeysRequestCommon(rc, keys -> ResponseUtil.Success(rc, keys)); - } catch (Exception e) { - LOGGER.error("Unknown error while handling keys request v1", e); - rc.fail(500); - } - } - public void handleKeysRequestV2(RoutingContext rc) { try { handleKeysRequestCommon(rc, keys -> ResponseUtil.SuccessV2(rc, keys)); @@ -591,20 +605,10 @@ public void handleKeysRequestV2(RoutingContext rc) { } } - public void handleKeysRequest(RoutingContext rc) { - try { - handleKeysRequestCommon(rc, keys -> sendJsonResponse(rc, keys)); - } catch (Exception e) { - LOGGER.error("Unknown error while handling keys request", e); - rc.fail(500); - } - } - - private String getSharingTokenExpirySeconds() { - return config.getString(Const.Config.SharingTokenExpiryProp); - } - public void handleKeysSharing(RoutingContext rc) { + RuntimeConfig config = this.getConfigFromRc(rc); + int sharingTokenExpirySeconds = config.getSharingTokenExpirySeconds(); + int maxSharingLifetimeSeconds = config.getMaxSharingLifetimeSeconds(); try { final ClientKey clientKey = AuthMiddleware.getAuthClient(ClientKey.class, rc); @@ -613,7 +617,7 @@ public void handleKeysSharing(RoutingContext rc) { Map keysetMap = keyManagerSnapshot.getAllKeysets(); final JsonObject resp = new JsonObject(); - addSharingHeaderFields(resp, keyManagerSnapshot, clientKey); + addSharingHeaderFields(resp, keyManagerSnapshot, clientKey, maxSharingLifetimeSeconds, sharingTokenExpirySeconds); final List accessibleKeys = getAccessibleKeys(keysetKeyStore, keyManagerSnapshot, clientKey); @@ -658,14 +662,19 @@ public void handleKeysBidstream(RoutingContext rc) { .collect(Collectors.toList()); final JsonObject resp = new JsonObject(); - addBidstreamHeaderFields(resp); + + RuntimeConfig config = this.getConfigFromRc(rc); + int maxBidstreamLifetimeSeconds = config.getMaxBidstreamLifetimeSeconds(); + + + addBidstreamHeaderFields(resp, maxBidstreamLifetimeSeconds); resp.put("keys", keysJson); addSites(resp, accessibleKeys, keysetMap); ResponseUtil.SuccessV2(rc, resp); } - private void addBidstreamHeaderFields(JsonObject resp) { + private void addBidstreamHeaderFields(JsonObject resp, int maxBidstreamLifetimeSeconds) { resp.put("max_bidstream_lifetime_seconds", maxBidstreamLifetimeSeconds + TOKEN_LIFETIME_TOLERANCE.toSeconds()); addIdentityScopeField(resp); addAllowClockSkewSecondsField(resp); @@ -703,7 +712,7 @@ private void addSites(JsonObject resp, List keys, Map tokenList = rc.queryParam("refresh_token"); - TokenResponseStatsCollector.PlatformType platformType = getPlatformType(rc); - Integer siteId = null; - if (tokenList == null || tokenList.size() == 0) { - SendClientErrorResponseAndRecordStats(ResponseStatus.ClientError, 400, rc, "Required Parameter Missing: refresh_token", siteId, TokenResponseStatsCollector.Endpoint.RefreshV1, TokenResponseStatsCollector.ResponseStatus.MissingParams, siteProvider, platformType); - return; - } - - String refreshToken = tokenList.get(0); - if (refreshToken.length() == V2RequestUtil.V2_REFRESH_PAYLOAD_LENGTH) { - // V2 token sent by V1 JSSDK. Decrypt and extract original refresh token - V2RequestUtil.V2Request v2req = V2RequestUtil.parseRefreshRequest(refreshToken, this.keyManager); - if (v2req.isValid()) { - refreshToken = (String) v2req.payload; - } else { - SendClientErrorResponseAndRecordStats(ResponseStatus.ClientError, 400, rc, v2req.errorMessage, siteId, TokenResponseStatsCollector.Endpoint.RefreshV1, TokenResponseStatsCollector.ResponseStatus.BadPayload, siteProvider, platformType); - return; - } - } - - try { - final RefreshResponse r = this.refreshIdentity(rc, refreshToken); - siteId = rc.get(Const.RoutingContextData.SiteId); - if (!r.isRefreshed()) { - if (r.isOptOut() || r.isDeprecated()) { - ResponseUtil.SuccessNoBody(ResponseStatus.OptOut, rc); - } else if (!AuthMiddleware.isAuthenticated(rc)) { - // unauthenticated clients get a generic error - ResponseUtil.Warning(ResponseStatus.GenericError, 400, rc, "Error refreshing token"); - } else if (r.isInvalidToken()) { - ResponseUtil.Warning(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented " + tokenList.get(0)); - } else if (r.isExpired()) { - ResponseUtil.Warning(ResponseStatus.ExpiredToken, 400, rc, "Expired Token presented"); - } else { - ResponseUtil.Error(ResponseStatus.UnknownError, 500, rc, "Unknown State"); - } - } else { - ResponseUtil.Success(rc, toJsonV1(r.getIdentityResponse())); - this.recordRefreshDurationStats(siteId, getApiContact(rc), r.getDurationSinceLastRefresh(), rc.request().headers().contains(ORIGIN_HEADER)); - } - - TokenResponseStatsCollector.recordRefresh(siteProvider, siteId, TokenResponseStatsCollector.Endpoint.RefreshV1, r, platformType); - } catch (Exception e) { - SendServerErrorResponseAndRecordStats(rc, "Unknown error while refreshing token", siteId, TokenResponseStatsCollector.Endpoint.RefreshV1, TokenResponseStatsCollector.ResponseStatus.Unknown, siteProvider, e, platformType); + private static final Map, Counter> CLIENT_VERSION_COUNTERS = new HashMap<>(); + private void recordOperatorServedSdkUsage(RoutingContext rc, Integer siteId, String apiContact, String clientVersion) { + if (siteId != null && apiContact != null && clientVersion != null) { + final String path = RoutingContextUtil.getPath(rc); + CLIENT_VERSION_COUNTERS.computeIfAbsent( + new Tuple.Tuple2<>(Integer.toString(siteId), clientVersion), + tuple -> Counter + .builder("uid2_client_sdk_versions_total") + .description("counter for how many http requests are processed per each operator-served sdk version") + .tags("site_id", tuple.getItem1(), "api_contact", apiContact, "client_version", tuple.getItem2(), "path", path) + .register(Metrics.globalRegistry) + ).increment(); } } private void handleTokenRefreshV2(RoutingContext rc) { Integer siteId = null; TokenResponseStatsCollector.PlatformType platformType = TokenResponseStatsCollector.PlatformType.Other; + + RuntimeConfig config = this.getConfigFromRc(rc); + Duration identityExpiresAfter = Duration.ofSeconds(config.getIdentityTokenExpiresAfterSeconds()); try { platformType = getPlatformType(rc); String tokenStr = (String) rc.data().get("request"); - final RefreshResponse r = this.refreshIdentity(rc, tokenStr); + final TokenRefreshResponse r = this.refreshIdentity(rc, tokenStr); siteId = rc.get(Const.RoutingContextData.SiteId); + final String apiContact = RoutingContextUtil.getApiContact(rc, clientKeyProvider); + recordOperatorServedSdkUsage(rc, siteId, apiContact, rc.request().headers().get(Const.Http.ClientVersionHeader)); if (!r.isRefreshed()) { if (r.isOptOut() || r.isDeprecated()) { ResponseUtil.SuccessNoBodyV2(ResponseStatus.OptOut, rc); } else if (!AuthMiddleware.isAuthenticated(rc)) { // unauthenticated clients get a generic error - ResponseUtil.Warning(ResponseStatus.GenericError, 400, rc, "Error refreshing token"); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.GenericError, 400, rc, "Error refreshing token"); } else if (r.isInvalidToken()) { - ResponseUtil.Warning(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented"); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented"); } else if (r.isExpired()) { - ResponseUtil.Warning(ResponseStatus.ExpiredToken, 400, rc, "Expired Token presented"); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.ExpiredToken, 400, rc, "Expired Token presented"); } else if (r.noActiveKey()) { SendServerErrorResponseAndRecordStats(rc, "No active encryption key available", siteId, TokenResponseStatsCollector.Endpoint.RefreshV2, TokenResponseStatsCollector.ResponseStatus.NoActiveKey, siteProvider, new KeyManager.NoActiveKeyException("No active encryption key available"), platformType); } else { - ResponseUtil.Error(ResponseStatus.UnknownError, 500, rc, "Unknown State"); + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.UnknownError, 500, rc, "Unknown State"); } } else { - ResponseUtil.SuccessV2(rc, toJsonV1(r.getIdentityResponse())); - this.recordRefreshDurationStats(siteId, getApiContact(rc), r.getDurationSinceLastRefresh(), rc.request().headers().contains(ORIGIN_HEADER)); + ResponseUtil.SuccessV2(rc, r.getIdentityResponse().toTokenGenerateResponseJson()); + this.recordRefreshDurationStats(siteId, getApiContact(rc), r.getDurationSinceLastRefresh(), rc.request().headers().contains(ORIGIN_HEADER), identityExpiresAfter); } TokenResponseStatsCollector.recordRefresh(siteProvider, siteId, TokenResponseStatsCollector.Endpoint.RefreshV2, r, platformType); } catch (Exception e) { @@ -868,50 +849,21 @@ private void handleTokenRefreshV2(RoutingContext rc) { } } - private void handleTokenValidateV1(RoutingContext rc) { - try { - final InputUtil.InputVal input = this.phoneSupport ? getTokenInputV1(rc) : getTokenInput(rc); - if (!checkForInvalidTokenInput(input, rc)) { - return; - } - if ((Arrays.equals(ValidateIdentityForEmailHash, input.getIdentityInput()) && input.getIdentityType() == IdentityType.Email) - || (Arrays.equals(ValidateIdentityForPhoneHash, input.getIdentityInput()) && input.getIdentityType() == IdentityType.Phone)) { - try { - final Instant now = Instant.now(); - if (this.idService.advertisingTokenMatches(rc.queryParam("token").get(0), input.toHashedDiiIdentity(this.identityScope, 0, now), now)) { - ResponseUtil.Success(rc, Boolean.TRUE); - } else { - ResponseUtil.Success(rc, Boolean.FALSE); - } - } catch (Exception e) { - ResponseUtil.Success(rc, Boolean.FALSE); - } - } else { - ResponseUtil.Success(rc, Boolean.FALSE); - } - } catch (ClientInputValidationException cie) { - ResponseUtil.Warning(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented"); - } catch (Exception e) { - LOGGER.error("Unknown error while validating token", e); - rc.fail(500); - } - } - private void handleTokenValidateV2(RoutingContext rc) { try { final JsonObject req = (JsonObject) rc.data().get("request"); final InputUtil.InputVal input = getTokenInputV2(req); - if (!checkForInvalidTokenInput(input, rc)) { + if (!isTokenInputValid(input, rc)) { return; } - if ((input.getIdentityType() == IdentityType.Email && Arrays.equals(ValidateIdentityForEmailHash, input.getIdentityInput())) - || (input.getIdentityType() == IdentityType.Phone && Arrays.equals(ValidateIdentityForPhoneHash, input.getIdentityInput()))) { + if ((input.getDiiType() == DiiType.Email && Arrays.equals(ValidateIdentityForEmailHash, input.getHashedDiiInput())) + || (input.getDiiType() == DiiType.Phone && Arrays.equals(ValidateIdentityForPhoneHash, input.getHashedDiiInput()))) { try { final Instant now = Instant.now(); final String token = req.getString("token"); - if (this.idService.advertisingTokenMatches(token, input.toHashedDiiIdentity(this.identityScope, 0, now), now)) { + if (this.idService.advertisingTokenMatches(token, input.toHashedDii(this.identityScope), now)) { ResponseUtil.SuccessV2(rc, Boolean.TRUE); } else { ResponseUtil.SuccessV2(rc, Boolean.FALSE); @@ -928,45 +880,24 @@ private void handleTokenValidateV2(RoutingContext rc) { } } - private void handleTokenGenerateV1(RoutingContext rc) { - final int siteId = AuthMiddleware.getAuthClient(rc).getSiteId(); - TokenResponseStatsCollector.PlatformType platformType = TokenResponseStatsCollector.PlatformType.Other; - try { - final InputUtil.InputVal input = this.phoneSupport ? this.getTokenInputV1(rc) : this.getTokenInput(rc); - platformType = getPlatformType(rc); - if (!checkForInvalidTokenInput(input, rc)) { - return; - } else { - final IdentityResponse t = this.idService.generateIdentity( - new IdentityRequest( - new SourcePublisher(siteId, 0, 0), - input.toHashedDiiIdentity(this.identityScope, 1, Instant.now()), - OptoutCheckPolicy.defaultPolicy())); - - //Integer.parseInt(rc.queryParam("privacy_bits").get(0)))); - - ResponseUtil.Success(rc, toJsonV1(t)); - recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV1, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, t.getAdvertisingTokenVersion(), platformType); - } - } catch (Exception e) { - SendServerErrorResponseAndRecordStats(rc, "Unknown error while generating token v1", siteId, TokenResponseStatsCollector.Endpoint.GenerateV1, TokenResponseStatsCollector.ResponseStatus.Unknown, siteProvider, e, platformType); - } - } - private void handleTokenGenerateV2(RoutingContext rc) { final Integer siteId = AuthMiddleware.getAuthClient(rc).getSiteId(); TokenResponseStatsCollector.PlatformType platformType = TokenResponseStatsCollector.PlatformType.Other; + + RuntimeConfig config = this.getConfigFromRc(rc); + Duration refreshIdentityAfter = Duration.ofSeconds(config.getRefreshIdentityTokenAfterSeconds()); + Duration refreshExpiresAfter = Duration.ofSeconds(config.getRefreshTokenExpiresAfterSeconds()); + Duration identityExpiresAfter = Duration.ofSeconds(config.getIdentityTokenExpiresAfterSeconds()); + try { JsonObject req = (JsonObject) rc.data().get("request"); platformType = getPlatformType(rc); final InputUtil.InputVal input = this.getTokenInputV2(req); - if (!checkForInvalidTokenInput(input, rc)) { - return; - } else { + if (isTokenInputValid(input, rc)) { final String apiContact = getApiContact(rc); - switch (validateUserConsent(req)) { + switch (validateUserConsent(req, apiContact)) { case INVALID: { SendClientErrorResponseAndRecordStats(ResponseStatus.ClientError, 400, rc, "User consent is invalid", siteId, TokenResponseStatsCollector.Endpoint.GenerateV2, TokenResponseStatsCollector.ResponseStatus.InvalidUserConsentString, siteProvider, platformType); return; @@ -980,8 +911,9 @@ private void handleTokenGenerateV2(RoutingContext rc) { break; } default: { - assert false : "Please update UIDOperatorVerticle.handleTokenGenerateV2 when changing UserConsentStatus"; - break; + final String errorMsg = "Please update UIDOperatorVerticle.handleTokenGenerateV2 when changing UserConsentStatus"; + LOGGER.error(errorMsg); + throw new IllegalStateException(errorMsg); } } @@ -992,16 +924,19 @@ private void handleTokenGenerateV2(RoutingContext rc) { SendClientErrorResponseAndRecordStats(ResponseStatus.ClientError, 400, rc, "Required opt-out policy argument for token/generate is missing or not set to 1", siteId, TokenResponseStatsCollector.Endpoint.GenerateV2, TokenResponseStatsCollector.ResponseStatus.BadPayload, siteProvider, platformType); return; } - - final IdentityResponse t = this.idService.generateIdentity( - new IdentityRequest( - new SourcePublisher(siteId, 0, 0), - input.toHashedDiiIdentity(this.identityScope, 1, Instant.now()), - OptoutCheckPolicy.respectOptOut())); - - if (t.isOptedOut()) { - if (optoutCheckPolicy.getItem1() == OptoutCheckPolicy.DoNotRespect) { // only legacy can use this policy - final InputUtil.InputVal optOutTokenInput = input.getIdentityType() == IdentityType.Email + final TokenGenerateResponse response = this.idService.generateIdentity( + new TokenGenerateRequest( + new SourcePublisher(siteId), + input.toHashedDii(this.identityScope), + OptoutCheckPolicy.respectOptOut()), + refreshIdentityAfter, + refreshExpiresAfter, + identityExpiresAfter); + + if (response.isOptedOut()) { + if (optoutCheckPolicy.getItem1() == OptoutCheckPolicy.DoNotRespect && !this.disableOptoutToken) { // only legacy can use + // this policy + final InputUtil.InputVal optOutTokenInput = input.getDiiType() == DiiType.Email ? InputUtil.InputVal.validEmail(OptOutTokenIdentityForEmail, OptOutTokenIdentityForEmail) : InputUtil.InputVal.validPhone(OptOutTokenIdentityForPhone, OptOutTokenIdentityForPhone); @@ -1009,21 +944,23 @@ private void handleTokenGenerateV2(RoutingContext rc) { pb.setLegacyBit(); pb.setClientSideTokenGenerateOptout(); - final IdentityResponse optOutTokens = this.idService.generateIdentity( - new IdentityRequest( - new SourcePublisher(siteId, 0, 0), - optOutTokenInput.toHashedDiiIdentity(this.identityScope, pb.getAsInt(), Instant.now()), - OptoutCheckPolicy.DoNotRespect)); - - ResponseUtil.SuccessV2(rc, toJsonV1(optOutTokens)); + final TokenGenerateResponse optOutTokens = this.idService.generateIdentity( + new TokenGenerateRequest( + new SourcePublisher(siteId), + optOutTokenInput.toHashedDii(this.identityScope), + OptoutCheckPolicy.DoNotRespect, pb, Instant.now()), + refreshIdentityAfter, + refreshExpiresAfter, + identityExpiresAfter); + ResponseUtil.SuccessV2(rc, optOutTokens.toTokenGenerateResponseJson()); recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV2, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, optOutTokens.getAdvertisingTokenVersion(), platformType); } else { // new participant, or legacy specified policy/optout_check=1 ResponseUtil.SuccessNoBodyV2("optout", rc); recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV2, TokenResponseStatsCollector.ResponseStatus.OptOut, siteProvider, null, platformType); } } else { - ResponseUtil.SuccessV2(rc, toJsonV1(t)); - recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV2, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, t.getAdvertisingTokenVersion(), platformType); + ResponseUtil.SuccessV2(rc, response.toTokenGenerateResponseJson()); + recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV2, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, response.getAdvertisingTokenVersion(), platformType); } } } catch (KeyManager.NoActiveKeyException e) { @@ -1035,106 +972,15 @@ private void handleTokenGenerateV2(RoutingContext rc) { } } - private void handleTokenGenerate(RoutingContext rc) { - final InputUtil.InputVal input = this.getTokenInput(rc); - Integer siteId = null; - if (input == null) { - SendClientErrorResponseAndRecordStats(ResponseStatus.ClientError, 400, rc, "Required Parameter Missing: exactly one of email or email_hash must be specified", siteId, TokenResponseStatsCollector.Endpoint.GenerateV0, TokenResponseStatsCollector.ResponseStatus.BadPayload, siteProvider, TokenResponseStatsCollector.PlatformType.Other); - return; - } - else if (!input.isValid()) { - SendClientErrorResponseAndRecordStats(ResponseStatus.ClientError, 400, rc, "Invalid email or email_hash", siteId, TokenResponseStatsCollector.Endpoint.GenerateV0, TokenResponseStatsCollector.ResponseStatus.BadPayload, siteProvider, TokenResponseStatsCollector.PlatformType.Other); - return; - } - - try { - siteId = AuthMiddleware.getAuthClient(rc).getSiteId(); - final IdentityResponse t = this.idService.generateIdentity( - new IdentityRequest( - new SourcePublisher(siteId, 0, 0), - input.toHashedDiiIdentity(this.identityScope, 1, Instant.now()), - OptoutCheckPolicy.defaultPolicy())); - - //Integer.parseInt(rc.queryParam("privacy_bits").get(0)))); - - recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV0, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, t.getAdvertisingTokenVersion(), TokenResponseStatsCollector.PlatformType.Other); - sendJsonResponse(rc, toJson(t)); - - } catch (Exception e) { - SendServerErrorResponseAndRecordStats(rc, "Unknown error while generating token", siteId, TokenResponseStatsCollector.Endpoint.GenerateV0, TokenResponseStatsCollector.ResponseStatus.Unknown, siteProvider, e, TokenResponseStatsCollector.PlatformType.Other); - } - } - - private void handleTokenRefresh(RoutingContext rc) { - final List tokenList = rc.queryParam("refresh_token"); - Integer siteId = null; - if (tokenList == null || tokenList.size() == 0) { - SendClientErrorResponseAndRecordStats(ResponseStatus.ClientError, 400, rc, "Required Parameter Missing: refresh_token", siteId, TokenResponseStatsCollector.Endpoint.RefreshV0, TokenResponseStatsCollector.ResponseStatus.MissingParams, siteProvider, TokenResponseStatsCollector.PlatformType.Other); - return; - } - - try { - final RefreshResponse r = this.refreshIdentity(rc, tokenList.get(0)); - - sendJsonResponse(rc, toJson(r.getIdentityResponse())); - - siteId = rc.get(Const.RoutingContextData.SiteId); - if (r.isRefreshed()) { - this.recordRefreshDurationStats(siteId, getApiContact(rc), r.getDurationSinceLastRefresh(), rc.request().headers().contains(ORIGIN_HEADER)); - } - TokenResponseStatsCollector.recordRefresh(siteProvider, siteId, TokenResponseStatsCollector.Endpoint.RefreshV0, r, TokenResponseStatsCollector.PlatformType.Other); - } catch (Exception e) { - SendServerErrorResponseAndRecordStats(rc, "Unknown error while refreshing token", siteId, TokenResponseStatsCollector.Endpoint.RefreshV0, TokenResponseStatsCollector.ResponseStatus.Unknown, siteProvider, e, TokenResponseStatsCollector.PlatformType.Other); - } - } - - private void handleValidate(RoutingContext rc) { - try { - final InputUtil.InputVal input = getTokenInput(rc); - if (input != null && input.isValid() && Arrays.equals(ValidateIdentityForEmailHash, input.getIdentityInput())) { - try { - final Instant now = Instant.now(); - if (this.idService.advertisingTokenMatches(rc.queryParam("token").get(0), input.toHashedDiiIdentity(this.identityScope, 0, now), now)) { - rc.response().end("true"); - } else { - rc.response().end("false"); - } - } catch (Exception e) { - rc.response().end("false"); - } - } else { - rc.response().end("not allowed"); - } - } catch (Exception e) { - LOGGER.error("Unknown error while validating token", e); - rc.fail(500); - } - } - - private void handleLogoutAsync(RoutingContext rc) { - final InputUtil.InputVal input = this.phoneSupport ? getTokenInputV1(rc) : getTokenInput(rc); - if (input.isValid()) { - final Instant now = Instant.now(); - this.idService.invalidateTokensAsync(input.toHashedDiiIdentity(this.identityScope, 0, now), now, ar -> { - if (ar.succeeded()) { - rc.response().end("OK"); - } else { - rc.fail(500); - } - }); - } else { - ResponseUtil.Warning(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented " + input); - } - } - private Future handleLogoutAsyncV2(RoutingContext rc) { final JsonObject req = (JsonObject) rc.data().get("request"); final InputUtil.InputVal input = getTokenInputV2(req); - if (input.isValid()) { + final String uidTraceId = rc.request().getHeader(Audit.UID_TRACE_ID_HEADER); + if (input != null && input.isValid()) { final Instant now = Instant.now(); Promise promise = Promise.promise(); - this.idService.invalidateTokensAsync(input.toHashedDiiIdentity(this.identityScope, 0, now), now, ar -> { + this.idService.invalidateTokensAsync(input.toHashedDii(this.identityScope), now, uidTraceId, ar -> { if (ar.succeeded()) { JsonObject body = new JsonObject(); body.put("optout", "OK"); @@ -1146,62 +992,11 @@ private Future handleLogoutAsyncV2(RoutingContext rc) { }); return promise.future(); } else { - ResponseUtil.Warning(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented " + input); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented " + input); return Future.failedFuture(""); } } - private void handleOptOutGet(RoutingContext rc) { - final InputUtil.InputVal input = getTokenInputV1(rc); - if (input.isValid()) { - try { - final Instant now = Instant.now(); - final HashedDiiIdentity hashedDiiIdentity = input.toHashedDiiIdentity(this.identityScope, 0, now); - final Instant result = this.idService.getLatestOptoutEntry(hashedDiiIdentity, now); - long timestamp = result == null ? -1 : result.getEpochSecond(); - rc.response().setStatusCode(200) - .setChunked(true) - .write(String.valueOf(timestamp)); - rc.response().end(); - } catch (Exception ex) { - LOGGER.error("Unexpected error while handling optout get", ex); - rc.fail(500); - } - } else { - ResponseUtil.Warning(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented " + input); - } - } - - private void handleBucketsV1(RoutingContext rc) { - final List qp = rc.queryParam("since_timestamp"); - if (qp != null && qp.size() > 0) { - final Instant sinceTimestamp; - try { - LocalDateTime ld = LocalDateTime.parse(qp.get(0), DateTimeFormatter.ISO_LOCAL_DATE_TIME); - sinceTimestamp = ld.toInstant(ZoneOffset.UTC); - LOGGER.info(String.format("identity bucket endpoint is called with since_timestamp %s and site id %s", ld, AuthMiddleware.getAuthClient(rc).getSiteId())); - } catch (Exception e) { - ResponseUtil.ClientError(rc, "invalid date, must conform to ISO 8601"); - return; - } - final List modified = this.idService.getModifiedBuckets(sinceTimestamp); - final JsonArray resp = new JsonArray(); - if (modified != null) { - for (SaltEntry e : modified) { - final JsonObject o = new JsonObject(); - o.put("bucket_id", e.getHashedId()); - Instant lastUpdated = Instant.ofEpochMilli(e.getLastUpdated()); - - o.put("last_updated", APIDateTimeFormatter.format(lastUpdated)); - resp.add(o); - } - ResponseUtil.Success(rc, resp); - } - } else { - ResponseUtil.ClientError(rc, "missing parameter since_timestamp"); - } - } - private void handleBucketsV2(RoutingContext rc) { final JsonObject req = (JsonObject) rc.data().get("request"); final String qp = req.getString("since_timestamp"); @@ -1213,7 +1008,7 @@ private void handleBucketsV2(RoutingContext rc) { sinceTimestamp = ld.toInstant(ZoneOffset.UTC); LOGGER.info(String.format("identity bucket endpoint is called with since_timestamp %s and site id %s", ld, AuthMiddleware.getAuthClient(rc).getSiteId())); } catch (Exception e) { - ResponseUtil.ClientError(rc, "invalid date, must conform to ISO 8601"); + ResponseUtil.LogInfoAndSend400Response(rc, "invalid date, must conform to ISO 8601"); return; } final List modified = this.idService.getModifiedBuckets(sinceTimestamp); @@ -1221,75 +1016,17 @@ private void handleBucketsV2(RoutingContext rc) { if (modified != null) { for (SaltEntry e : modified) { final JsonObject o = new JsonObject(); - o.put("bucket_id", e.getHashedId()); - Instant lastUpdated = Instant.ofEpochMilli(e.getLastUpdated()); + o.put("bucket_id", e.hashedId()); + Instant lastUpdated = Instant.ofEpochMilli(e.lastUpdated()); - o.put("last_updated", APIDateTimeFormatter.format(lastUpdated)); + o.put("last_updated", API_DATE_TIME_FORMATTER.format(lastUpdated)); resp.add(o); } ResponseUtil.SuccessV2(rc, resp); } } else { - ResponseUtil.ClientError(rc, "missing parameter since_timestamp"); - } - } - - private void handleIdentityMapV1(RoutingContext rc) { - final InputUtil.InputVal input = this.phoneSupport ? this.getTokenInputV1(rc) : this.getTokenInput(rc); - if (!checkForInvalidTokenInput(input, rc)) { - return; - } - try { - final Instant now = Instant.now(); - final RawUidResponse rawUidResponse = this.idService.map(input.toHashedDiiIdentity(this.identityScope, 0, now), now); - final JsonObject jsonObject = new JsonObject(); - jsonObject.put("identifier", input.getProvided()); - jsonObject.put("advertising_id", EncodingUtils.toBase64String(rawUidResponse.rawUid)); - jsonObject.put("bucket_id", rawUidResponse.bucketId); - ResponseUtil.Success(rc, jsonObject); - } catch (Exception e) { - ResponseUtil.Error(ResponseStatus.UnknownError, 500, rc, "Unknown State", e); - } - } - - private void handleIdentityMap(RoutingContext rc) { - final InputUtil.InputVal input = this.getTokenInput(rc); - - try { - if (input == null) { - ResponseUtil.ClientError(rc, "Required Parameter Missing: exactly one of email or email_hash must be specified"); - } - else if (!input.isValid()) { - ResponseUtil.ClientError(rc, "Invalid email or email_hash"); - } - else { - final Instant now = Instant.now(); - final RawUidResponse rawUidResponse = this.idService.map(input.toHashedDiiIdentity(this.identityScope, 0, now), now); - rc.response().end(EncodingUtils.toBase64String(rawUidResponse.rawUid)); - } - } catch (Exception ex) { - LOGGER.error("Unexpected error while mapping identity", ex); - rc.fail(500); - } - } - - private InputUtil.InputVal getTokenInput(RoutingContext rc) { - final InputUtil.InputVal input; - final List emailInput = rc.queryParam("email"); - final List emailHashInput = rc.queryParam("email_hash"); - if (emailInput != null && emailInput.size() > 0) { - if (emailHashInput != null && emailHashInput.size() > 0) { - // cannot specify both - input = null; - } else { - input = InputUtil.normalizeEmail(emailInput.get(0)); - } - } else if (emailHashInput != null && emailHashInput.size() > 0) { - input = InputUtil.normalizeEmailHash(emailHashInput.get(0)); - } else { - input = null; + ResponseUtil.LogInfoAndSend400Response(rc, "missing parameter since_timestamp"); } - return input; } private InputUtil.InputVal getTokenInputV2(JsonObject req) { @@ -1327,129 +1064,18 @@ private InputUtil.InputVal getTokenInputV2(JsonObject req) { return getInput != null ? getInput.get() : null; } - private InputUtil.InputVal getTokenInputV1(RoutingContext rc) { - final List emailInput = rc.queryParam("email"); - final List emailHashInput = rc.queryParam("email_hash"); - final List phoneInput = rc.queryParam("phone"); - final List phoneHashInput = rc.queryParam("phone_hash"); - - int validInputs = 0; - if (emailInput != null && emailInput.size() > 0) { - ++validInputs; - } - if (emailHashInput != null && emailHashInput.size() > 0) { - ++validInputs; - } - if (phoneInput != null && phoneInput.size() > 0) { - ++validInputs; - } - if (phoneHashInput != null && phoneHashInput.size() > 0) { - ++validInputs; - } - - if (validInputs != 1) { - // there can be only 1 set of valid input - return null; - } - - if (emailInput != null && emailInput.size() > 0) { - return InputUtil.normalizeEmail(emailInput.get(0)); - } else if (phoneInput != null && phoneInput.size() > 0) { - return InputUtil.normalizePhone(phoneInput.get(0)); - } else if (emailHashInput != null && emailHashInput.size() > 0) { - return InputUtil.normalizeEmailHash(emailHashInput.get(0)); - } else if (phoneHashInput != null && phoneHashInput.size() > 0) { - return InputUtil.normalizePhoneHash(phoneHashInput.get(0)); - } - - return null; - } - - private boolean checkForInvalidTokenInput(InputUtil.InputVal input, RoutingContext rc) { + private boolean isTokenInputValid(InputUtil.InputVal input, RoutingContext rc) { if (input == null) { - String message = this.phoneSupport ? "Required Parameter Missing: exactly one of [email, email_hash, phone, phone_hash] must be specified" : "Required Parameter Missing: exactly one of email or email_hash must be specified"; - ResponseUtil.ClientError(rc, message); + String message = this.phoneSupport ? ERROR_INVALID_INPUT_WITH_PHONE_SUPPORT : ERROR_INVALID_INPUT_EMAIL_MISSING; + ResponseUtil.LogInfoAndSend400Response(rc, message); return false; } else if (!input.isValid()) { - ResponseUtil.ClientError(rc, "Invalid Identifier"); + ResponseUtil.LogInfoAndSend400Response(rc, "Invalid Identifier"); return false; } return true; } - private InputUtil.InputVal[] getIdentityBulkInput(RoutingContext rc) { - final JsonObject obj = rc.body().asJsonObject(); - final JsonArray emails = obj.getJsonArray("email"); - final JsonArray emailHashes = obj.getJsonArray("email_hash"); - // FIXME TODO. Avoid Double Iteration. Turn to a decorator pattern - if (emails == null && emailHashes == null) { - ResponseUtil.ClientError(rc, "Exactly one of email or email_hash must be specified"); - return null; - } else if (emails != null && !emails.isEmpty()) { - if (emailHashes != null && !emailHashes.isEmpty()) { - ResponseUtil.ClientError(rc, "Only one of email or email_hash can be specified"); - return null; - } - return createInputList(emails, false); - } else { - return createInputList(emailHashes, true); - } - } - - - private InputUtil.InputVal[] getIdentityBulkInputV1(RoutingContext rc) { - final JsonObject obj = rc.body().asJsonObject(); - if(obj.isEmpty()) { - ResponseUtil.ClientError(rc, "Exactly one of [email, email_hash, phone, phone_hash] must be specified"); - return null; - } - final JsonArray emails = JsonParseUtils.parseArray(obj, "email", rc); - final JsonArray emailHashes = JsonParseUtils.parseArray(obj, "email_hash", rc); - final JsonArray phones = JsonParseUtils.parseArray(obj,"phone", rc); - final JsonArray phoneHashes = JsonParseUtils.parseArray(obj,"phone_hash", rc); - - if (emails == null && emailHashes == null && phones == null && phoneHashes == null) { - return null; - } - - int validInputs = 0; - int nonEmptyInputs = 0; - if (emails != null) { - ++validInputs; - if (!emails.isEmpty()) ++nonEmptyInputs; - } - if (emailHashes != null) { - ++validInputs; - if (!emailHashes.isEmpty()) ++nonEmptyInputs; - } - if (phones != null) { - ++validInputs; - if (!phones.isEmpty()) ++nonEmptyInputs; - } - if (phoneHashes != null) { - ++validInputs; - if (!phoneHashes.isEmpty()) ++nonEmptyInputs; - } - - if (validInputs == 0 || nonEmptyInputs > 1) { - ResponseUtil.ClientError(rc, "Exactly one of [email, email_hash, phone, phone_hash] must be specified"); - return null; - } - - if (emails != null && !emails.isEmpty()) { - return createInputListV1(emails, IdentityType.Email, InputUtil.IdentityInputType.Raw); - } else if (emailHashes != null && !emailHashes.isEmpty()) { - return createInputListV1(emailHashes, IdentityType.Email, InputUtil.IdentityInputType.Hash); - } else if (phones != null && !phones.isEmpty()) { - return createInputListV1(phones, IdentityType.Phone, InputUtil.IdentityInputType.Raw); - } else if (phoneHashes != null && !phoneHashes.isEmpty()) { - return createInputListV1(phoneHashes, IdentityType.Phone, InputUtil.IdentityInputType.Hash); - } else { - // handle empty array - return createInputListV1(null, IdentityType.Email, InputUtil.IdentityInputType.Raw); - } - } - private JsonObject handleIdentityMapCommon(RoutingContext rc, InputUtil.InputVal[] inputList) { final Instant now = Instant.now(); final JsonArray mapped = new JsonArray(); @@ -1460,13 +1086,13 @@ private JsonObject handleIdentityMapCommon(RoutingContext rc, InputUtil.InputVal for (int i = 0; i < count; ++i) { final InputUtil.InputVal input = inputList[i]; if (input != null && input.isValid()) { - final RawUidResponse rawUidResponse = idService.mapIdentity( - new MapRequest( - input.toHashedDiiIdentity(this.identityScope, 0, now), + final IdentityMapResponseItem identityMapResponseItem = idService.mapHashedDii( + new IdentityMapRequestItem( + input.toHashedDii(this.identityScope), OptoutCheckPolicy.respectOptOut(), now)); - if (rawUidResponse.isOptedOut()) { + if (identityMapResponseItem.isOptedOut()) { final JsonObject resp = new JsonObject(); resp.put("identifier", input.getProvided()); resp.put("reason", "optout"); @@ -1475,8 +1101,8 @@ private JsonObject handleIdentityMapCommon(RoutingContext rc, InputUtil.InputVal } else { final JsonObject resp = new JsonObject(); resp.put("identifier", input.getProvided()); - resp.put("advertising_id", EncodingUtils.toBase64String(rawUidResponse.rawUid)); - resp.put("bucket_id", rawUidResponse.bucketId); + resp.put("advertising_id", EncodingUtils.toBase64String(identityMapResponseItem.rawUid)); + resp.put("bucket_id", identityMapResponseItem.bucketId); mapped.add(resp); } } else { @@ -1496,39 +1122,76 @@ private JsonObject handleIdentityMapCommon(RoutingContext rc, InputUtil.InputVal return resp; } - private void handleIdentityMapBatchV1(RoutingContext rc) { - try { - final InputUtil.InputVal[] inputList = this.phoneSupport ? getIdentityBulkInputV1(rc) : getIdentityBulkInput(rc); - if (inputList == null) return; + private JsonObject processIdentityMapV3Response(RoutingContext rc, Map input) { + final Instant now = Instant.now(); + final JsonObject mappedResponse = new JsonObject(); + int invalidCount = 0; + int optoutCount = 0; + int inputTotalCount = 0; - final JsonObject resp = handleIdentityMapCommon(rc, inputList); - ResponseUtil.Success(rc, resp); - } catch (Exception e) { - ResponseUtil.Error(ResponseStatus.UnknownError, 500, rc, "Unknown error while mapping batched identity", e); + for (Map.Entry identityType : input.entrySet()) { + JsonArray mappedIdentityList = new JsonArray(); + final InputUtil.InputVal[] rawIdentityList = identityType.getValue(); + inputTotalCount += rawIdentityList.length; + + for (final InputUtil.InputVal rawId : rawIdentityList) { + final JsonObject resp = new JsonObject(); + if (rawId != null && rawId.isValid()) { + final IdentityMapResponseItem identityMapResponseItem = idService.mapHashedDii( + new IdentityMapRequestItem( + rawId.toHashedDii(this.identityScope), + OptoutCheckPolicy.respectOptOut(), + now)); + + if (identityMapResponseItem.isOptedOut()) { + resp.put("e", IdentityMapResponseType.OPTOUT.getValue()); + optoutCount++; + } else { + resp.put("u", EncodingUtils.toBase64String(identityMapResponseItem.rawUid)); + resp.put("p", identityMapResponseItem.previousRawUid == null ? null : EncodingUtils.toBase64String(identityMapResponseItem.previousRawUid)); + resp.put("r", identityMapResponseItem.refreshFrom / SECOND_IN_MILLIS); + } + } else { + resp.put("e", IdentityMapResponseType.INVALID_IDENTIFIER.getValue()); + invalidCount++; + } + mappedIdentityList.add(resp); + } + mappedResponse.put(identityType.getKey(), mappedIdentityList); + } + + recordIdentityMapStats(rc, inputTotalCount, invalidCount, optoutCount); + + return mappedResponse; + } + + private boolean validateServiceLink(RoutingContext rc) { + JsonObject requestJsonObject = (JsonObject) rc.data().get(REQUEST); + if (this.secureLinkValidatorService.validateRequest(rc, requestJsonObject, Role.MAPPER)) { + return true; } + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.Unauthorized, HttpStatus.SC_UNAUTHORIZED, rc, "Invalid link_id"); + return false; } private void handleIdentityMapV2(RoutingContext rc) { try { + final Integer siteId = RoutingContextUtil.getSiteId(rc); + final String apiContact = RoutingContextUtil.getApiContact(rc, clientKeyProvider); + recordOperatorServedSdkUsage(rc, siteId, apiContact, rc.request().headers().get(Const.Http.ClientVersionHeader)); + final InputUtil.InputVal[] inputList = getIdentityMapV2Input(rc); if (inputList == null) { - if (this.phoneSupport) - ResponseUtil.ClientError(rc, "Exactly one of [email, email_hash, phone, phone_hash] must be specified"); - else - ResponseUtil.ClientError(rc, "Required Parameter Missing: exactly one of email or email_hash must be specified"); + ResponseUtil.LogInfoAndSend400Response(rc, this.phoneSupport ? ERROR_INVALID_INPUT_WITH_PHONE_SUPPORT : ERROR_INVALID_INPUT_EMAIL_MISSING); return; } - JsonObject requestJsonObject = (JsonObject) rc.data().get(REQUEST); - if (!this.secureLinkValidatorService.validateRequest(rc, requestJsonObject, Role.MAPPER)) { - ResponseUtil.Error(ResponseStatus.Unauthorized, HttpStatus.SC_UNAUTHORIZED, rc, "Invalid link_id"); - return; - } + if (!validateServiceLink(rc)) { return; } final JsonObject resp = handleIdentityMapCommon(rc, inputList); ResponseUtil.SuccessV2(rc, resp); } catch (Exception e) { - ResponseUtil.Error(ResponseStatus.UnknownError, 500, rc, "Unknown error while mapping identity v2", e); + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.UnknownError, 500, rc, "Unknown error while mapping identity v2", e); } } @@ -1538,7 +1201,7 @@ private InputUtil.InputVal[] getIdentityMapV2Input(RoutingContext rc) { Supplier getInputList = null; final JsonArray emails = JsonParseUtils.parseArray(obj, "email", rc); if (emails != null && !emails.isEmpty()) { - getInputList = () -> createInputListV1(emails, IdentityType.Email, InputUtil.IdentityInputType.Raw); + getInputList = () -> createInputList(emails, DiiType.Email, InputUtil.DiiInputType.Raw); } final JsonArray emailHashes = JsonParseUtils.parseArray(obj, "email_hash", rc); @@ -1546,7 +1209,7 @@ private InputUtil.InputVal[] getIdentityMapV2Input(RoutingContext rc) { if (getInputList != null) { return null; // only one type of input is allowed } - getInputList = () -> createInputListV1(emailHashes, IdentityType.Email, InputUtil.IdentityInputType.Hash); + getInputList = () -> createInputList(emailHashes, DiiType.Email, InputUtil.DiiInputType.Hash); } final JsonArray phones = this.phoneSupport ? JsonParseUtils.parseArray(obj,"phone", rc) : null; @@ -1554,7 +1217,7 @@ private InputUtil.InputVal[] getIdentityMapV2Input(RoutingContext rc) { if (getInputList != null) { return null; // only one type of input is allowed } - getInputList = () -> createInputListV1(phones, IdentityType.Phone, InputUtil.IdentityInputType.Raw); + getInputList = () -> createInputList(phones, DiiType.Phone, InputUtil.DiiInputType.Raw); } final JsonArray phoneHashes = this.phoneSupport ? JsonParseUtils.parseArray(obj,"phone_hash", rc) : null; @@ -1562,7 +1225,7 @@ private InputUtil.InputVal[] getIdentityMapV2Input(RoutingContext rc) { if (getInputList != null) { return null; // only one type of input is allowed } - getInputList = () -> createInputListV1(phoneHashes, IdentityType.Phone, InputUtil.IdentityInputType.Hash); + getInputList = () -> createInputList(phoneHashes, DiiType.Phone, InputUtil.DiiInputType.Hash); } if (emails == null && emailHashes == null && phones == null && phoneHashes == null) { @@ -1570,37 +1233,51 @@ private InputUtil.InputVal[] getIdentityMapV2Input(RoutingContext rc) { } return getInputList == null ? - createInputListV1(null, IdentityType.Email, InputUtil.IdentityInputType.Raw) : // handle empty array + createInputList(null, DiiType.Email, InputUtil.DiiInputType.Raw) : // handle empty array getInputList.get(); } - private void handleIdentityMapBatch(RoutingContext rc) { + private void handleIdentityMapV3(RoutingContext rc) { try { - final JsonObject obj = rc.body().asJsonObject(); - final InputUtil.InputVal[] inputList; - final JsonArray emails = obj.getJsonArray("email"); - final JsonArray emailHashes = obj.getJsonArray("email_hash"); - if (emails == null && emailHashes == null) { - ResponseUtil.ClientError(rc, "Exactly one of email or email_hash must be specified"); + JsonObject jsonInput = (JsonObject) rc.data().get("request"); + + if (jsonInput == null || jsonInput.isEmpty()) { + ResponseUtil.LogInfoAndSend400Response(rc, phoneSupport ? ERROR_INVALID_MIXED_INPUT_WITH_PHONE_SUPPORT : ERROR_INVALID_MIXED_INPUT_EMAIL_MISSING); return; - } else if (emails != null && !emails.isEmpty()) { - if (emailHashes != null && !emailHashes.isEmpty()) { - ResponseUtil.ClientError(rc, "Only one of email or email_hash can be specified"); - return; - } - inputList = createInputList(emails, false); - } else { - inputList = createInputList(emailHashes, true); } - final JsonObject resp = handleIdentityMapCommon(rc, inputList); - sendJsonResponse(rc, resp); + IdentityMapV3Request input = OBJECT_MAPPER.readValue(jsonInput.toString(), IdentityMapV3Request.class); + final Map normalizedInput = processIdentityMapMixedInput(rc, input); + + if (!validateServiceLink(rc)) { return; } + + final JsonObject response = processIdentityMapV3Response(rc, normalizedInput); + ResponseUtil.SuccessV2(rc, response); + } catch (ClassCastException | JsonProcessingException processingException) { + ResponseUtil.LogInfoAndSend400Response(rc, "Incorrect request format"); } catch (Exception e) { - LOGGER.error("Unknown error while mapping batched identity", e); - rc.fail(500); + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.UnknownError, 500, rc, "Unknown error while mapping identity v3", e); } } + private Map processIdentityMapMixedInput(RoutingContext rc, IdentityMapV3Request input) { + final Map normalizedIdentities = new HashMap<>(); + + var normalizedEmails = parseIdentitiesInput(input.email(), DiiType.Email, InputUtil.DiiInputType.Raw, rc); + normalizedIdentities.put("email", normalizedEmails); + + var normalizedEmailHashes = parseIdentitiesInput(input.email_hash(), DiiType.Email, InputUtil.DiiInputType.Hash, rc); + normalizedIdentities.put("email_hash", normalizedEmailHashes); + + var normalizedPhones = parseIdentitiesInput(input.phone(), DiiType.Phone, InputUtil.DiiInputType.Raw, rc); + normalizedIdentities.put("phone", normalizedPhones); + + var normalizedPhoneHashes = parseIdentitiesInput(input.phone_hash(), DiiType.Phone, InputUtil.DiiInputType.Hash, rc); + normalizedIdentities.put("phone_hash", normalizedPhoneHashes); + + return normalizedIdentities; + } + private static String getApiContact(RoutingContext rc) { String apiContact; try { @@ -1617,18 +1294,18 @@ private void recordIdentityMapStats(RoutingContext rc, int inputCount, int inval String apiContact = getApiContact(rc); DistributionSummary ds = _identityMapMetricSummaries.computeIfAbsent(apiContact, k -> DistributionSummary - .builder("uid2.operator.identity.map.inputs") + .builder("uid2_operator_identity_map_inputs") .description("number of emails or email hashes passed to identity map batch endpoint") .tags("api_contact", apiContact) .register(Metrics.globalRegistry)); ds.record(inputCount); Tuple.Tuple2 ids = _identityMapUnmappedIdentifiers.computeIfAbsent(apiContact, k -> new Tuple.Tuple2<>( - Counter.builder("uid2.operator.identity.map.unmapped") + Counter.builder("uid2_operator_identity_map_unmapped_total") .description("invalid identifiers") .tags("api_contact", apiContact, "reason", "invalid") .register(Metrics.globalRegistry), - Counter.builder("uid2.operator.identity.map.unmapped") + Counter.builder("uid2_operator_identity_map_unmapped_total") .description("optout identifiers") .tags("api_contact", apiContact, "reason", "optout") .register(Metrics.globalRegistry))); @@ -1636,7 +1313,7 @@ private void recordIdentityMapStats(RoutingContext rc, int inputCount, int inval if (optoutCount > 0) ids.getItem2().increment(optoutCount); Counter rs = _identityMapRequestWithUnmapped.computeIfAbsent(apiContact, k -> Counter - .builder("uid2.operator.identity.map.unmapped_requests") + .builder("uid2_operator_identity_map_unmapped_requests_total") .description("number of requests with unmapped identifiers") .tags("api_contact", apiContact) .register(Metrics.globalRegistry)); @@ -1655,30 +1332,30 @@ private void recordIdentityMapStatsForServiceLinks(RoutingContext rc, String api final String serviceName = rc.get(SecureLinkValidatorService.SERVICE_NAME); final String metricKey = serviceName + serviceLinkName; DistributionSummary ds = _identityMapMetricSummaries.computeIfAbsent(metricKey, - k -> DistributionSummary.builder("uid2.operator.identity.map.services.inputs") - .description("number of emails or phone numbers passed to identity map batch endpoint by services") - .tags(Arrays.asList(Tag.of("api_contact", apiContact), - Tag.of("service_name", serviceName), - Tag.of("service_link_name", serviceLinkName))) - .register(Metrics.globalRegistry)); + k -> DistributionSummary.builder("uid2_operator_identity_map_services_inputs") + .description("number of emails or phone numbers passed to identity map batch endpoint by services") + .tags(Arrays.asList(Tag.of("api_contact", apiContact), + Tag.of("service_name", serviceName), + Tag.of("service_link_name", serviceLinkName))) + .register(Metrics.globalRegistry)); ds.record(inputCount); Tuple.Tuple2 counterTuple = _identityMapUnmappedIdentifiers.computeIfAbsent(metricKey, - k -> new Tuple.Tuple2<>( - Counter.builder("uid2.operator.identity.map.services.unmapped") - .description("number of invalid identifiers passed to identity map batch endpoint by services") - .tags(Arrays.asList(Tag.of("api_contact", apiContact), - Tag.of("reason", "invalid"), - Tag.of("service_name", serviceName), - Tag.of("service_link_name", serviceLinkName))) - .register(Metrics.globalRegistry), - Counter.builder("uid2.operator.identity.map.services.unmapped") - .description("number of optout identifiers passed to identity map batch endpoint by services") - .tags(Arrays.asList(Tag.of("api_contact", apiContact), - Tag.of("reason", "optout"), - Tag.of("service_name", serviceName), - Tag.of("service_link_name", serviceLinkName))) - .register(Metrics.globalRegistry))); + k -> new Tuple.Tuple2<>( + Counter.builder("uid2_operator_identity_map_services_unmapped_total") + .description("number of invalid identifiers passed to identity map batch endpoint by services") + .tags(Arrays.asList(Tag.of("api_contact", apiContact), + Tag.of("reason", "invalid"), + Tag.of("service_name", serviceName), + Tag.of("service_link_name", serviceLinkName))) + .register(Metrics.globalRegistry), + Counter.builder("uid2_operator_identity_map_services_unmapped_total") + .description("number of optout identifiers passed to identity map batch endpoint by services") + .tags(Arrays.asList(Tag.of("api_contact", apiContact), + Tag.of("reason", "optout"), + Tag.of("service_name", serviceName), + Tag.of("service_link_name", serviceLinkName))) + .register(Metrics.globalRegistry))); if (invalidCount > 0) counterTuple.getItem1().increment(invalidCount); if (optOutCount > 0) counterTuple.getItem2().increment(optOutCount); } @@ -1687,16 +1364,16 @@ private void recordIdentityMapStatsForServiceLinks(RoutingContext rc, String api private List parseOptoutStatusRequestPayload(RoutingContext rc) { final JsonObject requestObj = (JsonObject) rc.data().get("request"); if (requestObj == null) { - ResponseUtil.Error(ResponseStatus.ClientError, HttpStatus.SC_BAD_REQUEST, rc, "Invalid request body"); + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.ClientError, HttpStatus.SC_BAD_REQUEST, rc, "Invalid request body"); return null; } final JsonArray rawUidsJsonArray = requestObj.getJsonArray("advertising_ids"); if (rawUidsJsonArray == null) { - ResponseUtil.Error(ResponseStatus.ClientError, HttpStatus.SC_BAD_REQUEST, rc, "Required Parameter Missing: advertising_ids"); + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.ClientError, HttpStatus.SC_BAD_REQUEST, rc, "Required Parameter Missing: advertising_ids"); return null; } if (rawUidsJsonArray.size() > optOutStatusMaxRequestSize) { - ResponseUtil.Error(ResponseStatus.ClientError, HttpStatus.SC_BAD_REQUEST, rc, "Request payload is too large"); + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.ClientError, HttpStatus.SC_BAD_REQUEST, rc, "Request payload is too large"); return null; } List rawUID2sInputList = new ArrayList<>(rawUidsJsonArray.size()); @@ -1730,7 +1407,7 @@ private void handleOptoutStatus(RoutingContext rc) { ResponseUtil.SuccessV2(rc, bodyJsonObj); recordOptOutStatusEndpointStats(rc, rawUID2sInput.size(), optedOutJsonArray.size()); } catch (Exception e) { - ResponseUtil.Error(ResponseStatus.UnknownError, 500, rc, + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.UnknownError, 500, rc, "Unknown error while getting optout status", e); } } @@ -1738,14 +1415,14 @@ private void handleOptoutStatus(RoutingContext rc) { private void recordOptOutStatusEndpointStats(RoutingContext rc, int inputCount, int optOutCount) { String apiContact = getApiContact(rc); DistributionSummary inputDistSummary = optOutStatusCounters.computeIfAbsent(apiContact, k -> DistributionSummary - .builder("uid2.operator.optout.status.input_size") + .builder("uid2_operator_optout_status_input_size") .description("number of UIDs received in request") .tags("api_contact", apiContact) .register(Metrics.globalRegistry)); inputDistSummary.record(inputCount); DistributionSummary optOutDistSummary = optOutStatusCounters.computeIfAbsent(apiContact, k -> DistributionSummary - .builder("uid2.operator.optout.status.optout_size") + .builder("uid2_operator_optout_status_optout_size") .description("number of UIDs that have opted out") .tags("api_contact", apiContact) .register(Metrics.globalRegistry)); @@ -1766,7 +1443,7 @@ public TokenVersion getRefreshTokenVersion(String s) { } private void recordRefreshTokenVersionCount(String siteId, TokenVersion tokenVersion) { - Counter.builder("uid2_refresh_token_received_count") + Counter.builder("uid2_refresh_token_received_count_total") .description(String.format("Counter for the amount of refresh token %s received", tokenVersion.toString().toLowerCase())) .tags("site_id", siteId) .tags("refresh_token_version", tokenVersion.toString().toLowerCase()) @@ -1774,26 +1451,31 @@ private void recordRefreshTokenVersionCount(String siteId, TokenVersion tokenVer } - private RefreshResponse refreshIdentity(RoutingContext rc, String tokenStr) { - final RefreshTokenInput refreshTokenInput; + private TokenRefreshResponse refreshIdentity(RoutingContext rc, String tokenStr) { + final TokenRefreshRequest tokenRefreshRequest; try { if (AuthMiddleware.isAuthenticated(rc)) { rc.put(Const.RoutingContextData.SiteId, AuthMiddleware.getAuthClient(ClientKey.class, rc).getSiteId()); } - refreshTokenInput = this.encoder.decodeRefreshToken(tokenStr); + tokenRefreshRequest = this.encoder.decodeRefreshToken(tokenStr); } catch (ClientInputValidationException cie) { LOGGER.warn("Failed to decode refresh token for site ID: " + rc.data().get(Const.RoutingContextData.SiteId), cie); - return RefreshResponse.Invalid; + return TokenRefreshResponse.Invalid; } - if (refreshTokenInput == null) { - return RefreshResponse.Invalid; + if (tokenRefreshRequest == null) { + return TokenRefreshResponse.Invalid; } if (!AuthMiddleware.isAuthenticated(rc)) { - rc.put(Const.RoutingContextData.SiteId, refreshTokenInput.sourcePublisher.siteId); + rc.put(Const.RoutingContextData.SiteId, tokenRefreshRequest.sourcePublisher.siteId); } recordRefreshTokenVersionCount(String.valueOf(rc.data().get(Const.RoutingContextData.SiteId)), this.getRefreshTokenVersion(tokenStr)); - return this.idService.refreshIdentity(refreshTokenInput); + RuntimeConfig config = this.getConfigFromRc(rc); + Duration refreshIdentityAfter = Duration.ofSeconds(config.getRefreshIdentityTokenAfterSeconds()); + Duration refreshExpiresAfter = Duration.ofSeconds(config.getRefreshTokenExpiresAfterSeconds()); + Duration identityExpiresAfter = Duration.ofSeconds(config.getIdentityTokenExpiresAfterSeconds()); + + return this.idService.refreshIdentity(tokenRefreshRequest, refreshIdentityAfter, refreshExpiresAfter, identityExpiresAfter); } public static String getSiteName(ISiteStore siteStore, Integer siteId) { @@ -1819,10 +1501,10 @@ private TokenResponseStatsCollector.PlatformType getPlatformType(RoutingContext return origin != null ? TokenResponseStatsCollector.PlatformType.HasOriginHeader : TokenResponseStatsCollector.PlatformType.Other; } - private void recordRefreshDurationStats(Integer siteId, String apiContact, Duration durationSinceLastRefresh, boolean hasOriginHeader) { + private void recordRefreshDurationStats(Integer siteId, String apiContact, Duration durationSinceLastRefresh, boolean hasOriginHeader, Duration identityExpiresAfter) { DistributionSummary ds = _refreshDurationMetricSummaries.computeIfAbsent(new Tuple.Tuple2<>(apiContact, hasOriginHeader), k -> DistributionSummary - .builder("uid2.token_refresh_duration_seconds") + .builder("uid2_token_refresh_duration_seconds") .description("duration between token refreshes") .tag("site_id", String.valueOf(siteId)) .tag("site_name", getSiteName(siteProvider, siteId)) @@ -1832,10 +1514,10 @@ private void recordRefreshDurationStats(Integer siteId, String apiContact, Durat ); ds.record(durationSinceLastRefresh.getSeconds()); - boolean isExpired = durationSinceLastRefresh.compareTo(this.idService.getIdentityExpiryDuration()) > 0; + boolean isExpired = durationSinceLastRefresh.compareTo(identityExpiresAfter) > 0; Counter c = _advertisingTokenExpiryStatus.computeIfAbsent(new Tuple.Tuple3<>(String.valueOf(siteId), hasOriginHeader, isExpired), k -> Counter - .builder("uid2.advertising_token_expired_on_refresh") + .builder("uid2_advertising_token_expired_on_refresh_total") .description("status of advertiser token expiry") .tag("site_id", String.valueOf(siteId)) .tag("site_name", getSiteName(siteProvider, siteId)) @@ -1846,65 +1528,52 @@ private void recordRefreshDurationStats(Integer siteId, String apiContact, Durat c.increment(); } - private InputUtil.InputVal[] createInputList(JsonArray a, boolean inputAsHash) { - if (a == null || a.size() == 0) { + private InputUtil.InputVal[] createInputList(JsonArray a, DiiType diiType, InputUtil.DiiInputType inputType) { + if (a == null || a.isEmpty()) { return new InputUtil.InputVal[0]; } final int size = a.size(); final InputUtil.InputVal[] resp = new InputUtil.InputVal[size]; - for (int i = 0; i < size; ++i) { - if (inputAsHash) { - resp[i] = InputUtil.normalizeEmailHash(a.getString(i)); - } else { - resp[i] = InputUtil.normalizeEmail(a.getString(i)); - } + for (int i = 0; i < size; i++) { + resp[i] = normalizeIdentity(a.getString(i), diiType, inputType); } + return resp; + } + private InputUtil.InputVal normalizeIdentity(String identity, DiiType diiType, + InputUtil.DiiInputType inputType) { + return switch (diiType) { + case Email -> switch (inputType) { + case Raw -> InputUtil.normalizeEmail(identity); + case Hash -> InputUtil.normalizeEmailHash(identity); + }; + case Phone -> switch (inputType) { + case Raw -> InputUtil.normalizePhone(identity); + case Hash -> InputUtil.normalizePhoneHash(identity); + }; + }; } - private InputUtil.InputVal[] createInputListV1(JsonArray a, IdentityType identityType, InputUtil.IdentityInputType inputType) { - if (a == null || a.isEmpty()) { + private InputUtil.InputVal[] parseIdentitiesInput(String[] identities, DiiType identityType, InputUtil.DiiInputType inputType + , RoutingContext rc) { + if (identities == null || identities.length == 0) { return new InputUtil.InputVal[0]; } - final int size = a.size(); - final InputUtil.InputVal[] resp = new InputUtil.InputVal[size]; + final var normalizedIdentities = new InputUtil.InputVal[identities.length]; - if (identityType == IdentityType.Email) { - if (inputType == InputUtil.IdentityInputType.Raw) { - for (int i = 0; i < size; ++i) { - resp[i] = InputUtil.normalizeEmail(a.getString(i)); - } - } else if (inputType == InputUtil.IdentityInputType.Hash) { - for (int i = 0; i < size; ++i) { - resp[i] = InputUtil.normalizeEmailHash(a.getString(i)); - } - } else { - throw new IllegalStateException("inputType"); - } - } else if (identityType == IdentityType.Phone) { - if (inputType == InputUtil.IdentityInputType.Raw) { - for (int i = 0; i < size; ++i) { - resp[i] = InputUtil.normalizePhone(a.getString(i)); - } - } else if (inputType == InputUtil.IdentityInputType.Hash) { - for (int i = 0; i < size; ++i) { - resp[i] = InputUtil.normalizePhoneHash(a.getString(i)); - } - } else { - throw new IllegalStateException("inputType"); - } - } else { - throw new IllegalStateException("identityType"); + for (int i = 0; i < identities.length; i++) { + normalizedIdentities[i] = normalizeIdentity(identities[i], identityType, inputType); } - return resp; + return normalizedIdentities; } - private UserConsentStatus validateUserConsent(JsonObject req) { - // TCF string is an optional parameter and we should only check tcf if in EUID and the string is present + private UserConsentStatus validateUserConsent(JsonObject req, String apiContact) { + // TCF string is an optional parameter, and we should only check tcf if in EUID and the string is present if (identityScope.equals(IdentityScope.EUID) && req.containsKey("tcf_consent_string")) { + recordTokenGenerateTCFUsage(apiContact); TransparentConsentParseResult tcResult = this.getUserConsentV2(req); if (!tcResult.isSuccess()) { return UserConsentStatus.INVALID; @@ -1952,7 +1621,6 @@ private boolean meetPolicyCheckRequirements(RoutingContext rc) { return true; } - private Tuple.Tuple2 readOptoutCheckPolicy(JsonObject req) { if(req.containsKey(OPTOUT_CHECK_POLICY_PARAM)) { return new Tuple.Tuple2<>(OptoutCheckPolicy.fromValue(req.getInteger(OPTOUT_CHECK_POLICY_PARAM)), OPTOUT_CHECK_POLICY_PARAM); @@ -1965,12 +1633,19 @@ private Tuple.Tuple2 readOptoutCheckPolicy(JsonObject private void recordTokenGeneratePolicy(String apiContact, OptoutCheckPolicy policy, String policyParameterKey) { _tokenGeneratePolicyCounters.computeIfAbsent(new Tuple.Tuple3<>(apiContact, policy, policyParameterKey), triple -> Counter - .builder("uid2.token_generate_policy_usage") + .builder("uid2_token_generate_policy_usage_total") .description("Counter for token generate policy usage") .tags("api_contact", triple.getItem1(), "policy", String.valueOf(triple.getItem2()), "policy_parameter", triple.getItem3()) .register(Metrics.globalRegistry)).increment(); } + private void recordTokenGenerateTCFUsage(String apiContact) { + _tokenGenerateTCFUsage.computeIfAbsent(apiContact, contact -> Counter + .builder("uid2_token_generate_tcf_usage_total") + .description("Counter for token generate tcf usage") + .tags("api_contact", contact) + .register(Metrics.globalRegistry)).increment(); + } private TransparentConsentParseResult getUserConsentV2(JsonObject req) { final String rawTcString = req.getString("tcf_consent_string"); @@ -1986,16 +1661,6 @@ private TransparentConsentParseResult getUserConsentV2(JsonObject req) { } } - private JsonObject toJsonV1(IdentityResponse t) { - final JsonObject json = new JsonObject(); - json.put("advertising_token", t.getAdvertisingToken()); - json.put("refresh_token", t.getRefreshToken()); - json.put("identity_expires", t.getIdentityExpires().toEpochMilli()); - json.put("refresh_expires", t.getRefreshExpires().toEpochMilli()); - json.put("refresh_from", t.getRefreshFrom().toEpochMilli()); - return json; - } - private static MissingAclMode getMissingAclMode(ClientKey clientKey) { return clientKey.hasRole(Role.ID_READER) ? MissingAclMode.ALLOW_ALL : MissingAclMode.DENY_ALL; } @@ -2040,25 +1705,6 @@ private static JsonObject toJson(KeysetKey key) { return json; } - private JsonObject toJson(IdentityResponse t) { - final JsonObject json = new JsonObject(); - json.put("advertisement_token", t.getAdvertisingToken()); - json.put("advertising_token", t.getAdvertisingToken()); - json.put("refresh_token", t.getRefreshToken()); - - return json; - } - - private void sendJsonResponse(RoutingContext rc, JsonObject json) { - rc.response().putHeader(HttpHeaders.CONTENT_TYPE, "application/json") - .end(json.encode()); - } - - private void sendJsonResponse(RoutingContext rc, JsonArray json) { - rc.response().putHeader(HttpHeaders.CONTENT_TYPE, "application/json") - .end(json.encode()); - } - private void logInvalidOriginOrAppName(int siteId, String originOrAppName) { siteIdToInvalidOriginsAndAppNames.computeIfAbsent(siteId, k -> new HashSet<>()) .add(originOrAppName); diff --git a/src/main/java/com/uid2/operator/vertx/V2PayloadHandler.java b/src/main/java/com/uid2/operator/vertx/V2PayloadHandler.java index 07ab3ff58..e5b336d18 100644 --- a/src/main/java/com/uid2/operator/vertx/V2PayloadHandler.java +++ b/src/main/java/com/uid2/operator/vertx/V2PayloadHandler.java @@ -1,11 +1,12 @@ package com.uid2.operator.vertx; -import com.uid2.operator.model.IdentityScope; +import com.uid2.operator.model.identities.IdentityScope; import com.uid2.operator.model.KeyManager; import com.uid2.operator.monitoring.TokenResponseStatsCollector; import com.uid2.operator.service.EncodingUtils; import com.uid2.operator.service.ResponseUtil; import com.uid2.operator.service.V2RequestUtil; +import com.uid2.operator.util.HttpMediaType; import com.uid2.shared.InstantClock; import com.uid2.shared.Utils; import com.uid2.shared.auth.ClientKey; @@ -48,10 +49,10 @@ public void handle(RoutingContext rc, Handler apiHandler) { passThrough(rc, apiHandler); return; } + V2RequestUtil.V2Request request = V2RequestUtil.parseRequest(rc, AuthMiddleware.getAuthClient(ClientKey.class, rc), new InstantClock()); - V2RequestUtil.V2Request request = V2RequestUtil.parseRequest(rc.body().asString(), AuthMiddleware.getAuthClient(ClientKey.class, rc), new InstantClock()); if (!request.isValid()) { - ResponseUtil.ClientError(rc, request.errorMessage); + ResponseUtil.LogInfoAndSend400Response(rc, request.errorMessage); return; } rc.data().put("request", request.payload); @@ -67,9 +68,9 @@ public void handleAsync(RoutingContext rc, Function apiH return; } - V2RequestUtil.V2Request request = V2RequestUtil.parseRequest(rc.body().asString(), AuthMiddleware.getAuthClient(ClientKey.class, rc), new InstantClock()); + V2RequestUtil.V2Request request = V2RequestUtil.parseRequest(rc, AuthMiddleware.getAuthClient(ClientKey.class, rc), new InstantClock()); if (!request.isValid()) { - ResponseUtil.ClientError(rc, request.errorMessage); + ResponseUtil.LogInfoAndSend400Response(rc, request.errorMessage); return; } rc.data().put("request", request.payload); @@ -85,7 +86,7 @@ public void handleTokenGenerate(RoutingContext rc, Handler apiHa return; } - V2RequestUtil.V2Request request = V2RequestUtil.parseRequest(rc.body().asString(), AuthMiddleware.getAuthClient(ClientKey.class, rc), new InstantClock()); + V2RequestUtil.V2Request request = V2RequestUtil.parseRequest(rc, AuthMiddleware.getAuthClient(ClientKey.class, rc), new InstantClock()); if (!request.isValid()) { SendClientErrorResponseAndRecordStats(ResponseUtil.ResponseStatus.ClientError, 400, rc, request.errorMessage, null, TokenResponseStatsCollector.Endpoint.GenerateV2, TokenResponseStatsCollector.ResponseStatus.BadPayload, siteProvider, TokenResponseStatsCollector.PlatformType.Other); return; @@ -110,7 +111,7 @@ public void handleTokenGenerate(RoutingContext rc, Handler apiHa } catch (Exception ex){ LOGGER.error("Failed to generate token", ex); - ResponseUtil.Error(ResponseUtil.ResponseStatus.GenericError, 500, rc, ""); + ResponseUtil.LogErrorAndSendResponse(ResponseUtil.ResponseStatus.GenericError, 500, rc, ""); } } @@ -149,21 +150,21 @@ public void handleTokenRefresh(RoutingContext rc, Handler apiHan V2RequestUtil.handleRefreshTokenInResponseBody(bodyJson, this.keyManager, this.identityScope); if (request != null) { - rc.response().putHeader(HttpHeaders.CONTENT_TYPE, "text/plain"); // Encrypt whole payload using key shared with client. byte[] encryptedResp = AesGcm.encrypt( respJson.encode().getBytes(StandardCharsets.UTF_8), request.encryptionKey); - rc.response().end(Utils.toBase64String(encryptedResp)); + + writeResponseBody(rc, encryptedResp); } else { - rc.response().putHeader(HttpHeaders.CONTENT_TYPE, "application/json") + rc.response().putHeader(HttpHeaders.CONTENT_TYPE, HttpMediaType.APPLICATION_JSON.getType()) .end(respJson.encode()); } } catch (Exception ex){ LOGGER.error("Failed to refresh token", ex); - ResponseUtil.Error(ResponseUtil.ResponseStatus.GenericError, 500, rc, ""); + ResponseUtil.LogErrorAndSendResponse(ResponseUtil.ResponseStatus.GenericError, 500, rc, ""); } } @@ -174,18 +175,33 @@ private void passThrough(RoutingContext rc, Handler apiHandler) return; } JsonObject respJson = (JsonObject) rc.data().get("response"); - rc.response().putHeader(HttpHeaders.CONTENT_TYPE, "application/json") + rc.response().putHeader(HttpHeaders.CONTENT_TYPE, HttpMediaType.APPLICATION_JSON.getType()) .end(respJson.encode()); } - private void writeResponse(RoutingContext rc, byte[] nonce, JsonObject resp, byte[] keyBytes) { + public static byte[] encryptResponse(byte[] nonce, JsonObject resp, byte[] keyBytes) { Buffer buffer = Buffer.buffer(); buffer.appendLong(EncodingUtils.NowUTCMillis().toEpochMilli()); buffer.appendBytes(nonce); buffer.appendBytes(resp.encode().getBytes(StandardCharsets.UTF_8)); - rc.response().putHeader(HttpHeaders.CONTENT_TYPE, "text/plain"); - rc.response().end(Utils.toBase64String(AesGcm.encrypt(buffer.getBytes(), keyBytes))); + byte[] response = buffer.getBytes(); + return AesGcm.encrypt(response, keyBytes); + } + + private void writeResponseBody(RoutingContext rc, byte[] response) { + if (rc.request().headers().contains(HttpHeaders.CONTENT_TYPE, HttpMediaType.APPLICATION_OCTET_STREAM.getType(), true)) { + rc.response().putHeader(HttpHeaders.CONTENT_TYPE, HttpMediaType.APPLICATION_OCTET_STREAM.getType()) + .end(Buffer.buffer(response)); + } else { + rc.response().putHeader(HttpHeaders.CONTENT_TYPE, HttpMediaType.TEXT_PLAIN.getType()) + .end(Utils.toBase64String(response)); + } + } + + private void writeResponse(RoutingContext rc, byte[] nonce, JsonObject resp, byte[] keyBytes) { + var response = encryptResponse(nonce, resp, keyBytes); + writeResponseBody(rc, response); } private void handleResponse(RoutingContext rc, V2RequestUtil.V2Request request) { @@ -199,7 +215,7 @@ private void handleResponse(RoutingContext rc, V2RequestUtil.V2Request request) writeResponse(rc, request.nonce, respJson, request.encryptionKey); } catch (Exception ex) { LOGGER.error("Failed to generate response", ex); - ResponseUtil.Error(ResponseUtil.ResponseStatus.GenericError, 500, rc, ""); + ResponseUtil.LogErrorAndSendResponse(ResponseUtil.ResponseStatus.GenericError, 500, rc, ""); } } } diff --git a/src/main/resources/com.uid2.core/test/cloud_encryption_keys/cloud_encryption_keys.json b/src/main/resources/com.uid2.core/test/cloud_encryption_keys/cloud_encryption_keys.json new file mode 100644 index 000000000..a9134f518 --- /dev/null +++ b/src/main/resources/com.uid2.core/test/cloud_encryption_keys/cloud_encryption_keys.json @@ -0,0 +1,73 @@ +[ { + "id" : 1, + "siteId" : 999, + "activates" : 1720641670, + "created" : 1720641670, + "secret" : "mydrCudb2PZOm01Qn0SpthltmexHUAA11Hy1m+uxjVw=" +}, { + "id" : 2, + "siteId" : 999, + "activates" : 1720728070, + "created" : 1720641670, + "secret" : "FtdslrFSsvVXOuhOWGwEI+0QTkCvM8SGZAP3k2u3PgY=" +}, { + "id" : 3, + "siteId" : 999, + "activates" : 1720814470, + "created" : 1720641670, + "secret" : "/7zO6QbKrhZKIV36G+cU9UR4hZUVg5bD+KjbczICjHw=" +}, { + "id" : 4, + "siteId" : 123, + "activates" : 1720641671, + "created" : 1720641671, + "secret" : "XjiqRlWQQJGLr7xfV1qbueKwyzt881GVohuUkQt/ht4=" +}, { + "id" : 5, + "siteId" : 123, + "activates" : 1720728071, + "created" : 1720641671, + "secret" : "QmpIf5NzO+UROjl5XjB/BmF6paefM8n6ub9B2plC9aI=" +}, { + "id" : 6, + "siteId" : 123, + "activates" : 1720814471, + "created" : 1720641671, + "secret" : "40w9UMSYxGm+KldOWOXhBGI8QgjvUUQjivtkP4VpKV8=" +}, { + "id" : 7, + "siteId" : 124, + "activates" : 1720641671, + "created" : 1720641671, + "secret" : "QdwD0kQV1BwmLRD0PH1YpqgaOrgpVTfu08o98mSZ6uE=" +}, { + "id" : 8, + "siteId" : 124, + "activates" : 1720728071, + "created" : 1720641671, + "secret" : "yCVCM/HLf9/6k+aUNrx7w17VbyfSzI8JykLQLSR+CW0=" +}, { + "id" : 9, + "siteId" : 124, + "activates" : 1720814471, + "created" : 1720641671, + "secret" : "JqHl8BrTyx9XpR2lYj/5xvUpzgnibGeomETTwF4rn1U=" +}, { + "id" : 10, + "siteId" : 127, + "activates" : 1720641671, + "created" : 1720641671, + "secret" : "JqiG1b34AvrdO3Aj6cCcjOBJMijrDzTmrR+p9ZtP2es=" +}, { + "id" : 11, + "siteId" : 127, + "activates" : 1720728072, + "created" : 1720641672, + "secret" : "lp1CyHdfc7K0aO5JGpA+Ve5Z/V5LImtGEQwCg/YB0kY=" +}, { + "id" : 12, + "siteId" : 127, + "activates" : 1720814472, + "created" : 1720641672, + "secret" : "G99rFYJF+dnSlk/xG6fuC3WNqQxTLJbDIdVyPMbGQ6s=" +} ] diff --git a/src/main/resources/com.uid2.core/test/cloud_encryption_keys/metadata.json b/src/main/resources/com.uid2.core/test/cloud_encryption_keys/metadata.json new file mode 100644 index 000000000..6ca4c52f0 --- /dev/null +++ b/src/main/resources/com.uid2.core/test/cloud_encryption_keys/metadata.json @@ -0,0 +1,7 @@ +{ + "version": 1, + "generated": 1620253519, + "cloud_encryption_keys": { + "location": "/com.uid2.core/test/cloud_encryption_keys/cloud_encryption_keys.json" + } +} diff --git a/src/main/resources/com.uid2.core/test/runtime_config/metadata.json b/src/main/resources/com.uid2.core/test/runtime_config/metadata.json new file mode 100644 index 000000000..42971832e --- /dev/null +++ b/src/main/resources/com.uid2.core/test/runtime_config/metadata.json @@ -0,0 +1,9 @@ +{ + "version" : 1, + "runtime_config": { + "identity_token_expires_after_seconds": 3600, + "refresh_token_expires_after_seconds": 86400, + "refresh_identity_token_after_seconds": 900, + "sharing_token_expiry_seconds": 2592000 + } +} diff --git a/src/main/resources/com.uid2.core/test/salts/metadata.json b/src/main/resources/com.uid2.core/test/salts/metadata.json index 52747666b..127e23688 100644 --- a/src/main/resources/com.uid2.core/test/salts/metadata.json +++ b/src/main/resources/com.uid2.core/test/salts/metadata.json @@ -1,13 +1,20 @@ { "version" : 1, - "generated" : 1670883129, + "generated" : 1717548362, "first_level" : "fOGY/aRE44peL23i+cE9MkJrzmEeNZZziNZBfq7qqk8=", "id_prefix" : "b", "id_secret" : "HF6Qz42HBbVHINxhh191dB09BCuTWyBkNtrNicO4ZCw=", - "salts" : [{ + "salts" : [ + { "effective" : 1670796729291, "expires" : 1766125493000, "location" : "/com.uid2.core/test/salts/salts.txt.1670796729291", "size" : 2 - }] + },{ + "effective" : 1745907348982, + "expires" : 1780620362000, + "location" : "/com.uid2.core/test/salts/salts.txt.1745907348982", + "size" : 4 + } + ] } \ No newline at end of file diff --git a/src/main/resources/com.uid2.core/test/salts/salts.txt.1670796729291 b/src/main/resources/com.uid2.core/test/salts/salts.txt.1670796729291 index 7d2003505..d438edfa0 100644 --- a/src/main/resources/com.uid2.core/test/salts/salts.txt.1670796729291 +++ b/src/main/resources/com.uid2.core/test/salts/salts.txt.1670796729291 @@ -1,2 +1,2 @@ -1000000,1614556800000,vgv1BwiNRCW7F3VcNXHlZh+7oHJ4G4gCshbGcVOLnss= -1000001,1643235130717,vgv1BwiNRCW7F3VcNXHlZh+7oHJ4G4gCshbGcVOLnsP= \ No newline at end of file +1000000,1614556800000,vgv1BwiNRCW7F3VcNXHlZh+7oHJ4G4gCshbGcVOLnss=,,,,,,,, +1000001,1643235130717,vgv1BwiNRCW7F3VcNXHlZh+7oHJ4G4gCshbGcVOLnsP=,,,,,,,, \ No newline at end of file diff --git a/src/main/resources/com.uid2.core/test/salts/salts.txt.1745907348982 b/src/main/resources/com.uid2.core/test/salts/salts.txt.1745907348982 new file mode 100644 index 000000000..26db19017 --- /dev/null +++ b/src/main/resources/com.uid2.core/test/salts/salts.txt.1745907348982 @@ -0,0 +1,4 @@ +1000000,1614556800000,vgv1BwiNRCW7F3VcNXHlZh+7oHJ4G4gCshbGcVOLnss=,,,,,,,, +1000001,1643235130717,vgv1BwiNRCW7F3VcNXHlZh+7oHJ4G4gCshbGcVOLnss=,,,,,,,, +1000002,1614556000010,vgv1BwiNRCW7F3VcNXHlZh+7oHJ4G4gCshbGcVOLnss=,,,,,,,, +1000003,1643235230717,vgv1BwiNRCW7F3VcNXHlZh+7oHJ4G4gCshbGcVOLnss=,,,,,,,, \ No newline at end of file diff --git a/src/test/java/com/uid2/operator/ApiStoreReaderTest.java b/src/test/java/com/uid2/operator/ApiStoreReaderTest.java new file mode 100644 index 000000000..8aba38250 --- /dev/null +++ b/src/test/java/com/uid2/operator/ApiStoreReaderTest.java @@ -0,0 +1,104 @@ +package com.uid2.operator; + +import com.uid2.operator.reader.ApiStoreReader; +import com.uid2.shared.cloud.DownloadCloudStorage; +import com.uid2.shared.store.CloudPath; +import com.uid2.shared.store.parser.Parser; +import com.uid2.shared.store.parser.ParsingResult; +import com.uid2.shared.store.scope.GlobalScope; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.io.InputStream; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + + class ApiStoreReaderTest { + + @Mock + private DownloadCloudStorage mockStorage; + + @Mock + private Parser> mockParser; + + private final CloudPath metadataPath = new CloudPath("test/test-metadata.json"); + private final String dataType = "test-data-type"; + private final GlobalScope scope = new GlobalScope(metadataPath); + + private ApiStoreReader> reader; + + @BeforeEach + void setUp() { + MockitoAnnotations.openMocks(this); + reader = new ApiStoreReader<>(mockStorage, scope, mockParser, dataType); + } + + @Test + void getMetadataPathReturnsPathFromScope() { + CloudPath actual = reader.getMetadataPath(); + assertThat(actual).isEqualTo(metadataPath); + } + + @Test + void loadContentThrowsExceptionWhenContentsAreNull() { + assertThatThrownBy(() -> reader.loadContent(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("No contents provided for loading data type"); + } + + @Test + void loadContentThrowsExceptionWhenArrayNotFound() { + JsonObject contents = new JsonObject(); + assertThatThrownBy(() -> reader.loadContent(contents)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("No array of type: test-data-type, found in the contents"); + } + + @Test + void loadContentSuccessfullyLoadsData() throws Exception { + JsonObject contents = new JsonObject() + .put(dataType, new JsonArray().add("value1").add("value2")); + + List expectedData = Arrays.asList(new TestData("value1"), new TestData("value2")); + when(mockParser.deserialize(any(InputStream.class))) + .thenReturn(new ParsingResult<>(expectedData, expectedData.size())); + + long count = reader.loadContent(contents); + + assertThat(count).isEqualTo(2); + assertThat(reader.getSnapshot()).isEqualTo(expectedData); + } + + private static class TestData { + private final String value; + + TestData(String value) { + this.value = value; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TestData testData = (TestData) o; + return value.equals(testData.value); + } + + @Override + public int hashCode() { + return value.hashCode(); + } + } + } + + diff --git a/src/test/java/com/uid2/operator/DomainNameCheckUtilTest.java b/src/test/java/com/uid2/operator/DomainNameCheckUtilTest.java deleted file mode 100644 index 28accb41b..000000000 --- a/src/test/java/com/uid2/operator/DomainNameCheckUtilTest.java +++ /dev/null @@ -1,88 +0,0 @@ -package com.uid2.operator; - -import com.google.common.net.InternetDomainName; -import com.uid2.operator.util.DomainNameCheckUtil; -import org.junit.jupiter.api.Test; - -import java.net.MalformedURLException; -import java.net.URL; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; - -import static com.uid2.operator.util.DomainNameCheckUtil.isDomainNameAllowed; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - - -public class DomainNameCheckUtilTest { - - @Test - public void testDomainNameCheckSuccess() throws MalformedURLException { - Set allowedDomainNamesForProd = new HashSet<>(Arrays.asList("examplewebsite.com","e-wb.org","aussiedomain.id.au")); - - //most basic examples - assertTrue(isDomainNameAllowed("http://examplewebsite.com", allowedDomainNamesForProd)); - assertTrue(isDomainNameAllowed("https://examplewebsite.com", allowedDomainNamesForProd)); - //added subdomain and port number - assertTrue(isDomainNameAllowed("https://abc.examplewebsite.com:8080", allowedDomainNamesForProd)); - //added slash - assertTrue(isDomainNameAllowed("https://abc.examplewebsite.com:8080/", allowedDomainNamesForProd)); - //domain name casing is not all lower cased - assertTrue(isDomainNameAllowed("https://abc.eXampleWebsIte.com:8080/", allowedDomainNamesForProd)); - //points to a specific file and subdirectory - assertTrue(isDomainNameAllowed("https://abc.exAmplewEbsite.com:8080/blahh/a.html", allowedDomainNamesForProd)); - - //testing a bit more weird domain name - assertTrue(isDomainNameAllowed("http://e-wb.org", allowedDomainNamesForProd)); - assertTrue(isDomainNameAllowed("https://e-wb.org", allowedDomainNamesForProd)); - //added subdomain and port number - assertTrue(isDomainNameAllowed("https://abc.e-wb.org:8080", allowedDomainNamesForProd)); - //added slash - assertTrue(isDomainNameAllowed("https://abc.e-wb.org:8080/", allowedDomainNamesForProd)); - //domain name casing is not all lower cased - assertTrue(isDomainNameAllowed("https://abc.e-wb.org:8080/", allowedDomainNamesForProd)); - //points to a specific file and subdirectory - assertTrue(isDomainNameAllowed("https://abc.e-wb.org:8080/blahh/a.html", allowedDomainNamesForProd)); - - //testing for TLD with 2 suffixes (.id.au) - assertTrue(isDomainNameAllowed("http://aussiedomain.id.au", allowedDomainNamesForProd)); - assertTrue(isDomainNameAllowed("https://aussiedomain.id.au/head.html", allowedDomainNamesForProd)); - } - - @Test - public void testDomainNameCheckFailure() throws MalformedURLException { - - Set allowedDomainNamesForProd = new HashSet<>(Arrays.asList("examplewebsite.com","e-wb.org","aussiedomain.id.au")); - - //a few malformed URLs - assertFalse(isDomainNameAllowed("examplewebsite.com", allowedDomainNamesForProd)); - assertFalse(isDomainNameAllowed("examplewebsite.com:999999", allowedDomainNamesForProd)); - assertFalse(isDomainNameAllowed("abc:examplewebsite.com", allowedDomainNamesForProd)); - assertFalse(isDomainNameAllowed("/:$2231examplewebsite.com", allowedDomainNamesForProd)); - assertFalse(isDomainNameAllowed("/:$2231examplewebsite.com/23423/sfs.html", allowedDomainNamesForProd)); - - //reject disallowed domain names - assertFalse(isDomainNameAllowed("http://boohoo.id.au", allowedDomainNamesForProd)); - assertFalse(isDomainNameAllowed("https://blah12.com", allowedDomainNamesForProd)); - assertFalse(isDomainNameAllowed("http://123.boohoo.id.au", allowedDomainNamesForProd)); - assertFalse(isDomainNameAllowed("https://456.blah12.com", allowedDomainNamesForProd)); - } - - @Test - public void testLocalhostDomainNameCheck() - { - Set allowedDomainNamesForTesting = new HashSet<>(Arrays.asList("examplewebsite.com","e-wb.org","localhost")); - //most basic examples - assertTrue(isDomainNameAllowed("http://localhost", allowedDomainNamesForTesting)); - assertTrue(isDomainNameAllowed("https://localhost:8080/", allowedDomainNamesForTesting)); - //added subdomain and port number - assertTrue(isDomainNameAllowed("https://abc.localhost:8080", allowedDomainNamesForTesting)); - //added slash - assertTrue(isDomainNameAllowed("https://abc.localhost:8080/", allowedDomainNamesForTesting)); - //domain name casing is not all lower cased - assertTrue(isDomainNameAllowed("https://abc.locaLHost:8080/", allowedDomainNamesForTesting)); - //points to a specific file and subdirectory - assertTrue(isDomainNameAllowed("https://abc.localhost:8080/blahh/a.html", allowedDomainNamesForTesting)); - } -} diff --git a/src/test/java/com/uid2/operator/EUIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/EUIDOperatorVerticleTest.java index 8d168d42e..634d8b431 100644 --- a/src/test/java/com/uid2/operator/EUIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/EUIDOperatorVerticleTest.java @@ -3,7 +3,7 @@ import com.uid2.shared.model.TokenVersion; import org.junit.jupiter.api.Test; -import com.uid2.operator.model.IdentityScope; +import com.uid2.operator.model.identities.IdentityScope; import com.uid2.shared.auth.Role; import io.vertx.core.Vertx; @@ -18,12 +18,11 @@ public class EUIDOperatorVerticleTest extends UIDOperatorVerticleTest { public EUIDOperatorVerticleTest() throws IOException { } - @Override - protected TokenVersion getTokenVersion() {return TokenVersion.V3;} - @Override protected IdentityScope getIdentityScope() { return IdentityScope.EUID; } @Override + protected boolean useRawUidV3() { return true; } + @Override protected void addAdditionalTokenGenerateParams(JsonObject payload) { if (payload != null && !payload.containsKey("tcf_consent_string")) { payload.put("tcf_consent_string", "CPehNtWPehNtWABAMBFRACBoALAAAEJAAIYgAKwAQAKgArABAAqAAA"); diff --git a/src/test/java/com/uid2/operator/EUIDOperatorVerticleV4Test.java b/src/test/java/com/uid2/operator/EUIDOperatorVerticleV4Test.java deleted file mode 100644 index fb5ff985b..000000000 --- a/src/test/java/com/uid2/operator/EUIDOperatorVerticleV4Test.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.uid2.operator; - -import com.uid2.shared.model.TokenVersion; - -import java.io.IOException; - -public class EUIDOperatorVerticleV4Test extends EUIDOperatorVerticleTest { - public EUIDOperatorVerticleV4Test() throws IOException { - } - - @Override - protected TokenVersion getTokenVersion() { - return TokenVersion.V4; - } -} diff --git a/src/test/java/com/uid2/operator/ExtendedUIDOperatorVerticle.java b/src/test/java/com/uid2/operator/ExtendedUIDOperatorVerticle.java index c90259fba..43e06dc6b 100644 --- a/src/test/java/com/uid2/operator/ExtendedUIDOperatorVerticle.java +++ b/src/test/java/com/uid2/operator/ExtendedUIDOperatorVerticle.java @@ -4,9 +4,12 @@ import com.uid2.operator.monitoring.IStatsCollectorQueue; import com.uid2.operator.service.IUIDOperatorService; import com.uid2.operator.service.SecureLinkValidatorService; +import com.uid2.operator.store.IConfigStore; import com.uid2.operator.store.IOptOutStore; import com.uid2.operator.vertx.UIDOperatorVerticle; +import com.uid2.shared.audit.UidInstanceIdProvider; import com.uid2.shared.store.*; +import com.uid2.shared.store.salt.ISaltProvider; import io.vertx.core.Handler; import io.vertx.core.json.JsonObject; @@ -17,7 +20,8 @@ //An extended UIDOperatorVerticle to expose classes for testing purposes public class ExtendedUIDOperatorVerticle extends UIDOperatorVerticle { - public ExtendedUIDOperatorVerticle(JsonObject config, + public ExtendedUIDOperatorVerticle(IConfigStore configStore, + JsonObject config, boolean clientSideTokenGenerate, ISiteStore siteProvider, IClientKeyProvider clientKeyProvider, @@ -28,8 +32,9 @@ public ExtendedUIDOperatorVerticle(JsonObject config, Clock clock, IStatsCollectorQueue statsCollectorQueue, SecureLinkValidatorService secureLinkValidationService, - Handler saltRetrievalResponseHandler) { - super(config, clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, keyManager, saltProvider, optOutStore, clock, statsCollectorQueue, secureLinkValidationService, saltRetrievalResponseHandler); + Handler saltRetrievalResponseHandler, + UidInstanceIdProvider uidInstanceIdProvider) { + super(configStore, config, clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, keyManager, saltProvider, optOutStore, clock, statsCollectorQueue, secureLinkValidationService, saltRetrievalResponseHandler, uidInstanceIdProvider); } public IUIDOperatorService getIdService() { @@ -40,10 +45,6 @@ public void setKeySharingEndpointProvideAppNames(boolean enable) { this.keySharingEndpointProvideAppNames = enable; } - public void setMaxSharingLifetimeSeconds(int maxSharingLifetimeSeconds) { - this.maxSharingLifetimeSeconds = maxSharingLifetimeSeconds; - } - public void setLastInvalidOriginProcessTime(Instant lastInvalidOriginProcessTime) { this.lastInvalidOriginProcessTime = lastInvalidOriginProcessTime; } diff --git a/src/test/java/com/uid2/operator/InputNormalizationTest.java b/src/test/java/com/uid2/operator/InputNormalizationTest.java index adbefbeae..6f825608a 100644 --- a/src/test/java/com/uid2/operator/InputNormalizationTest.java +++ b/src/test/java/com/uid2/operator/InputNormalizationTest.java @@ -71,7 +71,7 @@ public void testValidEmailNormalization() { Assert.assertEquals(normalization.getProvided(), testCase[0]); Assert.assertTrue(normalization.isValid()); Assert.assertEquals(testCase[1], normalization.getNormalized()); - Assert.assertEquals(testCase[2], EncodingUtils.toBase64String(normalization.getIdentityInput())); + Assert.assertEquals(testCase[2], EncodingUtils.toBase64String(normalization.getHashedDiiInput())); } } @@ -90,7 +90,7 @@ public void testValidHashNormalization() { Assert.assertEquals(s, normalization.getProvided()); Assert.assertTrue(normalization.isValid()); Assert.assertEquals(masterHash, normalization.getNormalized()); - Assert.assertEquals(masterHash, EncodingUtils.toBase64String(normalization.getIdentityInput())); + Assert.assertEquals(masterHash, EncodingUtils.toBase64String(normalization.getHashedDiiInput())); } } diff --git a/src/test/java/com/uid2/operator/OperatorShutdownHandlerTest.java b/src/test/java/com/uid2/operator/OperatorShutdownHandlerTest.java index e4323226c..10a00b813 100644 --- a/src/test/java/com/uid2/operator/OperatorShutdownHandlerTest.java +++ b/src/test/java/com/uid2/operator/OperatorShutdownHandlerTest.java @@ -5,6 +5,7 @@ import ch.qos.logback.core.read.ListAppender; import com.uid2.operator.service.ShutdownService; import com.uid2.operator.vertx.OperatorShutdownHandler; +import com.uid2.shared.attest.AttestationResponseCode; import io.vertx.core.Vertx; import io.vertx.junit5.VertxExtension; import io.vertx.junit5.VertxTestContext; @@ -51,17 +52,18 @@ void afterEach() throws Exception { } @Test - void shutdownOnAttest401(VertxTestContext testContext) { + void shutdownOnAttestFailure(VertxTestContext testContext) { ListAppender logWatcher = new ListAppender<>(); logWatcher.start(); ((Logger) LoggerFactory.getLogger(OperatorShutdownHandler.class)).addAppender(logWatcher); // Revoke auth try { - this.operatorShutdownHandler.handleAttestResponse(Pair.of(401, "Unauthorized")); + this.operatorShutdownHandler.handleAttestResponse(Pair.of(AttestationResponseCode.AttestationFailure, "Unauthorized")); } catch (RuntimeException e) { verify(shutdownService).Shutdown(1); - Assertions.assertTrue(logWatcher.list.get(0).getFormattedMessage().contains("core attestation failed with 401, shutting down operator, core response: ")); + String message = logWatcher.list.get(0).getFormattedMessage(); + Assertions.assertEquals("core attestation failed with AttestationFailure, shutting down operator, core response: Unauthorized", logWatcher.list.get(0).getFormattedMessage()); testContext.completeNow(); } } @@ -72,11 +74,11 @@ void shutdownOnAttestFailedTooLong(VertxTestContext testContext) { logWatcher.start(); ((Logger) LoggerFactory.getLogger(OperatorShutdownHandler.class)).addAppender(logWatcher); - this.operatorShutdownHandler.handleAttestResponse(Pair.of(500, "")); + this.operatorShutdownHandler.handleAttestResponse(Pair.of(AttestationResponseCode.RetryableFailure, "")); when(clock.instant()).thenAnswer(i -> Instant.now().plus(12, ChronoUnit.HOURS).plusSeconds(60)); try { - this.operatorShutdownHandler.handleAttestResponse(Pair.of(500, "")); + this.operatorShutdownHandler.handleAttestResponse(Pair.of(AttestationResponseCode.RetryableFailure, "")); } catch (RuntimeException e) { verify(shutdownService).Shutdown(1); Assertions.assertTrue(logWatcher.list.get(0).getFormattedMessage().contains("core attestation has been in failed state for too long. shutting down operator")); @@ -90,13 +92,13 @@ void attestRecoverOnSuccess(VertxTestContext testContext) { logWatcher.start(); ((Logger) LoggerFactory.getLogger(OperatorShutdownHandler.class)).addAppender(logWatcher); - this.operatorShutdownHandler.handleAttestResponse(Pair.of(500, "")); + this.operatorShutdownHandler.handleAttestResponse(Pair.of(AttestationResponseCode.RetryableFailure, "")); when(clock.instant()).thenAnswer(i -> Instant.now().plus(6, ChronoUnit.HOURS)); - this.operatorShutdownHandler.handleAttestResponse(Pair.of(200, "")); + this.operatorShutdownHandler.handleAttestResponse(Pair.of(AttestationResponseCode.Success, "")); when(clock.instant()).thenAnswer(i -> Instant.now().plus(12, ChronoUnit.HOURS)); assertDoesNotThrow(() -> { - this.operatorShutdownHandler.handleAttestResponse(Pair.of(500, "")); + this.operatorShutdownHandler.handleAttestResponse(Pair.of(AttestationResponseCode.RetryableFailure, "")); }); verify(shutdownService, never()).Shutdown(anyInt()); testContext.completeNow(); diff --git a/src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java b/src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java new file mode 100644 index 000000000..040f92635 --- /dev/null +++ b/src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java @@ -0,0 +1,100 @@ +package com.uid2.operator; + +import com.uid2.operator.reader.ApiStoreReader; +import com.uid2.operator.reader.RotatingCloudEncryptionKeyApiProvider; +import com.uid2.shared.model.CloudEncryptionKey; +import com.uid2.shared.store.CloudPath; +import io.vertx.core.json.JsonObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +public class RotatingCloudEncryptionKeyApiProviderTest { + @Mock + private ApiStoreReader> mockApiStoreReader; + + private RotatingCloudEncryptionKeyApiProvider rotatingCloudEncryptionKeyApiProvider; + + @BeforeEach + public void setup() { + rotatingCloudEncryptionKeyApiProvider = new RotatingCloudEncryptionKeyApiProvider(mockApiStoreReader); + } + + @Test + public void testGetMetadata() throws Exception { + JsonObject expectedMetadata = new JsonObject().put("version", 1L); + when(mockApiStoreReader.getMetadata()).thenReturn(expectedMetadata); + + JsonObject metadata = rotatingCloudEncryptionKeyApiProvider.getMetadata(); + + assertEquals(expectedMetadata, metadata); + verify(mockApiStoreReader).getMetadata(); + } + + @Test + public void testGetMetadataPath() { + CloudPath expectedPath = new CloudPath("test/path"); + when(mockApiStoreReader.getMetadataPath()).thenReturn(expectedPath); + + CloudPath path = rotatingCloudEncryptionKeyApiProvider.getMetadataPath(); + + assertEquals(expectedPath, path); + verify(mockApiStoreReader).getMetadataPath(); + } + + @Test + public void testLoadContentWithMetadata() throws Exception { + JsonObject metadata = new JsonObject(); + when(mockApiStoreReader.loadContent(metadata, "cloud_encryption_keys")).thenReturn(1L); + + long version = rotatingCloudEncryptionKeyApiProvider.loadContent(metadata); + + assertEquals(1L, version); + verify(mockApiStoreReader).loadContent(metadata, "cloud_encryption_keys"); + } + + @Test + public void testGetAll() { + Map expectedKeys = new HashMap<>(); + CloudEncryptionKey key = new CloudEncryptionKey(1, 123, 1687635529, 1687808329, "secret"); + expectedKeys.put(1, key); + when(mockApiStoreReader.getSnapshot()).thenReturn(expectedKeys); + + Map keys = rotatingCloudEncryptionKeyApiProvider.getAll(); + + assertEquals(expectedKeys, keys); + verify(mockApiStoreReader).getSnapshot(); + } + + @Test + public void testGetAllWithNullSnapshot() { + when(mockApiStoreReader.getSnapshot()).thenReturn(null); + + Map keys = rotatingCloudEncryptionKeyApiProvider.getAll(); + + assertNotNull(keys); + assertTrue(keys.isEmpty()); + verify(mockApiStoreReader).getSnapshot(); + } + + @Test + public void testLoadContent() throws Exception { + JsonObject metadata = new JsonObject().put("version", 1L); + when(mockApiStoreReader.getMetadata()).thenReturn(metadata); + when(mockApiStoreReader.loadContent(metadata, "cloud_encryption_keys")).thenReturn(1L); + + rotatingCloudEncryptionKeyApiProvider.loadContent(); + + verify(mockApiStoreReader).getMetadata(); + verify(mockApiStoreReader).loadContent(metadata, "cloud_encryption_keys"); + } +} diff --git a/src/test/java/com/uid2/operator/RuntimeConfigStoreTest.java b/src/test/java/com/uid2/operator/RuntimeConfigStoreTest.java new file mode 100644 index 000000000..fdbe3d137 --- /dev/null +++ b/src/test/java/com/uid2/operator/RuntimeConfigStoreTest.java @@ -0,0 +1,129 @@ +package com.uid2.operator; + +import com.uid2.operator.store.RuntimeConfig; +import com.uid2.operator.store.RuntimeConfigStore; +import com.uid2.shared.cloud.ICloudStorage; +import io.vertx.core.json.JsonObject; +import io.vertx.junit5.VertxExtension; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.when; + +@ExtendWith(VertxExtension.class) +public class RuntimeConfigStoreTest { + private AutoCloseable mocks; + private RuntimeConfig runtimeConfig; + private JsonObject config = new JsonObject(); + @Mock + private ICloudStorage cloudStorage; + + @BeforeEach + public void setup() { + mocks = MockitoAnnotations.openMocks(this); + setupConfig(config); + runtimeConfig = setupRuntimeConfig(config); + } + + private void setupConfig(JsonObject config) { + config.put("identity_token_expires_after_seconds", 3600); + config.put("refresh_token_expires_after_seconds", 86400); + config.put("refresh_identity_token_after_seconds", 900); + config.put("sharing_token_expiry_seconds", 2592000); + } + + private RuntimeConfig setupRuntimeConfig(JsonObject config) { + return config.mapTo(RuntimeConfig.class); + } + + @AfterEach + public void teardown() throws Exception { + mocks.close(); + } + + @Test + public void loadRuntimeConfigSingleVersion() throws Exception { + final JsonObject metadataJson = new JsonObject() + .put("version", 2) + .put("runtime_config", this.config); + + when(cloudStorage.download("metadata")) + .thenReturn(new ByteArrayInputStream(metadataJson.toString().getBytes(StandardCharsets.US_ASCII))); + + RuntimeConfigStore runtimeConfigStore = new RuntimeConfigStore(cloudStorage, "metadata"); + + final JsonObject loadedMetadata = runtimeConfigStore.getMetadata(); + runtimeConfigStore.loadContent(loadedMetadata); + assertEquals(2, runtimeConfigStore.getVersion(loadedMetadata)); + assertThat(runtimeConfigStore.getConfig()) + .usingRecursiveComparison() + .isEqualTo(this.runtimeConfig); + } + + @Test + public void testFirstInvalidConfigThrowsRuntimeException() throws Exception { + JsonObject invalidConfig = new JsonObject() + .put("identity_token_expires_after_seconds", 1000) + .put("refresh_token_expires_after_seconds", 2000); + + final JsonObject metadataJson = new JsonObject() + .put("version", 1) + .put("runtime_config", invalidConfig); + + when(cloudStorage.download("metadata")) + .thenReturn(new ByteArrayInputStream(metadataJson.toString().getBytes(StandardCharsets.US_ASCII))); + + RuntimeConfigStore runtimeConfigStore = new RuntimeConfigStore(cloudStorage, "metadata"); + + final JsonObject loadedMetadata = runtimeConfigStore.getMetadata(); + assertThrows(RuntimeException.class, () -> { + runtimeConfigStore.loadContent(loadedMetadata); + }, "Expected a RuntimeException but the creation succeeded"); + } + + @Test + public void testInvalidConfigRevertsToPrevious() throws Exception { + JsonObject invalidConfig = new JsonObject() + .put("identity_token_expires_after_seconds", 1000) + .put("refresh_token_expires_after_seconds", 2000); + + final JsonObject v1MetadataJson = new JsonObject() + .put("version", 1) + .put("runtime_config", this.config); + final JsonObject v2MetadataJson = new JsonObject() + .put("version", 2) + .put("runtime_config", invalidConfig); + + RuntimeConfigStore runtimeConfigStore = new RuntimeConfigStore(cloudStorage, "metadata"); + + // First call, return valid config + when(cloudStorage.download("metadata")) + .thenReturn(new ByteArrayInputStream(v1MetadataJson.toString().getBytes(StandardCharsets.US_ASCII))); + + final JsonObject loadedMetadata1 = runtimeConfigStore.getMetadata(); + runtimeConfigStore.loadContent(loadedMetadata1); + assertEquals(1, runtimeConfigStore.getVersion(loadedMetadata1)); + assertThat(runtimeConfigStore.getConfig()) + .usingRecursiveComparison() + .isEqualTo(this.runtimeConfig); + + // Second call, return invalid config + when(cloudStorage.download("metadata")) + .thenReturn(new ByteArrayInputStream(v2MetadataJson.toString().getBytes(StandardCharsets.US_ASCII))); + + final JsonObject loadedMetadata2 = runtimeConfigStore.getMetadata(); + assertThrows(IllegalArgumentException.class, () -> runtimeConfigStore.loadContent(loadedMetadata2)); + assertThat(runtimeConfigStore.getConfig()) + .usingRecursiveComparison() + .isEqualTo(this.runtimeConfig); + } +} diff --git a/src/test/java/com/uid2/operator/StatsCollectorVerticleTest.java b/src/test/java/com/uid2/operator/StatsCollectorVerticleTest.java index d0f089da4..c383804de 100644 --- a/src/test/java/com/uid2/operator/StatsCollectorVerticleTest.java +++ b/src/test/java/com/uid2/operator/StatsCollectorVerticleTest.java @@ -12,6 +12,7 @@ import io.vertx.junit5.VertxExtension; import io.vertx.junit5.VertxTestContext; import static org.assertj.core.api.Assertions.*; +import static org.junit.Assert.assertEquals; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -19,6 +20,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.slf4j.LoggerFactory; +import java.util.Arrays; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -132,7 +134,7 @@ void allValidPathsAllowed(Vertx vertx, VertxTestContext testContext) throws Inte var messages = getMessages(); for(String endpoint: validEndpoints) { String withoutVersion = endpoint; - if (endpoint.startsWith("/v1/") || endpoint.startsWith("/v2/")) { + if (endpoint.startsWith("/v1/") || endpoint.startsWith("/v2/") || endpoint.startsWith("/v3/")) { withoutVersion = endpoint.substring(4); } else if (endpoint.startsWith("/")) { withoutVersion = endpoint.substring(1); diff --git a/src/test/java/com/uid2/operator/TokenEncodingTest.java b/src/test/java/com/uid2/operator/TokenEncodingTest.java index b53f79b4f..110fa9aee 100644 --- a/src/test/java/com/uid2/operator/TokenEncodingTest.java +++ b/src/test/java/com/uid2/operator/TokenEncodingTest.java @@ -1,11 +1,14 @@ package com.uid2.operator; import com.uid2.operator.model.*; -import com.uid2.operator.model.userIdentity.FirstLevelHashIdentity; -import com.uid2.operator.model.userIdentity.RawUidIdentity; +import com.uid2.operator.model.identities.DiiType; +import com.uid2.operator.model.identities.FirstLevelHash; +import com.uid2.operator.model.identities.IdentityScope; +import com.uid2.operator.model.identities.RawUid; import com.uid2.operator.service.EncodingUtils; import com.uid2.operator.service.EncryptedTokenEncoder; import com.uid2.operator.service.TokenUtils; +import com.uid2.operator.util.PrivacyBits; import com.uid2.shared.Const.Data; import com.uid2.shared.model.TokenVersion; import com.uid2.shared.store.CloudPath; @@ -18,6 +21,7 @@ import io.vertx.core.json.JsonObject; import org.junit.Assert; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.EnumSource; import java.time.Instant; @@ -52,71 +56,82 @@ public void testRefreshTokenEncoding(TokenVersion tokenVersion) { final EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(this.keyManager); final Instant now = EncodingUtils.NowUTCMillis(); - final byte[] firstLevelHash = TokenUtils.getFirstLevelHashFromIdentity("test@example.com", "some-salt"); + final byte[] firstLevelHash = TokenUtils.getFirstLevelHashFromRawDii("test@example.com", "some-salt"); - final RefreshTokenInput token = new RefreshTokenInput(tokenVersion, + final TokenRefreshRequest tokenRefreshRequest = new TokenRefreshRequest(tokenVersion, now, now.plusSeconds(360), new OperatorIdentity(101, OperatorType.Service, 102, 103), new SourcePublisher(111, 112, 113), - new FirstLevelHashIdentity(IdentityScope.UID2, IdentityType.Email, firstLevelHash, 121, now, now.minusSeconds(122)) + new FirstLevelHash(IdentityScope.UID2, DiiType.Email, firstLevelHash, now), + PrivacyBits.fromInt(121) ); if (tokenVersion == TokenVersion.V4) { - Assert.assertThrows(Exception.class, () -> encoder.encodeIntoRefreshToken(token, now)); + Assert.assertThrows(Exception.class, () -> encoder.encodeIntoRefreshToken(tokenRefreshRequest, now)); return; //V4 not supported for RefreshTokens } - final byte[] encodedBytes = encoder.encodeIntoRefreshToken(token, now); - final RefreshTokenInput decoded = encoder.decodeRefreshToken(EncodingUtils.toBase64String(encodedBytes)); + final byte[] encodedBytes = encoder.encodeIntoRefreshToken(tokenRefreshRequest, now); + final TokenRefreshRequest decoded = encoder.decodeRefreshToken(EncodingUtils.toBase64String(encodedBytes)); assertEquals(tokenVersion, decoded.version); - assertEquals(token.createdAt, decoded.createdAt); + assertEquals(tokenRefreshRequest.createdAt, decoded.createdAt); int addSeconds = (tokenVersion == TokenVersion.V2) ? 60 : 0; //todo: why is there a 60 second buffer in encodeV2() but not in encodeV3()? - assertEquals(token.expiresAt.plusSeconds(addSeconds), decoded.expiresAt); - assertTrue(token.firstLevelHashIdentity.matches(decoded.firstLevelHashIdentity)); - assertEquals(token.firstLevelHashIdentity.privacyBits, decoded.firstLevelHashIdentity.privacyBits); - assertEquals(token.firstLevelHashIdentity.establishedAt, decoded.firstLevelHashIdentity.establishedAt); - assertEquals(token.sourcePublisher.siteId, decoded.sourcePublisher.siteId); + assertEquals(tokenRefreshRequest.expiresAt.plusSeconds(addSeconds), decoded.expiresAt); + assertTrue(tokenRefreshRequest.firstLevelHash.matches(decoded.firstLevelHash)); + assertEquals(tokenRefreshRequest.privacyBits, decoded.privacyBits); + assertEquals(tokenRefreshRequest.firstLevelHash.establishedAt(), decoded.firstLevelHash.establishedAt()); + assertEquals(tokenRefreshRequest.sourcePublisher.siteId, decoded.sourcePublisher.siteId); Buffer b = Buffer.buffer(encodedBytes); int keyId = b.getInt(tokenVersion == TokenVersion.V2 ? 25 : 2); assertEquals(Data.RefreshKeySiteId, keyManager.getSiteIdFromKeyId(keyId)); assertNotNull(Metrics.globalRegistry - .get("uid2_refresh_token_served_count") + .get("uid2_refresh_token_served_count_total") .counter()); } @ParameterizedTest - @EnumSource(TokenVersion.class) - public void testAdvertisingTokenEncodings(TokenVersion tokenVersion) { + @CsvSource({"false, V4", //same as current UID2 prod (as at 2024-12-10) + "true, V4", //same as current EUID prod (as at 2024-12-10) + //the following combinations aren't used in any UID2/EUID environments but just testing them regardless + "false, V3", + "true, V3", + "false, V2", + "true, V2", + } + ) + public void testAdvertisingTokenEncodings(boolean useRawUIDv3, TokenVersion adTokenVersion) { final EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(this.keyManager); final Instant now = EncodingUtils.NowUTCMillis(); - final byte[] rawUid = UIDOperatorVerticleTest.getRawUid(IdentityType.Email, "test@example.com", IdentityScope.UID2, tokenVersion != TokenVersion.V2); + final byte[] rawUid = UIDOperatorVerticleTest.getRawUid(DiiType.Email, "test@example.com", IdentityScope.UID2, useRawUIDv3); - final AdvertisingTokenInput token = new AdvertisingTokenInput( - tokenVersion, + final AdvertisingTokenRequest adTokenRequest = new AdvertisingTokenRequest( + adTokenVersion, now, now.plusSeconds(60), new OperatorIdentity(101, OperatorType.Service, 102, 103), new SourcePublisher(111, 112, 113), - new RawUidIdentity(IdentityScope.UID2, IdentityType.Email, rawUid, 121, now, now.minusSeconds(122)) + new RawUid(IdentityScope.UID2, DiiType.Email, rawUid), + PrivacyBits.fromInt(121), + now ); - final byte[] encodedBytes = encoder.encodeIntoAdvertisingToken(token, now); - final AdvertisingTokenInput decoded = encoder.decodeAdvertisingToken(EncryptedTokenEncoder.bytesToBase64Token(encodedBytes, tokenVersion)); + final byte[] encodedBytes = encoder.encodeIntoAdvertisingToken(adTokenRequest, now); + final AdvertisingTokenRequest decoded = encoder.decodeAdvertisingToken(EncryptedTokenEncoder.bytesToBase64Token(encodedBytes, adTokenVersion)); - assertEquals(tokenVersion, decoded.version); - assertEquals(token.createdAt, decoded.createdAt); - assertEquals(token.expiresAt, decoded.expiresAt); - assertTrue(token.rawUidIdentity.matches(decoded.rawUidIdentity)); - assertEquals(token.rawUidIdentity.privacyBits, decoded.rawUidIdentity.privacyBits); - assertEquals(token.rawUidIdentity.establishedAt, decoded.rawUidIdentity.establishedAt); - assertEquals(token.sourcePublisher.siteId, decoded.sourcePublisher.siteId); + assertEquals(adTokenVersion, decoded.version); + assertEquals(adTokenRequest.createdAt, decoded.createdAt); + assertEquals(adTokenRequest.expiresAt, decoded.expiresAt); + assertTrue(adTokenRequest.rawUid.matches(decoded.rawUid)); + assertEquals(adTokenRequest.privacyBits, decoded.privacyBits); + assertEquals(adTokenRequest.establishedAt, decoded.establishedAt); + assertEquals(adTokenRequest.sourcePublisher.siteId, decoded.sourcePublisher.siteId); Buffer b = Buffer.buffer(encodedBytes); - int keyId = b.getInt(tokenVersion == TokenVersion.V2 ? 1 : 2); //TODO - extract master key from token should be a helper function + int keyId = b.getInt(adTokenVersion == TokenVersion.V2 ? 1 : 2); //TODO - extract master key from token should be a helper function assertEquals(Data.MasterKeySiteId, keyManager.getSiteIdFromKeyId(keyId)); } } diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index 5196dfaf2..2e0014567 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -1,34 +1,46 @@ package com.uid2.operator; import com.uid2.operator.model.*; -import com.uid2.operator.model.userIdentity.FirstLevelHashIdentity; -import com.uid2.operator.model.userIdentity.HashedDiiIdentity; -import com.uid2.operator.model.userIdentity.UserIdentity; +import com.uid2.operator.model.identities.*; import com.uid2.operator.service.*; +import com.uid2.operator.service.EncodingUtils; +import com.uid2.operator.service.EncryptedTokenEncoder; +import com.uid2.operator.service.InputUtil; +import com.uid2.operator.service.UIDOperatorService; import com.uid2.operator.store.IOptOutStore; +import com.uid2.operator.util.PrivacyBits; import com.uid2.operator.vertx.OperatorShutdownHandler; +import com.uid2.shared.audit.UidInstanceIdProvider; +import com.uid2.shared.model.SaltEntry; import com.uid2.shared.store.CloudPath; -import com.uid2.shared.store.RotatingSaltProvider; +import com.uid2.shared.store.salt.ISaltProvider; +import com.uid2.shared.store.salt.RotatingSaltProvider; import com.uid2.shared.cloud.EmbeddedResourceStorage; import com.uid2.shared.store.reader.RotatingKeysetKeyStore; import com.uid2.shared.store.reader.RotatingKeysetProvider; import com.uid2.shared.store.scope.GlobalScope; import com.uid2.shared.model.TokenVersion; +import io.vertx.core.Handler; import io.vertx.core.json.JsonObject; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; + + +import static com.uid2.operator.service.TokenUtils.getFirstLevelHashFromHashedDii; +import static com.uid2.operator.Const.Config.IdentityV3Prop; +import static java.time.temporal.ChronoUnit.DAYS; import static org.junit.jupiter.api.Assertions.*; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mock; import org.mockito.MockitoAnnotations; import java.nio.charset.StandardCharsets; import java.security.Security; -import java.time.Clock; -import java.time.Instant; +import java.time.*; import java.time.temporal.ChronoUnit; import static org.mockito.ArgumentMatchers.any; @@ -40,16 +52,25 @@ public class UIDOperatorServiceTest { @Mock private Clock clock; @Mock private OperatorShutdownHandler shutdownHandler; EncryptedTokenEncoder tokenEncoder; + UidInstanceIdProvider uidInstanceIdProvider; JsonObject uid2Config; JsonObject euidConfig; - UIDOperatorService uid2Service; - UIDOperatorService euidService; + ExtendedUIDOperatorService uid2Service; + ExtendedUIDOperatorService euidService; Instant now; - + RotatingSaltProvider saltProvider; final int IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS = 600; final int REFRESH_TOKEN_EXPIRES_AFTER_SECONDS = 900; final int REFRESH_IDENTITY_TOKEN_AFTER_SECONDS = 300; + final String FIRST_LEVEL_SALT = "first-level-salt"; + + class ExtendedUIDOperatorService extends UIDOperatorService { + public ExtendedUIDOperatorService(IOptOutStore optOutStore, ISaltProvider saltProvider, EncryptedTokenEncoder encoder, Clock clock, IdentityScope identityScope, Handler saltRetrievalResponseHandler, boolean identityV3Enabled, UidInstanceIdProvider uidInstanceIdProvider) { + super(optOutStore, saltProvider, encoder, clock, identityScope, saltRetrievalResponseHandler, identityV3Enabled, uidInstanceIdProvider); + } + } + @BeforeEach void setup() throws Exception { mocks = MockitoAnnotations.openMocks(this); @@ -66,7 +87,7 @@ void setup() throws Exception { new GlobalScope(new CloudPath("/com.uid2.core/test/keysets/metadata.json"))); keysetProvider.loadContent(); - RotatingSaltProvider saltProvider = new RotatingSaltProvider( + saltProvider = new RotatingSaltProvider( new EmbeddedResourceStorage(Main.class), "/com.uid2.core/test/salts/metadata.json"); saltProvider.loadContent(); @@ -79,37 +100,36 @@ void setup() throws Exception { uid2Config.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); uid2Config.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); uid2Config.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); - uid2Config.put("advertising_token_v4_percentage", 0); - uid2Config.put("site_ids_using_v4_tokens", "127,128"); - uid2Config.put("advertising_token_v3", false); // prod is using v2 token version for now - uid2Config.put("identity_v3", false); + uid2Config.put(IdentityV3Prop, false); - uid2Service = new UIDOperatorService( - uid2Config, + uidInstanceIdProvider = new UidInstanceIdProvider("test-instance", "id"); + + uid2Service = new ExtendedUIDOperatorService( optOutStore, saltProvider, tokenEncoder, this.clock, IdentityScope.UID2, - this.shutdownHandler::handleSaltRetrievalResponse + this.shutdownHandler::handleSaltRetrievalResponse, + uid2Config.getBoolean(IdentityV3Prop), + uidInstanceIdProvider ); euidConfig = new JsonObject(); euidConfig.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); euidConfig.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); euidConfig.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); - euidConfig.put("advertising_token_v4_percentage", 0); - euidConfig.put("advertising_token_v3", true); - euidConfig.put("identity_v3", true); + euidConfig.put(IdentityV3Prop, true); - euidService = new UIDOperatorService( - euidConfig, + euidService = new ExtendedUIDOperatorService( optOutStore, saltProvider, tokenEncoder, this.clock, IdentityScope.EUID, - this.shutdownHandler::handleSaltRetrievalResponse + this.shutdownHandler::handleSaltRetrievalResponse, + euidConfig.getBoolean(IdentityV3Prop), + uidInstanceIdProvider ); } @@ -123,111 +143,179 @@ private void setNow(Instant now) { when(clock.instant()).thenAnswer(i -> this.now); } - private HashedDiiIdentity createHashedDiiIdentity(String rawIdentityHash, IdentityScope scope, IdentityType type) { - return new HashedDiiIdentity( + private RotatingSaltProvider.SaltSnapshot setUpMockSalts() { + RotatingSaltProvider saltProvider = mock(RotatingSaltProvider.class); + RotatingSaltProvider.SaltSnapshot saltSnapshot = mock(RotatingSaltProvider.SaltSnapshot.class); + when(saltProvider.getSnapshot(any())).thenReturn(saltSnapshot); + when(saltSnapshot.getExpires()).thenReturn(Instant.now().plus(1, ChronoUnit.HOURS)); + when(saltSnapshot.getFirstLevelSalt()).thenReturn(FIRST_LEVEL_SALT); + + uid2Service = new ExtendedUIDOperatorService( + optOutStore, + saltProvider, + tokenEncoder, + this.clock, + IdentityScope.UID2, + this.shutdownHandler::handleSaltRetrievalResponse, + uid2Config.getBoolean(IdentityV3Prop), + uidInstanceIdProvider + ); + + return saltSnapshot; + } + + private HashedDii createHashedDii(String hashedDii, IdentityScope scope, DiiType type) { + return new HashedDii( scope, type, - rawIdentityHash.getBytes(StandardCharsets.UTF_8), - 0, - this.now.minusSeconds(234), - this.now.plusSeconds(12345) + hashedDii.getBytes(StandardCharsets.UTF_8) ); } - private AdvertisingTokenInput validateAndGetToken(EncryptedTokenEncoder tokenEncoder, String advertisingTokenString, IdentityScope scope, IdentityType type, int siteId) { - TokenVersion tokenVersion = (scope == IdentityScope.UID2) ? uid2Service.getAdvertisingTokenVersionForTests(siteId) : euidService.getAdvertisingTokenVersionForTests(siteId); - UIDOperatorVerticleTest.validateAdvertisingToken(advertisingTokenString, tokenVersion, scope, type); + private AdvertisingTokenRequest validateAndGetToken(EncryptedTokenEncoder tokenEncoder, String advertisingTokenString, IdentityScope scope, DiiType type, int siteId) { + UIDOperatorVerticleTest.validateAdvertisingToken(advertisingTokenString, TokenVersion.V4, scope, type); return tokenEncoder.decodeAdvertisingToken(advertisingTokenString); } - private void assertIdentityScopeIdentityTypeAndEstablishedAt(UserIdentity expctedValues, - UserIdentity actualValues) { - assertEquals(expctedValues.identityScope, actualValues.identityScope); - assertEquals(expctedValues.identityType, actualValues.identityType); - assertEquals(expctedValues.establishedAt, actualValues.establishedAt); + private void assertIdentityScopeIdentityType(IdentityScope expectedScope, DiiType expectedDiiType, + HashedDii hashedDii) { + assertEquals(expectedScope, hashedDii.identityScope()); + assertEquals(expectedDiiType, hashedDii.diiType()); + } + + private void assertIdentityScopeIdentityType(IdentityScope expectedScope, DiiType expectedDiiType, + RawUid rawUid) { + assertEquals(expectedScope, rawUid.identityScope()); + assertEquals(expectedDiiType, rawUid.diiType()); + } + + private void assertIdentityScopeIdentityType(IdentityScope expectedScope, DiiType expectedDiiType, + FirstLevelHash firstLevelHash) { + assertEquals(expectedScope, firstLevelHash.identityScope()); + assertEquals(expectedDiiType, firstLevelHash.diiType()); } + + + + @ParameterizedTest - @CsvSource({"123, V2","127, V4","128, V4"}) //site id 127 and 128 is for testing "site_ids_using_v4_tokens" + @CsvSource({"123, V4","127, V4","128, V4"}) public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { - final IdentityRequest identityRequest = new IdentityRequest( + IdentityScope expectedIdentityScope = IdentityScope.UID2; + DiiType expectedDiiType = DiiType.Email; + + + final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(siteId, 124, 125), - createHashedDiiIdentity("test-email-hash", IdentityScope.UID2, IdentityType.Email), - OptoutCheckPolicy.DoNotRespect + createHashedDii("test-email-hash", expectedIdentityScope, expectedDiiType), + OptoutCheckPolicy.DoNotRespect, PrivacyBits.fromInt(0), + this.now.minusSeconds(234) ); - final IdentityResponse identityResponse = uid2Service.generateIdentity(identityRequest); + final TokenGenerateResponse tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(identityResponse); + assertNotNull(tokenGenerateResponse); - UIDOperatorVerticleTest.validateAdvertisingToken(identityResponse.getAdvertisingToken(), tokenVersion, IdentityScope.UID2, IdentityType.Email); - AdvertisingTokenInput advertisingTokenInput = tokenEncoder.decodeAdvertisingToken(identityResponse.getAdvertisingToken());assertEquals(this.now.plusSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS), advertisingTokenInput.expiresAt); - assertEquals(identityRequest.sourcePublisher.siteId, advertisingTokenInput.sourcePublisher.siteId); - assertIdentityScopeIdentityTypeAndEstablishedAt(identityRequest.hashedDiiIdentity, advertisingTokenInput.rawUidIdentity); + UIDOperatorVerticleTest.validateAdvertisingToken(tokenGenerateResponse.getAdvertisingToken(), tokenVersion, IdentityScope.UID2, DiiType.Email); + AdvertisingTokenRequest advertisingTokenRequest = tokenEncoder.decodeAdvertisingToken(tokenGenerateResponse.getAdvertisingToken()); + assertEquals(this.now.plusSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS), advertisingTokenRequest.expiresAt); + assertEquals(tokenGenerateRequest.sourcePublisher.siteId, advertisingTokenRequest.sourcePublisher.siteId); + assertIdentityScopeIdentityType(expectedIdentityScope, expectedDiiType, + advertisingTokenRequest.rawUid); + assertEquals(tokenGenerateRequest.establishedAt, advertisingTokenRequest.establishedAt); + assertEquals(tokenGenerateRequest.privacyBits, advertisingTokenRequest.privacyBits); + + TokenRefreshRequest tokenRefreshRequest = tokenEncoder.decodeRefreshToken(tokenGenerateResponse.getRefreshToken()); + assertEquals(this.now, tokenRefreshRequest.createdAt); + assertEquals(this.now.plusSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), tokenRefreshRequest.expiresAt); + assertEquals(tokenGenerateRequest.sourcePublisher.siteId, tokenRefreshRequest.sourcePublisher.siteId); + assertIdentityScopeIdentityType(expectedIdentityScope, expectedDiiType, tokenRefreshRequest.firstLevelHash); + assertEquals(tokenGenerateRequest.establishedAt, tokenRefreshRequest.firstLevelHash.establishedAt()); + + final byte[] firstLevelHash = getFirstLevelHashFromHashedDii(tokenGenerateRequest.hashedDii.hashedDii(), + saltProvider.getSnapshot(this.now).getFirstLevelSalt() ); + assertArrayEquals(firstLevelHash, tokenRefreshRequest.firstLevelHash.firstLevelHash()); - RefreshTokenInput refreshTokenInput = tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); - assertEquals(this.now, refreshTokenInput.createdAt); - assertEquals(this.now.plusSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), refreshTokenInput.expiresAt); - assertEquals(identityRequest.sourcePublisher.siteId, refreshTokenInput.sourcePublisher.siteId); - assertIdentityScopeIdentityTypeAndEstablishedAt(identityRequest.hashedDiiIdentity, refreshTokenInput.firstLevelHashIdentity); setNow(Instant.now().plusSeconds(200)); reset(shutdownHandler); - final RefreshResponse refreshResponse = uid2Service.refreshIdentity(refreshTokenInput); + final TokenRefreshResponse refreshResponse = uid2Service.refreshIdentity(tokenRefreshRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); assertNotNull(refreshResponse); - assertEquals(RefreshResponse.Status.Refreshed, refreshResponse.getStatus()); + assertEquals(TokenRefreshResponse.Status.Refreshed, refreshResponse.getStatus()); assertNotNull(refreshResponse.getIdentityResponse()); - UIDOperatorVerticleTest.validateAdvertisingToken(refreshResponse.getIdentityResponse().getAdvertisingToken(), tokenVersion, IdentityScope.UID2, IdentityType.Email); - AdvertisingTokenInput advertisingTokenInput2 = tokenEncoder.decodeAdvertisingToken(refreshResponse.getIdentityResponse().getAdvertisingToken()); - assertEquals(this.now.plusSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS), advertisingTokenInput2.expiresAt); - assertEquals(advertisingTokenInput.sourcePublisher.siteId, advertisingTokenInput2.sourcePublisher.siteId); - assertIdentityScopeIdentityTypeAndEstablishedAt(advertisingTokenInput.rawUidIdentity, - advertisingTokenInput2.rawUidIdentity); - assertArrayEquals(advertisingTokenInput.rawUidIdentity.rawUid, - advertisingTokenInput2.rawUidIdentity.rawUid); - - RefreshTokenInput refreshTokenInput2 = tokenEncoder.decodeRefreshToken(refreshResponse.getIdentityResponse().getRefreshToken()); - assertEquals(this.now, refreshTokenInput2.createdAt); - assertEquals(this.now.plusSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), refreshTokenInput2.expiresAt); - assertEquals(refreshTokenInput.sourcePublisher.siteId, refreshTokenInput2.sourcePublisher.siteId); - assertIdentityScopeIdentityTypeAndEstablishedAt(refreshTokenInput.firstLevelHashIdentity, refreshTokenInput2.firstLevelHashIdentity); - assertArrayEquals(refreshTokenInput.firstLevelHashIdentity.firstLevelHash, refreshTokenInput2.firstLevelHashIdentity.firstLevelHash); + UIDOperatorVerticleTest.validateAdvertisingToken(refreshResponse.getIdentityResponse().getAdvertisingToken(), tokenVersion, IdentityScope.UID2, DiiType.Email); + AdvertisingTokenRequest advertisingTokenRequest2 = tokenEncoder.decodeAdvertisingToken(refreshResponse.getIdentityResponse().getAdvertisingToken()); + assertEquals(this.now.plusSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS), advertisingTokenRequest2.expiresAt); + assertEquals(advertisingTokenRequest.sourcePublisher.siteId, advertisingTokenRequest2.sourcePublisher.siteId); + assertIdentityScopeIdentityType(expectedIdentityScope, expectedDiiType, + advertisingTokenRequest2.rawUid); + assertEquals(advertisingTokenRequest.establishedAt, advertisingTokenRequest2.establishedAt); + assertArrayEquals(advertisingTokenRequest.rawUid.rawUid(), + advertisingTokenRequest2.rawUid.rawUid()); + assertEquals(tokenGenerateRequest.privacyBits, advertisingTokenRequest2.privacyBits); + + TokenRefreshRequest tokenRefreshRequest2 = tokenEncoder.decodeRefreshToken(refreshResponse.getIdentityResponse().getRefreshToken()); + assertEquals(this.now, tokenRefreshRequest2.createdAt); + assertEquals(this.now.plusSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), tokenRefreshRequest2.expiresAt); + assertEquals(tokenRefreshRequest.sourcePublisher.siteId, tokenRefreshRequest2.sourcePublisher.siteId); + assertIdentityScopeIdentityType(expectedIdentityScope, expectedDiiType, tokenRefreshRequest2.firstLevelHash); + assertEquals(tokenRefreshRequest.firstLevelHash.establishedAt(), tokenRefreshRequest2.firstLevelHash.establishedAt()); + assertArrayEquals(tokenRefreshRequest.firstLevelHash.firstLevelHash(), tokenRefreshRequest2.firstLevelHash.firstLevelHash()); + assertArrayEquals(firstLevelHash, tokenRefreshRequest2.firstLevelHash.firstLevelHash()); } @Test public void testTestOptOutKey_DoNotRespectOptout() { final InputUtil.InputVal inputVal = InputUtil.normalizeEmail(IdentityConst.OptOutIdentityForEmail); - final IdentityRequest identityRequest = new IdentityRequest( + final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(IdentityScope.UID2, 0, this.now), - OptoutCheckPolicy.DoNotRespect + inputVal.toHashedDii(IdentityScope.UID2), + OptoutCheckPolicy.DoNotRespect, PrivacyBits.fromInt(0), this.now ); - final IdentityResponse identityResponse = uid2Service.generateIdentity(identityRequest); + + final TokenGenerateResponse tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(identityResponse); - assertFalse(identityResponse.isOptedOut()); - - final RefreshTokenInput refreshTokenInput = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); - assertEquals(RefreshResponse.Optout, uid2Service.refreshIdentity(refreshTokenInput)); + assertNotNull(tokenGenerateResponse); + assertFalse(tokenGenerateResponse.isOptedOut()); + + final TokenRefreshRequest tokenRefreshRequest = this.tokenEncoder.decodeRefreshToken(tokenGenerateResponse.getRefreshToken()); + assertEquals(TokenRefreshResponse.Optout, uid2Service.refreshIdentity(tokenRefreshRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS))); } @Test public void testTestOptOutKey_RespectOptout() { final InputUtil.InputVal inputVal = InputUtil.normalizeEmail(IdentityConst.OptOutIdentityForEmail); - final IdentityRequest identityRequest = new IdentityRequest( + final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(IdentityScope.UID2, 0, this.now), - OptoutCheckPolicy.RespectOptOut + inputVal.toHashedDii(IdentityScope.UID2), + OptoutCheckPolicy.RespectOptOut, PrivacyBits.fromInt(0), this.now ); - final IdentityResponse identityResponse = uid2Service.generateIdentity(identityRequest); - assertTrue(identityResponse.isOptedOut()); + + final TokenGenerateResponse tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); + assertTrue(tokenGenerateResponse.isOptedOut()); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); } @@ -237,19 +325,25 @@ public void testTestOptOutKeyIdentityScopeMismatch() { final String email = "optout@example.com"; final InputUtil.InputVal inputVal = InputUtil.normalizeEmail(email); - final IdentityRequest identityRequest = new IdentityRequest( + final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(IdentityScope.EUID, 0, this.now), - OptoutCheckPolicy.DoNotRespect + inputVal.toHashedDii(IdentityScope.EUID), + OptoutCheckPolicy.DoNotRespect, PrivacyBits.fromInt(0), this.now ); - final IdentityResponse identityResponse = euidService.generateIdentity(identityRequest); + final TokenGenerateResponse tokenGenerateResponse = euidService.generateIdentity(tokenGenerateRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(identityResponse); + assertNotNull(tokenGenerateResponse); - final RefreshTokenInput refreshTokenInput = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); + final TokenRefreshRequest tokenRefreshRequest = this.tokenEncoder.decodeRefreshToken(tokenGenerateResponse.getRefreshToken()); reset(shutdownHandler); - assertEquals(RefreshResponse.Invalid, uid2Service.refreshIdentity(refreshTokenInput)); + assertEquals(TokenRefreshResponse.Invalid, uid2Service.refreshIdentity(tokenRefreshRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS))); verify(shutdownHandler, never()).handleSaltRetrievalResponse(anyBoolean()); } @@ -258,50 +352,64 @@ public void testTestOptOutKeyIdentityScopeMismatch() { "Email,test@example.com,EUID", "Phone,+01010101010,UID2", "Phone,+01010101010,EUID"}) - public void testGenerateTokenForOptOutUser(IdentityType type, String id, IdentityScope scope) { - final HashedDiiIdentity hashedDiiIdentity = createHashedDiiIdentity(TokenUtils.getIdentityHashString(id), + public void testGenerateTokenForOptOutUser(DiiType type, String id, IdentityScope scope) { + final HashedDii hashedDii = createHashedDii(TokenUtils.getHashedDiiString(id), scope, type); - final IdentityRequest identityRequestForceGenerate = new IdentityRequest( + final TokenGenerateRequest tokenGenerateRequestForceGenerate = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), - hashedDiiIdentity, - OptoutCheckPolicy.DoNotRespect); + hashedDii, + OptoutCheckPolicy.DoNotRespect, PrivacyBits.fromInt(0), + this.now.minusSeconds(234)); - final IdentityRequest identityRequestRespectOptOut = new IdentityRequest( + final TokenGenerateRequest tokenGenerateRequestRespectOptOut = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), - hashedDiiIdentity, - OptoutCheckPolicy.RespectOptOut); + hashedDii, + OptoutCheckPolicy.RespectOptOut, PrivacyBits.fromInt(0), + this.now.minusSeconds(234)); // the clock value shouldn't matter here - when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) + when(optOutStore.getLatestEntry(any(FirstLevelHash.class))) .thenReturn(Instant.now().minus(1, ChronoUnit.HOURS)); - final IdentityResponse identityResponse; - final AdvertisingTokenInput advertisingTokenInput; - final IdentityResponse identityResponseAfterOptOut; + final TokenGenerateResponse tokenGenerateResponse; + final AdvertisingTokenRequest advertisingTokenRequest; + final TokenGenerateResponse tokenGenerateResponseAfterOptOut; if (scope == IdentityScope.UID2) { - identityResponse = uid2Service.generateIdentity(identityRequestForceGenerate); + tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequestForceGenerate, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - advertisingTokenInput = validateAndGetToken(tokenEncoder, identityResponse.getAdvertisingToken(), IdentityScope.UID2, hashedDiiIdentity.identityType, identityRequestRespectOptOut.sourcePublisher.siteId); + advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), IdentityScope.UID2, hashedDii.diiType(), tokenGenerateRequestRespectOptOut.sourcePublisher.siteId); reset(shutdownHandler); - identityResponseAfterOptOut = uid2Service.generateIdentity(identityRequestRespectOptOut); + tokenGenerateResponseAfterOptOut = uid2Service.generateIdentity(tokenGenerateRequestRespectOptOut, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } else { - identityResponse = euidService.generateIdentity(identityRequestForceGenerate); + tokenGenerateResponse = euidService.generateIdentity(tokenGenerateRequestForceGenerate, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - advertisingTokenInput = validateAndGetToken(tokenEncoder, identityResponse.getAdvertisingToken(), IdentityScope.EUID, hashedDiiIdentity.identityType, identityRequestRespectOptOut.sourcePublisher.siteId); + advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), IdentityScope.EUID, hashedDii.diiType(), tokenGenerateRequestRespectOptOut.sourcePublisher.siteId); reset(shutdownHandler); - identityResponseAfterOptOut = euidService.generateIdentity(identityRequestRespectOptOut); + tokenGenerateResponseAfterOptOut = euidService.generateIdentity(tokenGenerateRequestRespectOptOut, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(identityResponse); - assertNotNull(advertisingTokenInput.rawUidIdentity); - assertNotNull(identityResponseAfterOptOut); - assertTrue(identityResponseAfterOptOut.getAdvertisingToken() == null || identityResponseAfterOptOut.getAdvertisingToken().isEmpty()); - + assertNotNull(tokenGenerateResponse); + assertNotNull(advertisingTokenRequest.rawUid); + assertNotNull(tokenGenerateResponseAfterOptOut); + assertTrue(tokenGenerateResponseAfterOptOut.getAdvertisingToken() == null || tokenGenerateResponseAfterOptOut.getAdvertisingToken().isEmpty()); + assertTrue(tokenGenerateResponseAfterOptOut.isOptedOut()); } @ParameterizedTest @@ -309,45 +417,45 @@ public void testGenerateTokenForOptOutUser(IdentityType type, String id, Identit "Email,test@example.com,EUID", "Phone,+01010101010,UID2", "Phone,+01010101010,EUID"}) - public void testIdentityMapForOptOutUser(IdentityType type, String identity, IdentityScope scope) { - final HashedDiiIdentity hashedDiiIdentity = createHashedDiiIdentity(identity, scope, type); + public void testIdentityMapForOptOutUser(DiiType type, String identity, IdentityScope scope) { + final HashedDii hashedDii = createHashedDii(TokenUtils.getHashedDiiString(identity), scope, type); final Instant now = Instant.now(); - final MapRequest mapRequestForceMap = new MapRequest( - hashedDiiIdentity, + final IdentityMapRequestItem mapRequestForceIdentityMapItem = new IdentityMapRequestItem( + hashedDii, OptoutCheckPolicy.DoNotRespect, now); - final MapRequest mapRequestRespectOptOut = new MapRequest( - hashedDiiIdentity, + final IdentityMapRequestItem identityMapRequestItemRespectOptOut = new IdentityMapRequestItem( + hashedDii, OptoutCheckPolicy.RespectOptOut, now); // the clock value shouldn't matter here - when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) + when(optOutStore.getLatestEntry(any(FirstLevelHash.class))) .thenReturn(Instant.now().minus(1, ChronoUnit.HOURS)); - final RawUidResponse rawUidResponse; - final RawUidResponse rawUidResponseShouldBeOptOut; + final IdentityMapResponseItem identityMapResponseItem; + final IdentityMapResponseItem identityMapResponseItemShouldBeOptOut; if (scope == IdentityScope.UID2) { verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - rawUidResponse = uid2Service.mapIdentity(mapRequestForceMap); + identityMapResponseItem = uid2Service.mapHashedDii(mapRequestForceIdentityMapItem); reset(shutdownHandler); - rawUidResponseShouldBeOptOut = uid2Service.mapIdentity(mapRequestRespectOptOut); + identityMapResponseItemShouldBeOptOut = uid2Service.mapHashedDii(identityMapRequestItemRespectOptOut); } else { verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - rawUidResponse = euidService.mapIdentity(mapRequestForceMap); + identityMapResponseItem = euidService.mapHashedDii(mapRequestForceIdentityMapItem); reset(shutdownHandler); - rawUidResponseShouldBeOptOut = euidService.mapIdentity(mapRequestRespectOptOut); + identityMapResponseItemShouldBeOptOut = euidService.mapHashedDii(identityMapRequestItemRespectOptOut); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(rawUidResponse); - assertFalse(rawUidResponse.isOptedOut()); - assertNotNull(rawUidResponseShouldBeOptOut); - assertTrue(rawUidResponseShouldBeOptOut.isOptedOut()); + assertNotNull(identityMapResponseItem); + assertFalse(identityMapResponseItem.isOptedOut()); + assertNotNull(identityMapResponseItemShouldBeOptOut); + assertTrue(identityMapResponseItemShouldBeOptOut.isOptedOut()); } private enum TestIdentityInputType { @@ -393,25 +501,31 @@ private InputUtil.InputVal generateInputVal(TestIdentityInputType type, String i void testSpecialIdentityOptOutTokenGenerate(TestIdentityInputType type, String id, IdentityScope scope) { InputUtil.InputVal inputVal = generateInputVal(type, id); - final IdentityRequest identityRequest = new IdentityRequest( + final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(scope, 0, this.now), - OptoutCheckPolicy.RespectOptOut + inputVal.toHashedDii(scope), + OptoutCheckPolicy.RespectOptOut, PrivacyBits.fromInt(0), this.now ); // identity has no optout record, ensure generate still returns optout when(this.optOutStore.getLatestEntry(any())).thenReturn(null); - IdentityResponse identityResponse; + TokenGenerateResponse tokenGenerateResponse; if(scope == IdentityScope.EUID) { - identityResponse = euidService.generateIdentity(identityRequest); + tokenGenerateResponse = euidService.generateIdentity(tokenGenerateRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } else { - identityResponse = uid2Service.generateIdentity(identityRequest); + tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertEquals(identityResponse, IdentityResponse.OptOutIdentityResponse); + assertEquals(tokenGenerateResponse, TokenGenerateResponse.OptOutResponse); } @ParameterizedTest @@ -426,25 +540,25 @@ void testSpecialIdentityOptOutTokenGenerate(TestIdentityInputType type, String i void testSpecialIdentityOptOutIdentityMap(TestIdentityInputType type, String id, IdentityScope scope) { InputUtil.InputVal inputVal = generateInputVal(type, id); - final MapRequest mapRequestRespectOptOut = new MapRequest( - inputVal.toHashedDiiIdentity(scope, 0, this.now), + final IdentityMapRequestItem identityMapRequestItemRespectOptOut = new IdentityMapRequestItem( + inputVal.toHashedDii(scope), OptoutCheckPolicy.RespectOptOut, now); // identity has no optout record, ensure map still returns optout when(this.optOutStore.getLatestEntry(any())).thenReturn(null); - final RawUidResponse rawUidResponse; + final IdentityMapResponseItem identityMapResponseItem; if(scope == IdentityScope.EUID) { - rawUidResponse = euidService.mapIdentity(mapRequestRespectOptOut); + identityMapResponseItem = euidService.mapHashedDii(identityMapRequestItemRespectOptOut); } else { - rawUidResponse = uid2Service.mapIdentity(mapRequestRespectOptOut); + identityMapResponseItem = uid2Service.mapHashedDii(identityMapRequestItemRespectOptOut); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(rawUidResponse); - assertTrue(rawUidResponse.isOptedOut()); + assertNotNull(identityMapResponseItem); + assertTrue(identityMapResponseItem.isOptedOut()); } @ParameterizedTest @@ -459,30 +573,39 @@ void testSpecialIdentityOptOutIdentityMap(TestIdentityInputType type, String id, void testSpecialIdentityOptOutTokenRefresh(TestIdentityInputType type, String id, IdentityScope scope) { InputUtil.InputVal inputVal = generateInputVal(type, id); - final IdentityRequest identityRequest = new IdentityRequest( + final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(scope, 0, this.now), - OptoutCheckPolicy.DoNotRespect + inputVal.toHashedDii(scope), + OptoutCheckPolicy.DoNotRespect, PrivacyBits.fromInt(0), this.now ); - IdentityResponse identityResponse; + TokenGenerateResponse tokenGenerateResponse; if(scope == IdentityScope.EUID) { - identityResponse = euidService.generateIdentity(identityRequest); + tokenGenerateResponse = euidService.generateIdentity(tokenGenerateRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } else { - identityResponse = uid2Service.generateIdentity(identityRequest); + tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(identityResponse); - assertNotEquals(IdentityResponse.OptOutIdentityResponse, identityResponse); + assertNotNull(tokenGenerateResponse); + assertNotEquals(TokenGenerateResponse.OptOutResponse, tokenGenerateResponse); // identity has no optout record, ensure refresh still returns optout when(this.optOutStore.getLatestEntry(any())).thenReturn(null); - final RefreshTokenInput refreshTokenInput = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); + final TokenRefreshRequest tokenRefreshRequest = this.tokenEncoder.decodeRefreshToken(tokenGenerateResponse.getRefreshToken()); reset(shutdownHandler); - assertEquals(RefreshResponse.Optout, (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(refreshTokenInput)); + assertEquals(TokenRefreshResponse.Optout, (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(tokenRefreshRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS))); verify(shutdownHandler, never()).handleSaltRetrievalResponse(anyBoolean()); } @@ -498,33 +621,42 @@ void testSpecialIdentityOptOutTokenRefresh(TestIdentityInputType type, String id void testSpecialIdentityRefreshOptOutGenerate(TestIdentityInputType type, String id, IdentityScope scope) { InputUtil.InputVal inputVal = generateInputVal(type, id); - final IdentityRequest identityRequest = new IdentityRequest( + final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(scope, 0, this.now), - OptoutCheckPolicy.RespectOptOut + inputVal.toHashedDii(scope), + OptoutCheckPolicy.RespectOptOut, PrivacyBits.fromInt(0), this.now ); // identity has optout record, ensure still generates when(this.optOutStore.getLatestEntry(any())).thenReturn(Instant.now()); - IdentityResponse identityResponse; + TokenGenerateResponse tokenGenerateResponse; if(scope == IdentityScope.EUID) { - identityResponse = euidService.generateIdentity(identityRequest); + tokenGenerateResponse = euidService.generateIdentity(tokenGenerateRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } else { - identityResponse = uid2Service.generateIdentity(identityRequest); + tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(identityResponse); - assertNotEquals(IdentityResponse.OptOutIdentityResponse, identityResponse); + assertNotNull(tokenGenerateResponse); + assertNotEquals(TokenGenerateResponse.OptOutResponse, tokenGenerateResponse); // identity has no optout record, ensure refresh still returns optout when(this.optOutStore.getLatestEntry(any())).thenReturn(null); - final RefreshTokenInput refreshTokenInput = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); + final TokenRefreshRequest tokenRefreshRequest = this.tokenEncoder.decodeRefreshToken(tokenGenerateResponse.getRefreshToken()); reset(shutdownHandler); - assertEquals(RefreshResponse.Optout, (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(refreshTokenInput)); + assertEquals(TokenRefreshResponse.Optout, (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(tokenRefreshRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS))); verify(shutdownHandler, never()).handleSaltRetrievalResponse(anyBoolean()); } @@ -540,25 +672,25 @@ void testSpecialIdentityRefreshOptOutGenerate(TestIdentityInputType type, String void testSpecialIdentityRefreshOptOutIdentityMap(TestIdentityInputType type, String id, IdentityScope scope) { InputUtil.InputVal inputVal = generateInputVal(type, id); - final MapRequest mapRequestRespectOptOut = new MapRequest( - inputVal.toHashedDiiIdentity(scope, 0, this.now), + final IdentityMapRequestItem identityMapRequestItemRespectOptOut = new IdentityMapRequestItem( + inputVal.toHashedDii(scope), OptoutCheckPolicy.RespectOptOut, now); // all identities have optout records, ensure refresh-optout identities still map when(this.optOutStore.getLatestEntry(any())).thenReturn(Instant.now()); - final RawUidResponse rawUidResponse; + final IdentityMapResponseItem identityMapResponseItem; if(scope == IdentityScope.EUID) { - rawUidResponse = euidService.mapIdentity(mapRequestRespectOptOut); + identityMapResponseItem = euidService.mapHashedDii(identityMapRequestItemRespectOptOut); } else { - rawUidResponse = uid2Service.mapIdentity(mapRequestRespectOptOut); + identityMapResponseItem = uid2Service.mapHashedDii(identityMapRequestItemRespectOptOut); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(rawUidResponse); - assertFalse(rawUidResponse.isOptedOut()); + assertNotNull(identityMapResponseItem); + assertFalse(identityMapResponseItem.isOptedOut()); } @ParameterizedTest @@ -573,29 +705,35 @@ void testSpecialIdentityRefreshOptOutIdentityMap(TestIdentityInputType type, Str void testSpecialIdentityValidateGenerate(TestIdentityInputType type, String id, IdentityScope scope) { InputUtil.InputVal inputVal = generateInputVal(type, id); - final IdentityRequest identityRequest = new IdentityRequest( + final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(scope, 0, this.now), - OptoutCheckPolicy.RespectOptOut + inputVal.toHashedDii(scope), + OptoutCheckPolicy.RespectOptOut, PrivacyBits.fromInt(0), this.now ); // all identities have optout records, ensure validate identities still get generated when(this.optOutStore.getLatestEntry(any())).thenReturn(Instant.now()); - IdentityResponse identityResponse; - AdvertisingTokenInput advertisingTokenInput; + TokenGenerateResponse tokenGenerateResponse; + AdvertisingTokenRequest advertisingTokenRequest; if (scope == IdentityScope.EUID) { - identityResponse = euidService.generateIdentity(identityRequest); + tokenGenerateResponse = euidService.generateIdentity(tokenGenerateRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } else { - identityResponse = uid2Service.generateIdentity(identityRequest); + tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } - advertisingTokenInput = validateAndGetToken(tokenEncoder, identityResponse.getAdvertisingToken(), scope, identityRequest.hashedDiiIdentity.identityType, identityRequest.sourcePublisher.siteId); + advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), scope, tokenGenerateRequest.hashedDii.diiType(), tokenGenerateRequest.sourcePublisher.siteId); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(identityResponse); - assertNotEquals(IdentityResponse.OptOutIdentityResponse, identityResponse); - assertNotNull(advertisingTokenInput.rawUidIdentity); + assertNotNull(tokenGenerateResponse); + assertNotEquals(TokenGenerateResponse.OptOutResponse, tokenGenerateResponse); + assertNotNull(advertisingTokenRequest.rawUid); } @@ -611,25 +749,25 @@ void testSpecialIdentityValidateGenerate(TestIdentityInputType type, String id, void testSpecialIdentityValidateIdentityMap(TestIdentityInputType type, String id, IdentityScope scope) { InputUtil.InputVal inputVal = generateInputVal(type, id); - final MapRequest mapRequestRespectOptOut = new MapRequest( - inputVal.toHashedDiiIdentity(scope, 0, this.now), + final IdentityMapRequestItem identityMapRequestItemRespectOptOut = new IdentityMapRequestItem( + inputVal.toHashedDii(scope), OptoutCheckPolicy.RespectOptOut, now); // all identities have optout records, ensure validate identities still get mapped when(this.optOutStore.getLatestEntry(any())).thenReturn(Instant.now()); - final RawUidResponse rawUidResponse; + final IdentityMapResponseItem identityMapResponseItem; if(scope == IdentityScope.EUID) { - rawUidResponse = euidService.mapIdentity(mapRequestRespectOptOut); + identityMapResponseItem = euidService.mapHashedDii(identityMapRequestItemRespectOptOut); } else { - rawUidResponse = uid2Service.mapIdentity(mapRequestRespectOptOut); + identityMapResponseItem = uid2Service.mapHashedDii(identityMapRequestItemRespectOptOut); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(rawUidResponse); - assertFalse(rawUidResponse.isOptedOut()); + assertNotNull(identityMapResponseItem); + assertFalse(identityMapResponseItem.isOptedOut()); } @ParameterizedTest @@ -641,28 +779,38 @@ void testSpecialIdentityValidateIdentityMap(TestIdentityInputType type, String i "EmailHash,blah@unifiedid.com,EUID"}) void testNormalIdentityOptIn(TestIdentityInputType type, String id, IdentityScope scope) { InputUtil.InputVal inputVal = generateInputVal(type, id); - final IdentityRequest identityRequest = new IdentityRequest( + final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(scope, 0, this.now), + inputVal.toHashedDii(scope), OptoutCheckPolicy.DoNotRespect ); - IdentityResponse identityResponse; + TokenGenerateResponse tokenGenerateResponse; if(scope == IdentityScope.EUID) { - identityResponse = euidService.generateIdentity(identityRequest); + tokenGenerateResponse = euidService.generateIdentity(tokenGenerateRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } else { - identityResponse = uid2Service.generateIdentity(identityRequest); + tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotEquals(identityResponse, IdentityResponse.OptOutIdentityResponse); - assertNotNull(identityResponse); - - final RefreshTokenInput refreshTokenInput = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); - RefreshResponse refreshResponse = (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(refreshTokenInput); + assertNotEquals(tokenGenerateResponse, TokenGenerateResponse.OptOutResponse); + assertNotNull(tokenGenerateResponse); + + final TokenRefreshRequest tokenRefreshRequest = this.tokenEncoder.decodeRefreshToken(tokenGenerateResponse.getRefreshToken()); + TokenRefreshResponse refreshResponse = + (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(tokenRefreshRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); assertTrue(refreshResponse.isRefreshed()); assertNotNull(refreshResponse.getIdentityResponse()); - assertNotEquals(RefreshResponse.Optout, refreshResponse); + assertNotEquals(TokenRefreshResponse.Optout, refreshResponse); } @ParameterizedTest @@ -679,76 +827,192 @@ void testExpiredSaltsNotifiesShutdownHandler(TestIdentityInputType type, String saltProvider.loadContent(); UIDOperatorService uid2Service = new UIDOperatorService( - uid2Config, optOutStore, saltProvider, tokenEncoder, this.clock, IdentityScope.UID2, - this.shutdownHandler::handleSaltRetrievalResponse + this.shutdownHandler::handleSaltRetrievalResponse, + uid2Config.getBoolean(IdentityV3Prop), + uidInstanceIdProvider ); UIDOperatorService euidService = new UIDOperatorService( - euidConfig, optOutStore, saltProvider, tokenEncoder, this.clock, IdentityScope.EUID, - this.shutdownHandler::handleSaltRetrievalResponse + this.shutdownHandler::handleSaltRetrievalResponse, + euidConfig.getBoolean(IdentityV3Prop), + uidInstanceIdProvider ); when(this.optOutStore.getLatestEntry(any())).thenReturn(null); InputUtil.InputVal inputVal = generateInputVal(type, id); - final IdentityRequest identityRequest = new IdentityRequest( + final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(scope, 0, this.now), - OptoutCheckPolicy.RespectOptOut); + inputVal.toHashedDii(scope), + OptoutCheckPolicy.RespectOptOut, PrivacyBits.fromInt(0), this.now); - IdentityResponse identityResponse; - AdvertisingTokenInput advertisingTokenInput; + TokenGenerateResponse tokenGenerateResponse; + AdvertisingTokenRequest advertisingTokenRequest; reset(shutdownHandler); if(scope == IdentityScope.EUID) { - identityResponse = euidService.generateIdentity(identityRequest); - advertisingTokenInput = validateAndGetToken(tokenEncoder, identityResponse.getAdvertisingToken(), IdentityScope.EUID, identityRequest.hashedDiiIdentity.identityType, identityRequest.sourcePublisher.siteId); + tokenGenerateResponse = euidService.generateIdentity(tokenGenerateRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); + advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), IdentityScope.EUID, tokenGenerateRequest.hashedDii.diiType(), tokenGenerateRequest.sourcePublisher.siteId); } else { - identityResponse = uid2Service.generateIdentity(identityRequest); - advertisingTokenInput = validateAndGetToken(tokenEncoder, identityResponse.getAdvertisingToken(), IdentityScope.UID2, identityRequest.hashedDiiIdentity.identityType, identityRequest.sourcePublisher.siteId); + tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); + advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), IdentityScope.UID2, tokenGenerateRequest.hashedDii.diiType(), tokenGenerateRequest.sourcePublisher.siteId); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(true); verify(shutdownHandler, never()).handleSaltRetrievalResponse(false); - assertNotNull(identityResponse); - assertNotEquals(IdentityResponse.OptOutIdentityResponse, identityResponse); - assertNotNull(advertisingTokenInput.rawUidIdentity); + assertNotNull(tokenGenerateResponse); + assertNotEquals(TokenGenerateResponse.OptOutResponse, tokenGenerateResponse); + assertNotNull(advertisingTokenRequest.rawUid); - final RefreshTokenInput refreshTokenInput = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); + final TokenRefreshRequest tokenRefreshRequest = this.tokenEncoder.decodeRefreshToken(tokenGenerateResponse.getRefreshToken()); reset(shutdownHandler); - RefreshResponse refreshResponse = (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(refreshTokenInput); + TokenRefreshResponse refreshResponse = (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(tokenRefreshRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(true); verify(shutdownHandler, never()).handleSaltRetrievalResponse(false); assertTrue(refreshResponse.isRefreshed()); assertNotNull(refreshResponse.getIdentityResponse()); - assertNotEquals(RefreshResponse.Optout, refreshResponse); + assertNotEquals(TokenRefreshResponse.Optout, refreshResponse); - final MapRequest mapRequest = new MapRequest( - inputVal.toHashedDiiIdentity(scope, 0, this.now), + final IdentityMapRequestItem identityMapRequestItem = new IdentityMapRequestItem( + inputVal.toHashedDii(scope), OptoutCheckPolicy.RespectOptOut, now); - final RawUidResponse rawUidResponse; + final IdentityMapResponseItem identityMapResponseItem; reset(shutdownHandler); if(scope == IdentityScope.EUID) { - rawUidResponse = euidService.mapIdentity(mapRequest); + identityMapResponseItem = euidService.mapHashedDii(identityMapRequestItem); } else { - rawUidResponse = uid2Service.mapIdentity(mapRequest); + identityMapResponseItem = uid2Service.mapHashedDii(identityMapRequestItem); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(true); verify(shutdownHandler, never()).handleSaltRetrievalResponse(false); - assertNotNull(rawUidResponse); - assertFalse(rawUidResponse.isOptedOut()); + assertNotNull(identityMapResponseItem); + assertFalse(identityMapResponseItem.isOptedOut()); + + } + + @Test + void testMappedIdentityWithPreviousSaltReturnsPreviousUid() { + var saltSnapshot = setUpMockSalts(); + + long lastUpdated = this.now.minus(90, DAYS).plusMillis(1).toEpochMilli(); // 1 millis before 90 days old + long refreshFrom = lastUpdated + Duration.ofDays(120).toMillis(); + + SaltEntry salt = new SaltEntry(1, "1", lastUpdated, "salt", refreshFrom, "previousSalt", null, null); + when(saltSnapshot.getRotatingSalt(any())).thenReturn(salt); + + var email = "test@uid.com"; + InputUtil.InputVal emailInput = generateInputVal(TestIdentityInputType.Email, email); + IdentityMapRequestItem mapRequest = new IdentityMapRequestItem(emailInput.toHashedDii(IdentityScope.UID2), + OptoutCheckPolicy.RespectOptOut, now); + + IdentityMapResponseItem mappedIdentity = uid2Service.mapHashedDii(mapRequest); + + var expectedCurrentUID = UIDOperatorVerticleTest.getRawUid(DiiType.Email, email, FIRST_LEVEL_SALT, salt.currentSalt(), IdentityScope.UID2, uid2Config.getBoolean(IdentityV3Prop)); + var expectedPreviousUID = UIDOperatorVerticleTest.getRawUid(DiiType.Email, email, FIRST_LEVEL_SALT, salt.previousSalt(), IdentityScope.UID2, uid2Config.getBoolean(IdentityV3Prop)); + assertArrayEquals(expectedCurrentUID, mappedIdentity.rawUid); + assertArrayEquals(expectedPreviousUID, mappedIdentity.previousRawUid); + } + + @ParameterizedTest + @ValueSource(strings = {"0", "1"}) + void testMappedIdentityWithOutdatedPreviousSaltReturnsNoPreviousUid(long extraMsAfter90DaysOld) { + var saltSnapshot = setUpMockSalts(); + + long lastUpdated = this.now.minus(90, DAYS).minusMillis(extraMsAfter90DaysOld).toEpochMilli(); + long refreshFrom = lastUpdated + Duration.ofDays(120).toMillis(); + + SaltEntry salt = new SaltEntry(1, "1", lastUpdated, "salt", refreshFrom, "previousSalt", null, null); + when(saltSnapshot.getRotatingSalt(any())).thenReturn(salt); + + var email = "test@uid.com"; + InputUtil.InputVal emailInput = generateInputVal(TestIdentityInputType.Email, email); + IdentityMapRequestItem mapRequest = new IdentityMapRequestItem(emailInput.toHashedDii(IdentityScope.UID2), OptoutCheckPolicy.RespectOptOut, now); + + IdentityMapResponseItem mappedIdentity = uid2Service.mapHashedDii(mapRequest); + var expectedCurrentUID = UIDOperatorVerticleTest.getRawUid(DiiType.Email, email, FIRST_LEVEL_SALT, salt.currentSalt(), IdentityScope.UID2, uid2Config.getBoolean(IdentityV3Prop)); + assertArrayEquals(expectedCurrentUID, mappedIdentity.rawUid); + assertArrayEquals(null , mappedIdentity.previousRawUid); + } + + @Test + void testMappedIdentityWithNoPreviousSaltReturnsNoPreviousUid() { + var saltSnapshot = setUpMockSalts(); + + long lastUpdated = this.now.toEpochMilli(); + long refreshFrom = this.now.plus(30, DAYS).toEpochMilli(); + + SaltEntry salt = new SaltEntry(1, "1", lastUpdated, "salt", refreshFrom, null, null, null); + when(saltSnapshot.getRotatingSalt(any())).thenReturn(salt); + + var email = "test@uid.com"; + InputUtil.InputVal emailInput = generateInputVal(TestIdentityInputType.Email, email); + IdentityMapRequestItem mapRequest = new IdentityMapRequestItem(emailInput.toHashedDii(IdentityScope.UID2), OptoutCheckPolicy.RespectOptOut, now); + + IdentityMapResponseItem mappedIdentity = uid2Service.mapHashedDii(mapRequest); + + var expectedCurrentUID = UIDOperatorVerticleTest.getRawUid(DiiType.Email, email, FIRST_LEVEL_SALT, salt.currentSalt(), IdentityScope.UID2, uid2Config.getBoolean(IdentityV3Prop)); + assertArrayEquals(expectedCurrentUID, mappedIdentity.rawUid); + assertArrayEquals(null, mappedIdentity.previousRawUid); + } + + @ParameterizedTest + @ValueSource(strings = {"0", "30"}) + void testMappedIdentityWithValidRefreshFrom(int refreshFromDays) { + var saltSnapshot = setUpMockSalts(); + + long lastUpdated = this.now.minus(30, DAYS).toEpochMilli(); + long refreshFrom = this.now.plus(refreshFromDays, DAYS).toEpochMilli(); + + SaltEntry salt = new SaltEntry(1, "1", lastUpdated, "salt", refreshFrom, null, null, null); + when(saltSnapshot.getRotatingSalt(any())).thenReturn(salt); + + var email = "test@uid.com"; + InputUtil.InputVal emailInput = generateInputVal(TestIdentityInputType.Email, email); + IdentityMapRequestItem mapRequest = new IdentityMapRequestItem(emailInput.toHashedDii(IdentityScope.UID2), OptoutCheckPolicy.RespectOptOut, now); + + IdentityMapResponseItem mappedIdentity = uid2Service.mapHashedDii(mapRequest); + + assertEquals(refreshFrom, mappedIdentity.refreshFrom); + } + + @Test + void testMappedIdentityWithOutdatedRefreshFrom() { + var saltSnapshot = setUpMockSalts(); + + long lastUpdated = this.now.minus(31, DAYS).toEpochMilli(); + long outdatedRefreshFrom = this.now.minus(1, DAYS).toEpochMilli(); + + SaltEntry salt = new SaltEntry(1, "1", lastUpdated, "salt", outdatedRefreshFrom, null, null, null); + when(saltSnapshot.getRotatingSalt(any())).thenReturn(salt); + + var email = "test@uid.com"; + InputUtil.InputVal emailInput = generateInputVal(TestIdentityInputType.Email, email); + IdentityMapRequestItem mapRequest = new IdentityMapRequestItem(emailInput.toHashedDii(IdentityScope.UID2), OptoutCheckPolicy.RespectOptOut, now); + + IdentityMapResponseItem mappedIdentity = uid2Service.mapHashedDii(mapRequest); + long expectedRefreshFrom = this.now.truncatedTo(DAYS).plus(1, DAYS).toEpochMilli(); + assertEquals(expectedRefreshFrom, mappedIdentity.refreshFrom); } } diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index ea8385779..30c8bee42 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -4,17 +4,23 @@ import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.read.ListAppender; import com.uid2.operator.model.*; -import com.uid2.operator.model.IdentityScope; -import com.uid2.operator.model.userIdentity.FirstLevelHashIdentity; +import com.uid2.operator.model.identities.IdentityScope; +import com.uid2.operator.model.identities.DiiType; +import com.uid2.operator.model.identities.FirstLevelHash; import com.uid2.operator.monitoring.IStatsCollectorQueue; import com.uid2.operator.monitoring.TokenResponseStatsCollector; import com.uid2.operator.service.*; +import com.uid2.operator.store.IConfigStore; import com.uid2.operator.store.IOptOutStore; +import com.uid2.operator.store.RuntimeConfig; +import com.uid2.operator.util.HttpMediaType; import com.uid2.operator.util.PrivacyBits; import com.uid2.operator.util.Tuple; import com.uid2.operator.vertx.OperatorShutdownHandler; import com.uid2.operator.vertx.UIDOperatorVerticle; import com.uid2.shared.Utils; +import com.uid2.shared.audit.Audit; +import com.uid2.shared.audit.UidInstanceIdProvider; import com.uid2.shared.auth.ClientKey; import com.uid2.shared.auth.Keyset; import com.uid2.shared.auth.KeysetSnapshot; @@ -27,6 +33,7 @@ import com.uid2.shared.secret.KeyHasher; import com.uid2.shared.store.*; import com.uid2.shared.store.reader.RotatingKeysetProvider; +import com.uid2.shared.store.salt.ISaltProvider; import io.micrometer.core.instrument.Metrics; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import io.vertx.core.AsyncResult; @@ -34,6 +41,7 @@ import io.vertx.core.Handler; import io.vertx.core.Vertx; import io.vertx.core.buffer.Buffer; +import io.vertx.core.http.HttpHeaders; import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; import io.vertx.ext.web.RoutingContext; @@ -45,14 +53,15 @@ import org.junit.jupiter.api.*; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.CsvSource; -import org.junit.jupiter.params.provider.MethodSource; -import org.junit.jupiter.params.provider.ValueSource; +import org.junit.jupiter.params.provider.*; import org.mockito.ArgumentCaptor; import org.mockito.Mock; -import org.mockito.MockitoAnnotations; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; import org.slf4j.LoggerFactory; + +import static java.time.temporal.ChronoUnit.DAYS; import static org.assertj.core.api.Assertions.*; import javax.crypto.SecretKey; @@ -60,16 +69,15 @@ import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.security.*; -import java.time.Clock; -import java.time.Duration; -import java.time.Instant; +import java.time.*; import java.time.temporal.ChronoUnit; import java.util.*; +import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; import static com.uid2.operator.ClientSideTokenGenerateTestUtil.decrypt; -import static com.uid2.operator.IdentityConst.*; +import static com.uid2.operator.model.identities.IdentityConst.*; import static com.uid2.operator.service.EncodingUtils.getSha256; import static com.uid2.operator.vertx.UIDOperatorVerticle.*; import static com.uid2.shared.Const.Data.*; @@ -78,13 +86,14 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; -@ExtendWith(VertxExtension.class) +@ExtendWith({VertxExtension.class, MockitoExtension.class}) +@MockitoSettings(strictness = Strictness.LENIENT) public class UIDOperatorVerticleTest { private final Instant now = Instant.now().truncatedTo(ChronoUnit.MILLIS); private static final Instant legacyClientCreationDateTime = Instant.ofEpochSecond(OPT_OUT_CHECK_CUTOFF_DATE).minus(1, ChronoUnit.SECONDS); private static final Instant newClientCreationDateTime = Instant.ofEpochSecond(OPT_OUT_CHECK_CUTOFF_DATE).plus(1, ChronoUnit.SECONDS); private static final String firstLevelSalt = "first-level-salt"; - private static final SaltEntry rotatingSalt123 = new SaltEntry(123, "hashed123", 0, "salt123"); + private static final SaltEntry rotatingSalt123 = new SaltEntry(123, "hashed123", 0, "salt123", null, null, null, null); private static final Duration identityExpiresAfter = Duration.ofMinutes(10); private static final Duration refreshExpiresAfter = Duration.ofMinutes(15); private static final Duration refreshIdentityAfter = Duration.ofMinutes(5); @@ -101,39 +110,64 @@ public class UIDOperatorVerticleTest { private static final int optOutStatusMaxRequestSize = 1000; - private AutoCloseable mocks; - @Mock private ISiteStore siteProvider; - @Mock private IClientKeyProvider clientKeyProvider; - @Mock private IClientSideKeypairStore clientSideKeypairProvider; - @Mock private IClientSideKeypairStore.IClientSideKeypairStoreSnapshot clientSideKeypairSnapshot; - @Mock private IKeysetKeyStore keysetKeyStore; - @Mock private RotatingKeysetProvider keysetProvider; - @Mock private ISaltProvider saltProvider; - @Mock private SecureLinkValidatorService secureLinkValidatorService; - @Mock private ISaltProvider.ISaltSnapshot saltProviderSnapshot; - @Mock private IOptOutStore optOutStore; - @Mock private Clock clock; - @Mock private IStatsCollectorQueue statsCollectorQueue; - @Mock private OperatorShutdownHandler shutdownHandler; + @Mock + private ISiteStore siteProvider; + @Mock + private IClientKeyProvider clientKeyProvider; + @Mock + private IClientSideKeypairStore clientSideKeypairProvider; + @Mock + private IClientSideKeypairStore.IClientSideKeypairStoreSnapshot clientSideKeypairSnapshot; + @Mock + private IKeysetKeyStore keysetKeyStore; + @Mock + private RotatingKeysetProvider keysetProvider; + @Mock + private ISaltProvider saltProvider; + @Mock + private SecureLinkValidatorService secureLinkValidatorService; + @Mock + private ISaltProvider.ISaltSnapshot saltProviderSnapshot; + @Mock + private IOptOutStore optOutStore; + @Mock + private Clock clock; + @Mock + private IStatsCollectorQueue statsCollectorQueue; + @Mock + private OperatorShutdownHandler shutdownHandler; + @Mock + private IConfigStore configStore; + private UidInstanceIdProvider uidInstanceIdProvider; private SimpleMeterRegistry registry; private ExtendedUIDOperatorVerticle uidOperatorVerticle; + private RuntimeConfig runtimeConfig; private final JsonObject config = new JsonObject(); @BeforeEach public void deployVerticle(Vertx vertx, VertxTestContext testContext, TestInfo testInfo) { - mocks = MockitoAnnotations.openMocks(this); when(saltProvider.getSnapshot(any())).thenReturn(saltProviderSnapshot); when(saltProviderSnapshot.getExpires()).thenReturn(Instant.now().plus(1, ChronoUnit.HOURS)); when(clock.instant()).thenAnswer(i -> now); when(this.secureLinkValidatorService.validateRequest(any(RoutingContext.class), any(JsonObject.class), any(Role.class))).thenReturn(true); + when(this.clientKeyProvider.getClientKey(clientKey)).thenReturn(new ClientKey("key-hash", "key-salt", "secret", "name", Instant.now(), Set.of(), 1, "key-id")); setupConfig(config); + runtimeConfig = setupRuntimeConfig(config); + // TODO: Remove this when we remove tokenGenerateOptOutTokenWithDisableOptoutTokenFF test + if(testInfo.getTestMethod().isPresent() && + testInfo.getTestMethod().get().getName().equals("tokenGenerateOptOutTokenWithDisableOptoutTokenFF")) { + config.put(Const.Config.DisableOptoutTokenProp, true); + } if(testInfo.getDisplayName().equals("cstgNoPhoneSupport(Vertx, VertxTestContext)")) { config.put("enable_phone_support", false); } + when(configStore.getConfig()).thenAnswer(x -> runtimeConfig); + + this.uidInstanceIdProvider = new UidInstanceIdProvider("test-instance", "id"); - this.uidOperatorVerticle = new ExtendedUIDOperatorVerticle(config, config.getBoolean("client_side_token_generate"), siteProvider, clientKeyProvider, clientSideKeypairProvider, new KeyManager(keysetKeyStore, keysetProvider), saltProvider, optOutStore, clock, statsCollectorQueue, secureLinkValidatorService, shutdownHandler::handleSaltRetrievalResponse); + this.uidOperatorVerticle = new ExtendedUIDOperatorVerticle(configStore, config, config.getBoolean("client_side_token_generate"), siteProvider, clientKeyProvider, clientSideKeypairProvider, new KeyManager(keysetKeyStore, keysetProvider), saltProvider, optOutStore, clock, statsCollectorQueue, secureLinkValidatorService, shutdownHandler::handleSaltRetrievalResponse, uidInstanceIdProvider); vertx.deployVerticle(uidOperatorVerticle, testContext.succeeding(id -> testContext.completeNow())); @@ -144,7 +178,10 @@ public void deployVerticle(Vertx vertx, VertxTestContext testContext, TestInfo t @AfterEach public void teardown() throws Exception { Metrics.globalRegistry.remove(registry); - mocks.close(); + } + + private RuntimeConfig setupRuntimeConfig(JsonObject config) { + return config.mapTo(RuntimeConfig.class); } private void setupConfig(JsonObject config) { @@ -156,10 +193,7 @@ private void setupConfig(JsonObject config) { config.put(Const.Config.SharingTokenExpiryProp, 60 * 60 * 24 * 30); config.put("identity_scope", getIdentityScope().toString()); - config.put("advertising_token_v3", getTokenVersion() == TokenVersion.V3); - config.put("advertising_token_v4_percentage", getTokenVersion() == TokenVersion.V4 ? 100 : 0); - config.put("site_ids_using_v4_tokens", ""); - config.put("identity_v3", useIdentityV3()); + config.put(Const.Config.IdentityV3Prop, useRawUidV3()); config.put("client_side_token_generate", true); config.put("key_sharing_endpoint_provide_app_names", true); config.put("client_side_token_generate_log_invalid_http_origins", true); @@ -167,6 +201,8 @@ private void setupConfig(JsonObject config) { config.put(Const.Config.AllowClockSkewSecondsProp, 3600); config.put(Const.Config.OptOutStatusApiEnabled, true); config.put(Const.Config.OptOutStatusMaxRequestSize, optOutStatusMaxRequestSize); + config.put(Const.Config.DisableOptoutTokenProp, false); + config.put(Const.Config.ConfigScanPeriodMsProp, 10000); } private static byte[] makeAesKey(String prefix) { @@ -211,6 +247,10 @@ private String getUrlForEndpoint(String endpoint) { } private void send(String apiVersion, Vertx vertx, String endpoint, boolean isV1Get, String v1GetParam, JsonObject postPayload, int expectedHttpCode, Handler handler) { + send(apiVersion, vertx, endpoint, isV1Get, v1GetParam, postPayload, expectedHttpCode, handler, Collections.emptyMap()); + } + + private void send(String apiVersion, Vertx vertx, String endpoint, boolean isV1Get, String v1GetParam, JsonObject postPayload, int expectedHttpCode, Handler handler, Map additionalHeaders) { if (apiVersion.equals("v2")) { ClientKey ck = (ClientKey) clientKeyProvider.get(""); @@ -221,46 +261,59 @@ private void send(String apiVersion, Vertx vertx, String endpoint, boolean isV1G assertEquals(expectedHttpCode, ar.result().statusCode()); if (ar.result().statusCode() == 200) { - byte[] decrypted = AesGcm.decrypt(Utils.decodeBase64String(ar.result().bodyAsString()), 0, ck.getSecretBytes()); + byte[] byteResp = new byte[0]; + if (ar.result().headers().contains(HttpHeaders.CONTENT_TYPE, HttpMediaType.APPLICATION_OCTET_STREAM.getType(), true)) { + byteResp = ar.result().bodyAsBuffer().getBytes(); + } else if (ar.result().headers().contains(HttpHeaders.CONTENT_TYPE, HttpMediaType.TEXT_PLAIN.getType(), true)) { + byteResp = Utils.decodeBase64String(ar.result().bodyAsString()); + } + + byte[] decrypted = AesGcm.decrypt(byteResp, 0, ck.getSecretBytes()); + assertArrayEquals(Buffer.buffer().appendLong(nonce).getBytes(), Buffer.buffer(decrypted).slice(8, 16).getBytes()); JsonObject respJson = new JsonObject(new String(decrypted, 16, decrypted.length - 16, StandardCharsets.UTF_8)); - handler.handle(respJson); } else { handler.handle(tryParseResponse(ar.result())); } - }); + }, additionalHeaders); } else if (isV1Get) { get(vertx, endpoint + (v1GetParam != null ? "?" + v1GetParam : ""), ar -> { assertTrue(ar.succeeded()); assertEquals(expectedHttpCode, ar.result().statusCode()); handler.handle(tryParseResponse(ar.result())); - }); + }, additionalHeaders); } else { post(vertx, endpoint, postPayload, ar -> { assertTrue(ar.succeeded()); assertEquals(expectedHttpCode, ar.result().statusCode()); handler.handle(tryParseResponse(ar.result())); - }); + }, additionalHeaders); } } protected void sendTokenGenerate(String apiVersion, Vertx vertx, String v1GetParam, JsonObject v2PostPayload, int expectedHttpCode, Handler handler) { - sendTokenGenerate(apiVersion, vertx, v1GetParam, v2PostPayload, expectedHttpCode, null, handler, true); + sendTokenGenerate(apiVersion, vertx, v1GetParam, v2PostPayload, expectedHttpCode, null, handler, true, Collections.emptyMap()); + } + + + protected void sendTokenGenerate(String apiVersion, Vertx vertx, String v1GetParam, JsonObject v2PostPayload, int expectedHttpCode, + Handler handler, Map additionalHeaders) { + sendTokenGenerate(apiVersion, vertx, v1GetParam, v2PostPayload, expectedHttpCode, null, handler, true, additionalHeaders); } protected void sendTokenGenerate(String apiVersion, Vertx vertx, String v1GetParam, JsonObject v2PostPayload, int expectedHttpCode, Handler handler, boolean additionalParams) { - sendTokenGenerate(apiVersion, vertx, v1GetParam, v2PostPayload, expectedHttpCode, null, handler, additionalParams); + sendTokenGenerate(apiVersion, vertx, v1GetParam, v2PostPayload, expectedHttpCode, null, handler, additionalParams, Collections.emptyMap()); } private void sendTokenGenerate(String apiVersion, Vertx vertx, String v1GetParam, JsonObject v2PostPayload, int expectedHttpCode, String referer, Handler handler, boolean additionalParams) { - sendTokenGenerate(apiVersion, vertx, v1GetParam, v2PostPayload, expectedHttpCode, referer, handler, additionalParams, null, null); + sendTokenGenerate(apiVersion, vertx, v1GetParam, v2PostPayload, expectedHttpCode, referer, handler, additionalParams, Collections.emptyMap()); } - private void sendTokenGenerate(String apiVersion, Vertx vertx, String v1GetParam, JsonObject v2PostPayload, int expectedHttpCode, String referer, Handler handler, boolean additionalParams, String headerName, String headerValue) { + private void sendTokenGenerate(String apiVersion, Vertx vertx, String v1GetParam, JsonObject v2PostPayload, int expectedHttpCode, String referer, Handler handler, boolean additionalParams, Map additionalHeaders) { if (apiVersion.equals("v2")) { ClientKey ck = (ClientKey) clientKeyProvider.get(""); @@ -275,7 +328,13 @@ private void sendTokenGenerate(String apiVersion, Vertx vertx, String v1GetParam assertEquals(expectedHttpCode, ar.result().statusCode()); if (ar.result().statusCode() == 200) { - byte[] decrypted = AesGcm.decrypt(Utils.decodeBase64String(ar.result().bodyAsString()), 0, ck.getSecretBytes()); + byte[] byteResp = new byte[0]; + if (ar.result().headers().contains(HttpHeaders.CONTENT_TYPE, HttpMediaType.APPLICATION_OCTET_STREAM.getType(), true)) { + byteResp = ar.result().bodyAsBuffer().getBytes(); + } else if (ar.result().headers().contains(HttpHeaders.CONTENT_TYPE, HttpMediaType.TEXT_PLAIN.getType(), true)) { + byteResp = Utils.decodeBase64String(ar.result().bodyAsString()); + } + byte[] decrypted = AesGcm.decrypt(byteResp, 0, ck.getSecretBytes()); assertArrayEquals(Buffer.buffer().appendLong(nonce).getBytes(), Buffer.buffer(decrypted).slice(8, 16).getBytes()); @@ -287,36 +346,43 @@ private void sendTokenGenerate(String apiVersion, Vertx vertx, String v1GetParam } else { handler.handle(tryParseResponse(ar.result())); } - }, headerName, headerValue); + }, additionalHeaders); } else { get(vertx, apiVersion + "/token/generate" + (v1GetParam != null ? "?" + v1GetParam : ""), ar -> { assertTrue(ar.succeeded()); assertEquals(expectedHttpCode, ar.result().statusCode()); handler.handle(tryParseResponse(ar.result())); - }, headerName, headerValue); + }, additionalHeaders); } } private void sendTokenRefresh(String apiVersion, Vertx vertx, VertxTestContext testContext, String refreshToken, String v2RefreshDecryptSecret, int expectedHttpCode, Handler handler) { - sendTokenRefresh(apiVersion, vertx, null, null, testContext, refreshToken, v2RefreshDecryptSecret, expectedHttpCode, handler); + sendTokenRefresh(apiVersion, vertx, testContext, refreshToken, v2RefreshDecryptSecret, expectedHttpCode, handler, Collections.emptyMap()); } - private void sendTokenRefresh(String apiVersion, Vertx vertx, String headerName, String headerValue, VertxTestContext testContext, String refreshToken, String v2RefreshDecryptSecret, int expectedHttpCode, - Handler handler) { + private void sendTokenRefresh(String apiVersion, Vertx vertx, VertxTestContext testContext, String refreshToken, String v2RefreshDecryptSecret, int expectedHttpCode, + Handler handler, Map additionalHeaders) { if (apiVersion.equals("v2")) { WebClient client = WebClient.create(vertx); HttpRequest refreshHttpRequest = client.postAbs(getUrlForEndpoint("v2/token/refresh")); - if (headerName != null) { - refreshHttpRequest.putHeader(headerName, headerValue); + refreshHttpRequest.putHeader(HttpHeaders.CONTENT_TYPE.toString(), HttpMediaType.TEXT_PLAIN.getType()); + for (Map.Entry entry : additionalHeaders.entrySet()) { + refreshHttpRequest.putHeader(entry.getKey(), entry.getValue()); } + refreshHttpRequest - .putHeader("content-type", "text/plain") .sendBuffer(Buffer.buffer(refreshToken.getBytes(StandardCharsets.UTF_8)), testContext.succeeding(response -> testContext.verify(() -> { assertEquals(expectedHttpCode, response.statusCode()); if (response.statusCode() == 200 && v2RefreshDecryptSecret != null) { - byte[] decrypted = AesGcm.decrypt(Utils.decodeBase64String(response.bodyAsString()), 0, Utils.decodeBase64String(v2RefreshDecryptSecret)); + byte[] byteResp = new byte[0]; + if (response.headers().contains(HttpHeaders.CONTENT_TYPE, HttpMediaType.APPLICATION_OCTET_STREAM.getType(), true)) { + byteResp = response.bodyAsBuffer().getBytes(); + } else if (response.headers().contains(HttpHeaders.CONTENT_TYPE, HttpMediaType.TEXT_PLAIN.getType(), true)) { + byteResp = Utils.decodeBase64String(response.bodyAsString()); + } + byte[] decrypted = AesGcm.decrypt(byteResp, 0, Utils.decodeBase64String(v2RefreshDecryptSecret)); JsonObject respJson = new JsonObject(new String(decrypted, StandardCharsets.UTF_8)); if (respJson.getString("status").equals("success")) @@ -332,7 +398,7 @@ private void sendTokenRefresh(String apiVersion, Vertx vertx, String headerName, assertEquals(expectedHttpCode, response.statusCode()); JsonObject json = response.bodyAsJsonObject(); handler.handle(json); - })), headerName, headerValue); + })), additionalHeaders); } } @@ -364,41 +430,52 @@ private JsonObject tryParseResponse(HttpResponse resp) { } private void get(Vertx vertx, String endpoint, Handler>> handler) { - WebClient client = WebClient.create(vertx); - ClientKey ck = clientKeyProvider.getClientKey(""); - HttpRequest req = client.getAbs(getUrlForEndpoint(endpoint)); - if (ck != null) - req.putHeader("Authorization", "Bearer " + clientKey); - req.send(handler); + get(vertx, endpoint, handler, Collections.emptyMap()); } - private void get(Vertx vertx, String endpoint, Handler>> handler, String headerName, String headerValue) { + private void get(Vertx vertx, String endpoint, Handler>> handler, Map additionalHeaders) { WebClient client = WebClient.create(vertx); ClientKey ck = clientKeyProvider.getClientKey(""); HttpRequest req = client.getAbs(getUrlForEndpoint(endpoint)); if (ck != null) { req.putHeader("Authorization", "Bearer " + clientKey); } - if (headerName != null) { - req.putHeader(headerName, headerValue); + + for (Map.Entry entry : additionalHeaders.entrySet()) { + req.putHeader(entry.getKey(), entry.getValue()); } + req.send(handler); } - private void post(Vertx vertx, String endpoint, JsonObject body, Handler>> handler) { + private void post(Vertx vertx, String endpoint, JsonObject body, Handler>> handler, Map additionalHeaders) { WebClient client = WebClient.create(vertx); ClientKey ck = clientKeyProvider.getClientKey(""); HttpRequest req = client.postAbs(getUrlForEndpoint(endpoint)); if (ck != null) req.putHeader("Authorization", "Bearer " + clientKey); + + for (Map.Entry entry : additionalHeaders.entrySet()) { + req.putHeader(entry.getKey(), entry.getValue()); + } + req.sendJsonObject(body, handler); } private void postV2(ClientKey ck, Vertx vertx, String endpoint, JsonObject body, long nonce, String referer, Handler>> handler) { - postV2(ck, vertx, endpoint, body, nonce, referer, handler, null, null); + postV2(ck, vertx, endpoint, body, nonce, referer, handler, Collections.emptyMap()); } - private void postV2(ClientKey ck, Vertx vertx, String endpoint, JsonObject body, long nonce, String referer, Handler>> handler, String headerName, String headerValue) { + + private void postV2(ClientKey ck, Vertx vertx, String endpoint, JsonObject body, long nonce, String referer, Handler>> handler, Map additionalHeaders) { WebClient client = WebClient.create(vertx); + final String apiKey = ck == null ? "" : clientKey; + HttpRequest request = client.postAbs(getUrlForEndpoint(endpoint)) + .putHeader(HttpHeaders.AUTHORIZATION.toString(), "Bearer " + apiKey) + .putHeader(HttpHeaders.CONTENT_TYPE.toString(), HttpMediaType.TEXT_PLAIN.getType()); + + for (Map.Entry entry : additionalHeaders.entrySet()) { + request.putHeader(entry.getKey(), entry.getValue()); + } Buffer b = Buffer.buffer(); b.appendLong(now.toEpochMilli()); @@ -409,22 +486,19 @@ private void postV2(ClientKey ck, Vertx vertx, String endpoint, JsonObject body, Buffer bufBody = Buffer.buffer(); bufBody.appendByte((byte) 1); + byte[] payload = b.getBytes(); if (ck != null) { - bufBody.appendBytes(AesGcm.encrypt(b.getBytes(), ck.getSecretBytes())); + bufBody.appendBytes(AesGcm.encrypt(payload, ck.getSecretBytes())); } - - final String apiKey = ck == null ? "" : clientKey; - HttpRequest request = client.postAbs(getUrlForEndpoint(endpoint)) - .putHeader("Authorization", "Bearer " + apiKey) - .putHeader("content-type", "text/plain"); - if (headerName != null) { - request.putHeader(headerName, headerValue); - } - if (referer != null) { request.putHeader("Referer", referer); } - request.sendBuffer(Buffer.buffer(Utils.toBase64String(bufBody.getBytes()).getBytes(StandardCharsets.UTF_8)), handler); + + if (request.headers().contains(HttpHeaders.CONTENT_TYPE.toString(), HttpMediaType.APPLICATION_OCTET_STREAM.getType(), true)) { + request.sendBuffer(bufBody, handler); + } else { + request.sendBuffer(Buffer.buffer(Utils.toBase64String(bufBody.getBytes()).getBytes(StandardCharsets.UTF_8)), handler); + } } private void checkEncryptionKeysResponse(JsonObject response, KeysetKey... expectedKeys) { @@ -523,7 +597,6 @@ private void setupKeysetsMock(Keyset... keysets) { private void setupKeysetsMock(Map keysets) { KeysetSnapshot keysetSnapshot = new KeysetSnapshot(keysets); - when(keysetProvider.getSnapshot(any())).thenReturn(keysetSnapshot); //note that this getSnapshot() overload should be removed; it ignores the argument passed in when(keysetProvider.getSnapshot()).thenReturn(keysetSnapshot); } @@ -595,14 +668,14 @@ protected void setupSiteKey(int siteId, int keyId, int keysetId) { } private void generateTokens(String apiVersion, Vertx vertx, String inputType, String input, Handler handler) { - generateTokens(apiVersion, vertx, inputType, input, handler, null, null); + generateTokens(apiVersion, vertx, inputType, input, handler, Collections.emptyMap()); } - private void generateTokens(String apiVersion, Vertx vertx, String inputType, String input, Handler handler, String headerName, String headerValue) { + private void generateTokens(String apiVersion, Vertx vertx, String inputType, String input, Handler handler, Map additionalHeaders) { String v1Param = inputType + "=" + urlEncode(input); JsonObject v2Payload = new JsonObject(); v2Payload.put(inputType, input); - sendTokenGenerate(apiVersion, vertx, v1Param, v2Payload, 200, null, handler, true, headerName, headerValue); + sendTokenGenerate(apiVersion, vertx, v1Param, v2Payload, 200, null, handler, true, additionalHeaders); } private static void assertEqualsClose(Instant expected, Instant actual, int withinSeconds) { @@ -622,26 +695,27 @@ private void assertTokenStatusMetrics(Integer siteId, TokenResponseStatsCollecto assertEquals(1, actual); } - private byte[] getRawUidFromIdentity(IdentityType identityType, String identityString, String firstLevelSalt, String rotatingSalt) { - return getRawUid(identityType, identityString, firstLevelSalt, rotatingSalt, getIdentityScope(), useIdentityV3()); + private byte[] getRawUidFromRawDii(DiiType diiType, String rawDii, String firstLevelSalt, String rotatingSalt) { + return getRawUid(diiType, rawDii, firstLevelSalt, rotatingSalt, getIdentityScope(), useRawUidV3()); } - private static byte[] getRawUid(IdentityType identityType, String identityString, String firstLevelSalt, String rotatingSalt, IdentityScope identityScope, boolean useIdentityV3) { + public static byte[] getRawUid(DiiType diiType, String rawDii, String firstLevelSalt, String rotatingSalt, IdentityScope identityScope, boolean useIdentityV3) { return !useIdentityV3 - ? TokenUtils.getRawUidV2FromIdentity(identityString, firstLevelSalt, rotatingSalt) - : TokenUtils.getRawUidV3FromIdentity(identityScope, identityType, identityString, firstLevelSalt, rotatingSalt); + ? TokenUtils.getRawUidV2FromRawDii(rawDii, firstLevelSalt, rotatingSalt) + : TokenUtils.getRawUidV3FromRawDii(identityScope, diiType, rawDii, firstLevelSalt, rotatingSalt); } - public static byte[] getRawUid(IdentityType identityType, String identityString, IdentityScope identityScope, boolean useIdentityV3) { + public static byte[] getRawUid(DiiType diiType, String rawDii, IdentityScope identityScope, boolean useIdentityV3) { return !useIdentityV3 - ? TokenUtils.getRawUidV2FromIdentity(identityString, firstLevelSalt, rotatingSalt123.getSalt()) - : TokenUtils.getRawUidV3FromIdentity(identityScope, identityType, identityString, firstLevelSalt, rotatingSalt123.getSalt()); + ? TokenUtils.getRawUidV2FromRawDii(rawDii, firstLevelSalt, rotatingSalt123.currentSalt()) + : TokenUtils.getRawUidV3FromRawDii(identityScope, diiType, rawDii, firstLevelSalt,rotatingSalt123.currentSalt()); } - private byte[] getRawUidFromIdentityHash(IdentityType identityType, String identityString, String firstLevelSalt, String rotatingSalt) { - return !useIdentityV3() - ? TokenUtils.getRawUidV2FromIdentityHash(identityString, firstLevelSalt, rotatingSalt) - : TokenUtils.getRawUidV3FromIdentityHash(getIdentityScope(), identityType, identityString, firstLevelSalt, rotatingSalt); + + private byte[] getRawUidFromHashedDii(DiiType diiType, String hashedDii, String firstLevelSalt, String rotatingSalt) { + return !useRawUidV3() + ? TokenUtils.getRawUidV2FromHashedDii(hashedDii, firstLevelSalt, rotatingSalt) + : TokenUtils.getRawUidV3FromHashedDii(getIdentityScope(), diiType, hashedDii, firstLevelSalt, rotatingSalt); } private JsonObject createBatchEmailsRequestPayload() { @@ -664,9 +738,9 @@ private JsonObject setupIdentityMapServiceLinkTest() { return req; } - protected TokenVersion getTokenVersion() {return TokenVersion.V2;} + protected TokenVersion getTokenVersion() {return TokenVersion.V4;} - final boolean useIdentityV3() { return getTokenVersion() != TokenVersion.V2; } + protected boolean useRawUidV3() { return false; } protected IdentityScope getIdentityScope() { return IdentityScope.UID2; } protected void addAdditionalTokenGenerateParams(JsonObject payload) {} @@ -676,8 +750,9 @@ void verticleDeployed(Vertx vertx, VertxTestContext testContext) { } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) - void keyLatestNoAcl(String apiVersion, Vertx vertx, VertxTestContext testContext) { + @CsvSource({"v2, text/plain", + "v2, application/octet-stream"}) + void keyLatestNoAcl(String apiVersion, String contentType, Vertx vertx, VertxTestContext testContext) { fakeAuth(5, Role.ID_READER); Keyset[] keysets = { new Keyset(MasterKeysetId, MasterKeySiteId, "masterKeyset", null, now.getEpochSecond(), true, true), @@ -695,11 +770,11 @@ void keyLatestNoAcl(String apiVersion, Vertx vertx, VertxTestContext testContext System.out.println(respJson); checkEncryptionKeysResponse(respJson, encryptionKeys); testContext.completeNow(); - }); + }, Map.of(HttpHeaders.CONTENT_TYPE.toString(), contentType)); } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void keyLatestWithAcl(String apiVersion, Vertx vertx, VertxTestContext testContext) { fakeAuth(5, Role.ID_READER); Keyset[] keysets = { @@ -724,7 +799,7 @@ void keyLatestWithAcl(String apiVersion, Vertx vertx, VertxTestContext testConte } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void keyLatestClientBelongsToReservedSiteId(String apiVersion, Vertx vertx, VertxTestContext testContext) { fakeAuth(AdvertisingTokenSiteId, Role.ID_READER); KeysetKey[] encryptionKeys = { @@ -736,7 +811,7 @@ void keyLatestClientBelongsToReservedSiteId(String apiVersion, Vertx vertx, Vert } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void keyLatestHideRefreshKey(String apiVersion, Vertx vertx, VertxTestContext testContext) { fakeAuth(5, Role.ID_READER); Keyset[] keysets = { @@ -761,11 +836,11 @@ void keyLatestHideRefreshKey(String apiVersion, Vertx vertx, VertxTestContext te } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenGenerateBothEmailAndHashSpecified(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String emailAddress = "test@uid2.com"; - final String emailHash = TokenUtils.getIdentityHashString(emailAddress); + final String emailHash = TokenUtils.getHashedDiiString(emailAddress); fakeAuth(clientSiteId, Role.GENERATOR); setupSalts(); setupKeys(); @@ -786,7 +861,7 @@ void tokenGenerateBothEmailAndHashSpecified(String apiVersion, Vertx vertx, Vert } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenGenerateNoEmailOrHashSpecified(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.GENERATOR); @@ -813,25 +888,27 @@ private void assertStatsCollector(String path, String referer, String apiContact assertEquals(siteId, messageItem.getSiteId()); } - private AdvertisingTokenInput validateAndGetToken(EncryptedTokenEncoder encoder, JsonObject body, IdentityType identityType) { //See UID2-79+Token+and+ID+format+v3 + private AdvertisingTokenRequest validateAndGetToken(EncryptedTokenEncoder encoder, JsonObject body, DiiType diiType) { //See UID2-79+Token+and+ID+format+v3 final String advertisingTokenString = body.getString("advertising_token"); - validateAdvertisingToken(advertisingTokenString, getTokenVersion(), getIdentityScope(), identityType); - AdvertisingTokenInput advertisingTokenInput = encoder.decodeAdvertisingToken(advertisingTokenString); - if (getTokenVersion() == TokenVersion.V4) { - assertEquals(identityType, advertisingTokenInput.rawUidIdentity.identityType); + validateAdvertisingToken(advertisingTokenString, getTokenVersion(), getIdentityScope(), diiType); + AdvertisingTokenRequest advertisingTokenRequest = encoder.decodeAdvertisingToken(advertisingTokenString); + // without useRawUidV3() the assert will be trigger as there's no IdentityType in v4 token generated with + // a raw UID v2 as old raw UID format doesn't store the identity type (and scope) + if (useRawUidV3() && getTokenVersion() == TokenVersion.V4) { + assertEquals(diiType, advertisingTokenRequest.rawUid.diiType()); } - return advertisingTokenInput; + return advertisingTokenRequest; } - public static void validateAdvertisingToken(String advertisingTokenString, TokenVersion tokenVersion, IdentityScope identityScope, IdentityType identityType) { + public static void validateAdvertisingToken(String advertisingTokenString, TokenVersion tokenVersion, IdentityScope identityScope, DiiType diiType) { if (tokenVersion == TokenVersion.V2) { assertEquals("Ag", advertisingTokenString.substring(0, 2)); } else { String firstChar = advertisingTokenString.substring(0, 1); if (identityScope == IdentityScope.UID2) { - assertEquals(identityType == IdentityType.Email ? "A" : "B", firstChar); + assertEquals(diiType == DiiType.Email ? "A" : "B", firstChar); } else { - assertEquals(identityType == IdentityType.Email ? "E" : "F", firstChar); + assertEquals(diiType == DiiType.Email ? "E" : "F", firstChar); } String secondChar = advertisingTokenString.substring(1, 2); @@ -848,27 +925,25 @@ public static void validateAdvertisingToken(String advertisingTokenString, Token } } - RefreshTokenInput decodeRefreshToken(EncryptedTokenEncoder encoder, String refreshTokenString, IdentityType identityType) { - RefreshTokenInput refreshTokenInput = encoder.decodeRefreshToken(refreshTokenString); - assertEquals(getIdentityScope(), refreshTokenInput.firstLevelHashIdentity.identityScope); - assertEquals(identityType, refreshTokenInput.firstLevelHashIdentity.identityType); - return refreshTokenInput; - } - RefreshTokenInput decodeRefreshToken(EncryptedTokenEncoder encoder, String refreshTokenString) { - return decodeRefreshToken(encoder, refreshTokenString, IdentityType.Email); + TokenRefreshRequest decodeRefreshToken(EncryptedTokenEncoder encoder, String refreshTokenString, DiiType diiType) { + TokenRefreshRequest tokenRefreshRequest = encoder.decodeRefreshToken(refreshTokenString); + assertEquals(getIdentityScope(), tokenRefreshRequest.firstLevelHash.identityScope()); + assertEquals(diiType, tokenRefreshRequest.firstLevelHash.diiType()); + return tokenRefreshRequest; } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) - void identityMapNewClientNoPolicySpecified(String apiVersion, Vertx vertx, VertxTestContext testContext) { + @CsvSource({"v2, text/plain", + "v2, application/octet-stream"}) + void identityMapNewClientNoPolicySpecified(String apiVersion, String contentType, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, newClientCreationDateTime, Role.MAPPER); setupSalts(); setupKeys(); // the clock value shouldn't matter here - when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) - .thenReturn(now.minus(1, ChronoUnit.HOURS)); + when(optOutStore.getLatestEntry(any(FirstLevelHash.class))) + .thenReturn(now.minus(1, ChronoUnit.HOURS)); JsonObject req = new JsonObject(); JsonArray emails = new JsonArray(); @@ -883,7 +958,7 @@ void identityMapNewClientNoPolicySpecified(String apiVersion, Vertx vertx, Vertx Assertions.assertEquals(emails.getString(0), unmappedArr.getJsonObject(0).getString("identifier")); Assertions.assertEquals("optout", unmappedArr.getJsonObject(0).getString("reason")); testContext.completeNow(); - }); + }, Map.of(HttpHeaders.CONTENT_TYPE.toString(), contentType)); } @ParameterizedTest @@ -894,8 +969,8 @@ void identityMapNewClientWrongPolicySpecified(String apiVersion, String policyPa setupSalts(); setupKeys(); // the clock value shouldn't matter here - when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) - .thenReturn(now.minus(1, ChronoUnit.HOURS)); + when(optOutStore.getLatestEntry(any(FirstLevelHash.class))) + .thenReturn(now.minus(1, ChronoUnit.HOURS)); JsonObject req = new JsonObject(); JsonArray emails = new JsonArray(); emails.add("random-optout-user@email.io"); @@ -999,6 +1074,236 @@ void identityMapNewClientWrongPolicySpecifiedOlderKeySuccessful(String policyPar }); } + @ParameterizedTest + @ValueSource(strings = { + "text/plain", + "application/octet-stream" + }) + void v3IdentityMapMixedInputSuccess(String contentType, Vertx vertx, VertxTestContext testContext) { + final int clientSiteId = 201; + fakeAuth(clientSiteId, Role.MAPPER); + setupSalts(); + + var lastUpdated = Instant.now().minus(1, DAYS); + var refreshFrom = lastUpdated.plus(30, DAYS); + + SaltEntry salt = new SaltEntry(1, "1", lastUpdated.toEpochMilli(), "salt", refreshFrom.toEpochMilli(), "previousSalt", null, null); + when(saltProviderSnapshot.getRotatingSalt(any())).thenReturn(salt); + + var phoneHash = TokenUtils.getHashedDiiString("+15555555555"); + JsonObject request = new JsonObject(String.format(""" + { + "email": ["test1@uid2.com", "test2@uid2.com"], + "phone": [], + "phone_hash": ["%s"] + } + """, phoneHash) + ); + + send("v2", vertx, "v3/identity/map", false, null, request, 200, respJson -> { + JsonObject body = respJson.getJsonObject("body"); + assertEquals(Set.of("email", "email_hash", "phone", "phone_hash"), body.fieldNames()); + + var mappedEmails = body.getJsonArray("email"); + assertEquals(2, mappedEmails.size()); + + var mappedEmailExpected1 = JsonObject.of( + "u", EncodingUtils.toBase64String(getRawUidFromRawDii(DiiType.Email, "test1@uid2.com", firstLevelSalt, salt.currentSalt())), + "p", EncodingUtils.toBase64String(getRawUidFromRawDii(DiiType.Email,"test1@uid2.com", firstLevelSalt, salt.previousSalt())), + "r", refreshFrom.getEpochSecond() + ); + assertEquals(mappedEmailExpected1, mappedEmails.getJsonObject(0)); + + var mappedEmailExpected2 = JsonObject.of( + "u", EncodingUtils.toBase64String(getRawUidFromRawDii(DiiType.Email, "test2@uid2.com", firstLevelSalt, salt.currentSalt())), + "p", EncodingUtils.toBase64String(getRawUidFromRawDii(DiiType.Email,"test2@uid2.com", firstLevelSalt, salt.previousSalt())), + "r", refreshFrom.getEpochSecond() + ); + assertEquals(mappedEmailExpected2, mappedEmails.getJsonObject(1)); + + assertEquals(0, body.getJsonArray("email_hash").size()); + assertEquals(0, body.getJsonArray("phone").size()); + + var mappedPhoneHash = body.getJsonArray("phone_hash"); + assertEquals(1, mappedPhoneHash.size()); + + var mappedPhoneHashExpected = JsonObject.of( + "u", EncodingUtils.toBase64String(getRawUidFromHashedDii(DiiType.Phone, phoneHash, firstLevelSalt, salt.currentSalt())), + "p", EncodingUtils.toBase64String(getRawUidFromHashedDii(DiiType.Phone, phoneHash, firstLevelSalt, salt.previousSalt())), + "r", refreshFrom.getEpochSecond() + ); + assertEquals(mappedPhoneHashExpected, mappedPhoneHash.getJsonObject(0)); + + assertEquals("success", respJson.getString("status")); + testContext.completeNow(); + }, Map.of(HttpHeaders.CONTENT_TYPE.toString(), contentType)); + } + + @Test + void v3IdentityMapUnmappedIdentitiesOptoutAndInvalid(Vertx vertx, VertxTestContext testContext) { + final int clientSiteId = 201; + fakeAuth(clientSiteId, Role.MAPPER); + setupSalts(); + + // optout + when(this.optOutStore.getLatestEntry(any())).thenReturn(Instant.now()); + + Instant lastUpdated = Instant.now().minus(1, DAYS); + Instant refreshFrom = lastUpdated.plus(30, DAYS); + + SaltEntry salt = new SaltEntry(1, "1", lastUpdated.toEpochMilli(), "salt", refreshFrom.toEpochMilli(), "previousSalt", null, null); + when(saltProviderSnapshot.getRotatingSalt(any())).thenReturn(salt); + + JsonObject request = new JsonObject(""" + { "email": ["test1@uid2.com", "invalid_email"] } + """ + ); + + send("v2", vertx, "v3/identity/map", false, null, request, 200, respJson -> { + JsonObject body = respJson.getJsonObject("body"); + + JsonObject expected = new JsonObject(""" + { + "email": [{"e": "optout"}, {"e": "invalid identifier"}], + "email_hash": [], + "phone": [], + "phone_hash": [] + } + """); + + assertEquals(expected, body); + + testContext.completeNow(); + }); + } + + @ParameterizedTest + @ValueSource(strings = {"{\"email\": []}", "{\"email_hash\": null}" }) + void v3IdentityMapEmptyInputFormats(String inputPayload, Vertx vertx, VertxTestContext testContext) { + final int clientSiteId = 201; + fakeAuth(clientSiteId, Role.MAPPER); + setupSalts(); + + Instant lastUpdated = Instant.now().minus(1, DAYS); + Instant refreshFrom = lastUpdated.plus(30, DAYS); + + SaltEntry salt = new SaltEntry(1, "1", lastUpdated.toEpochMilli(), "salt", refreshFrom.toEpochMilli(), "previousSalt", null, null); + when(saltProviderSnapshot.getRotatingSalt(any())).thenReturn(salt); + + JsonObject request = inputPayload == null ? null : new JsonObject(inputPayload); + + send("v2", vertx, "v3/identity/map", false, null, request, 200, respJson -> { + JsonObject body = respJson.getJsonObject("body"); + JsonObject expected = new JsonObject(""" + { + "email": [], + "email_hash": [], + "phone": [], + "phone_hash": [] + } + """); + assertEquals(expected, body); + testContext.completeNow(); + }); + } + + @ParameterizedTest + @NullSource + @ValueSource(strings = {"{}"}) + void v3IdentityMapMissingValidInputKeys(String inputPayload, Vertx vertx, VertxTestContext testContext) { + final int clientSiteId = 201; + fakeAuth(clientSiteId, Role.MAPPER); + + JsonObject request = inputPayload == null ? null : new JsonObject(inputPayload); + + send("v2", vertx, "v3/identity/map", false, null, request, 400, respJson -> { + assertEquals("Required Parameter Missing: one or more of [email, email_hash, phone, phone_hash] must be specified", respJson.getString("message")); + testContext.completeNow(); + }); + } + + @ParameterizedTest + @ValueSource(strings = {"{\"invalid_key\": []}", + "{\"email\": [ null ]}", + "{\"email\": [ \"some_email\", null ]}" + }) + void v3IdentityMapIncorrectInputFormats(String inputPayload, Vertx vertx, VertxTestContext testContext) { + final int clientSiteId = 201; + fakeAuth(clientSiteId, Role.MAPPER); + + JsonObject request = new JsonObject(inputPayload); + + send("v2", vertx, "v3/identity/map", false, null, request, 400, respJson -> { + assertEquals("Incorrect request format", respJson.getString("message")); + testContext.completeNow(); + }); + } + + @ParameterizedTest + @NullSource + @ValueSource(strings = {"previousSalt"}) + void v3IdentityMapNoPreviousAdvertisingId(String previousSalt, Vertx vertx, VertxTestContext testContext) { + final int clientSiteId = 201; + fakeAuth(clientSiteId, Role.MAPPER); + setupSalts(); + + var lastUpdatedOver90Days = Instant.now().minus(120, DAYS).toEpochMilli(); + var refreshFrom = Instant.now().plus(30, DAYS); + + SaltEntry salt = new SaltEntry(1, "1", lastUpdatedOver90Days, "salt", refreshFrom.toEpochMilli(), previousSalt, null, null); + when(saltProviderSnapshot.getRotatingSalt(any())).thenReturn(salt); + + JsonObject request = new JsonObject(""" + { "email": ["test1@uid2.com"] } + """); + + send("v2", vertx, "v3/identity/map", false, null, request, 200, respJson -> { + JsonObject body = respJson.getJsonObject("body"); + var mappedEmails = body.getJsonArray("email"); + + var expectedMappedEmails = JsonObject.of( + "u", EncodingUtils.toBase64String(getRawUidFromRawDii(DiiType.Email, "test1@uid2.com", firstLevelSalt, salt.currentSalt())), + "p", null, + "r", refreshFrom.getEpochSecond() + ); + assertEquals(expectedMappedEmails, mappedEmails.getJsonObject(0)); + + testContext.completeNow(); + }); + } + + @Test + void v3IdentityMapOutdatedRefreshFrom(Vertx vertx, VertxTestContext testContext) { + final int clientSiteId = 201; + fakeAuth(clientSiteId, Role.MAPPER); + setupSalts(); + + Instant asOf = Instant.now(); + var lastUpdated = asOf.minus(120, DAYS).toEpochMilli(); + var outdatedRefreshFrom = asOf.minus(30, DAYS).toEpochMilli(); + + SaltEntry salt = new SaltEntry(1, "1", lastUpdated, "salt", outdatedRefreshFrom, null, null, null); + when(saltProviderSnapshot.getRotatingSalt(any())).thenReturn(salt); + + JsonObject request = new JsonObject(""" + { "email": ["test1@uid2.com"] } + """); + + send("v2", vertx, "v3/identity/map", false, null, request, 200, respJson -> { + JsonObject body = respJson.getJsonObject("body"); + var mappedEmails = body.getJsonArray("email"); + + var expectedMappedEmails = JsonObject.of( + "u", EncodingUtils.toBase64String(getRawUidFromRawDii(DiiType.Email, "test1@uid2.com", firstLevelSalt, salt.currentSalt())), + "p", null, + "r", asOf.truncatedTo(DAYS).plus(1, DAYS).getEpochSecond() + ); + assertEquals(expectedMappedEmails, mappedEmails.getJsonObject(0)); + + testContext.completeNow(); + }); + } + @Test void tokenGenerateNewClientNoPolicySpecified(Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; @@ -1128,8 +1433,8 @@ void tokenGenerateNewClientWrongPolicySpecifiedOlderKeySuccessful(String policyP "policy,+01234567890,Phone", "optout_check,someoptout@example.com,Email", "optout_check,+01234567890,Phone"}) - void tokenGenerateOptOutToken(String policyParameterKey, String identity, IdentityType identityType, - Vertx vertx, VertxTestContext testContext) { + void tokenGenerateOptOutToken(String policyParameterKey, String identity, DiiType diiType, + Vertx vertx, VertxTestContext testContext) { ClientKey oldClientKey = new ClientKey( null, null, @@ -1148,13 +1453,13 @@ void tokenGenerateOptOutToken(String policyParameterKey, String identity, Identi setupKeys(); JsonObject v2Payload = new JsonObject(); - v2Payload.put(identityType.name().toLowerCase(), identity); + v2Payload.put(diiType.name().toLowerCase(), identity); v2Payload.put(policyParameterKey, OptoutCheckPolicy.DoNotRespect.policy); sendTokenGenerate("v2", vertx, "", v2Payload, 200, json -> { - InputUtil.InputVal optOutTokenInput = identityType == IdentityType.Email ? + InputUtil.InputVal optOutTokenInput = diiType == DiiType.Email ? InputUtil.InputVal.validEmail(OptOutTokenIdentityForEmail, OptOutTokenIdentityForEmail) : InputUtil.InputVal.validPhone(OptOutIdentityForPhone, OptOutTokenIdentityForPhone); @@ -1167,23 +1472,23 @@ void tokenGenerateOptOutToken(String policyParameterKey, String identity, Identi EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, identityType); - RefreshTokenInput refreshTokenInput = encoder.decodeRefreshToken(body.getString("decrypted_refresh_token")); - final byte[] rawUid = getRawUidFromIdentity(identityType, + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, diiType); + TokenRefreshRequest tokenRefreshRequest = encoder.decodeRefreshToken(body.getString("decrypted_refresh_token")); + final byte[] rawUid = getRawUidFromRawDii(diiType, optOutTokenInput.getNormalized(), firstLevelSalt, - rotatingSalt123.getSalt()); - final byte[] firstLevelHash = TokenUtils.getFirstLevelHashFromIdentity(optOutTokenInput.getNormalized(), firstLevelSalt); - assertArrayEquals(rawUid, advertisingTokenInput.rawUidIdentity.rawUid); - assertArrayEquals(firstLevelHash, refreshTokenInput.firstLevelHashIdentity.firstLevelHash); + rotatingSalt123.currentSalt()); + final byte[] firstLevelHash = TokenUtils.getFirstLevelHashFromRawDii(optOutTokenInput.getNormalized(), firstLevelSalt); + assertArrayEquals(rawUid, advertisingTokenRequest.rawUid.rawUid()); + assertArrayEquals(firstLevelHash, tokenRefreshRequest.firstLevelHash.firstLevelHash()); String advertisingTokenString = body.getString("advertising_token"); final Instant now = Instant.now(); final String token = advertisingTokenString; - final boolean matchedOptedOutIdentity = this.uidOperatorVerticle.getIdService().advertisingTokenMatches(token, optOutTokenInput.toHashedDiiIdentity(getIdentityScope(), 0, now), now); + final boolean matchedOptedOutIdentity = this.uidOperatorVerticle.getIdService().advertisingTokenMatches(token, optOutTokenInput.toHashedDii(getIdentityScope()), now); assertTrue(matchedOptedOutIdentity); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenGenerated()); - assertTrue(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenOptedOut()); + assertFalse(advertisingTokenRequest.privacyBits.isClientSideTokenGenerated()); + assertTrue(advertisingTokenRequest.privacyBits.isClientSideTokenOptedOut()); assertTokenStatusMetrics( 201, @@ -1191,7 +1496,7 @@ void tokenGenerateOptOutToken(String policyParameterKey, String identity, Identi TokenResponseStatsCollector.ResponseStatus.Success, TokenResponseStatsCollector.PlatformType.Other); - sendTokenRefresh("v2", vertx, ClientVersionHeader, tvosClientVersionHeaderValue, testContext, body.getString("refresh_token"), body.getString("refresh_response_key"), 200, refreshRespJson -> + sendTokenRefresh("v2", vertx, testContext, body.getString("refresh_token"), body.getString("refresh_response_key"), 200, refreshRespJson -> { assertEquals("optout", refreshRespJson.getString("status")); JsonObject refreshBody = refreshRespJson.getJsonObject("body"); @@ -1202,13 +1507,59 @@ void tokenGenerateOptOutToken(String policyParameterKey, String identity, Identi TokenResponseStatsCollector.ResponseStatus.OptOut, TokenResponseStatsCollector.PlatformType.InApp); testContext.completeNow(); - }); + }, Map.of(ClientVersionHeader, tvosClientVersionHeaderValue)); + }); + } + + @ParameterizedTest // TODO: remove test after optout check phase 3 + @CsvSource({"policy,someoptout@example.com,Email", + "policy,+01234567890,Phone", + "optout_check,someoptout@example.com,Email", + "optout_check,+01234567890,Phone"}) + void tokenGenerateOptOutTokenWithDisableOptoutTokenFF(String policyParameterKey, String identity, DiiType identityType, + Vertx vertx, VertxTestContext testContext) { + ClientKey oldClientKey = new ClientKey( + null, + null, + Utils.toBase64String(clientSecret), + "test-contact", + newClientCreationDateTime.minusSeconds(5), + Set.of(Role.GENERATOR), + 201, + null + ); + when(clientKeyProvider.get(any())).thenReturn(oldClientKey); + when(clientKeyProvider.getClientKey(any())).thenReturn(oldClientKey); + when(clientKeyProvider.getOldestClientKey(201)).thenReturn(oldClientKey); + when(this.optOutStore.getLatestEntry(any())).thenReturn(Instant.now()); + setupSalts(); + setupKeys(); + + JsonObject v2Payload = new JsonObject(); + v2Payload.put(identityType.name().toLowerCase(), identity); + v2Payload.put(policyParameterKey, OptoutCheckPolicy.DoNotRespect.policy); + + sendTokenGenerate("v2", vertx, + "", v2Payload, 200, + json -> { + assertEquals("optout", json.getString("status")); + + decodeV2RefreshToken(json); + + assertTokenStatusMetrics( + 201, + TokenResponseStatsCollector.Endpoint.GenerateV2, + TokenResponseStatsCollector.ResponseStatus.OptOut, + TokenResponseStatsCollector.PlatformType.Other); + + testContext.completeNow(); }); } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) - void tokenGenerateForEmail(String apiVersion, Vertx vertx, VertxTestContext testContext) { + @CsvSource({"v2, text/plain", + "v2, application/octet-stream"}) + void tokenGenerateForEmail(String apiVersion, String contentType, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String emailAddress = "test@uid2.com"; fakeAuth(clientSiteId, Role.GENERATOR); @@ -1227,32 +1578,48 @@ void tokenGenerateForEmail(String apiVersion, Vertx vertx, VertxTestContext test assertNotNull(body); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Email); - - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenGenerated()); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenOptedOut()); - assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, DiiType.Email); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), DiiType.Email); - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token")); - assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenInput.firstLevelHashIdentity.firstLevelHash); - - assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("identity_expires")), 10); - assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_expires")), 10); - assertEqualsClose(now.plusMillis(refreshIdentityAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_from")), 10); + assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, tokenRefreshRequest, clientSiteId, + getRawUidFromRawDii(DiiType.Email, emailAddress, firstLevelSalt, rotatingSalt123.currentSalt()), + PrivacyBits.DEFAULT, + body, + TokenUtils.getFirstLevelHashFromRawDii(emailAddress, firstLevelSalt)); assertStatsCollector("/" + apiVersion + "/token/generate", null, "test-contact", clientSiteId); testContext.completeNow(); - }); + }, + Map.of(HttpHeaders.CONTENT_TYPE.toString(), contentType)); + } + + public void assertAdvertisingTokenRefreshTokenRequests(AdvertisingTokenRequest advertisingTokenRequest, TokenRefreshRequest tokenRefreshRequest, + int expectedClientSiteId, byte[] expectedRawUidIdentity, PrivacyBits expectedPrivacyBits, JsonObject identityResponse, byte[] firstLevelHashIdentity) { + + assertEquals(expectedClientSiteId, advertisingTokenRequest.sourcePublisher.siteId); + assertEquals(expectedClientSiteId, tokenRefreshRequest.sourcePublisher.siteId); + assertArrayEquals(expectedRawUidIdentity, advertisingTokenRequest.rawUid.rawUid()); + + verifyPrivacyBits(expectedPrivacyBits, advertisingTokenRequest, tokenRefreshRequest); + verifyFirstLevelHashIdentityAndEstablishedAt(firstLevelHashIdentity, tokenRefreshRequest, identityResponse, advertisingTokenRequest.establishedAt); + + assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(identityResponse.getLong("identity_expires")), 10); + assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(identityResponse.getLong("refresh_expires")), 10); + assertEqualsClose(now.plusMillis(refreshIdentityAfter.toMillis()), Instant.ofEpochMilli(identityResponse.getLong("refresh_from")), 10); + } + + public void verifyPrivacyBits(PrivacyBits expectedValue, AdvertisingTokenRequest advertisingTokenRequest, + TokenRefreshRequest tokenRefreshRequest) { + assertEquals(advertisingTokenRequest.privacyBits, expectedValue); + assertEquals(advertisingTokenRequest.privacyBits, tokenRefreshRequest.privacyBits); } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenGenerateForEmailHash(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; - final String emailHash = TokenUtils.getIdentityHashString("test@uid2.com"); + final String emailHash = TokenUtils.getHashedDiiString("test@uid2.com"); fakeAuth(clientSiteId, Role.GENERATOR); setupSalts(); setupKeys(); @@ -1269,85 +1636,115 @@ void tokenGenerateForEmailHash(String apiVersion, Vertx vertx, VertxTestContext assertNotNull(body); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Email); - - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenGenerated()); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenOptedOut()); - assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentityHash(IdentityType.Email, emailHash, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, DiiType.Email); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, apiVersion.equals("v2") ? body.getString("decrypted_refresh_token") : body.getString("refresh_token"), DiiType.Email); - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, apiVersion.equals("v2") ? body.getString("decrypted_refresh_token") : body.getString("refresh_token")); - assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentityHash(emailHash, firstLevelSalt), refreshTokenInput.firstLevelHashIdentity.firstLevelHash); - - assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("identity_expires")), 10); - assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_expires")), 10); - assertEqualsClose(now.plusMillis(refreshIdentityAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_from")), 10); + assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, tokenRefreshRequest, clientSiteId, + getRawUidFromHashedDii(DiiType.Email, emailHash, firstLevelSalt, rotatingSalt123.currentSalt()), + PrivacyBits.DEFAULT, + body, + TokenUtils.getFirstLevelHashFromHashedDii(emailHash, firstLevelSalt)); testContext.completeNow(); }); } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) - void tokenGenerateThenRefresh(String apiVersion, Vertx vertx, VertxTestContext testContext) { + @CsvSource({"v2, text/plain", + "v2, application/octet-stream"}) + void tokenGenerateThenRefresh(String apiVersion, String contentType, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String emailAddress = "test@uid2.com"; fakeAuth(clientSiteId, Role.GENERATOR); setupSalts(); setupKeys(); + Map additionalHeaders = Map.of(ClientVersionHeader, iosClientVersionHeaderValue, + HttpHeaders.CONTENT_TYPE.toString(), contentType); + generateTokens(apiVersion, vertx, "email", emailAddress, genRespJson -> { assertEquals("success", genRespJson.getString("status")); JsonObject bodyJson = genRespJson.getJsonObject("body"); assertNotNull(bodyJson); String genRefreshToken = bodyJson.getString("refresh_token"); + EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); + + AdvertisingTokenRequest firstAdvertisingTokenRequest = validateAndGetToken(encoder, bodyJson, + DiiType.Email); + + TokenRefreshRequest firstTokenRefreshRequest = decodeRefreshToken(encoder, bodyJson.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), DiiType.Email); + + assertEquals(firstAdvertisingTokenRequest.establishedAt, firstTokenRefreshRequest.firstLevelHash.establishedAt()); when(this.optOutStore.getLatestEntry(any())).thenReturn(null); - sendTokenRefresh(apiVersion, vertx, ClientVersionHeader, iosClientVersionHeaderValue, testContext, genRefreshToken, bodyJson.getString("refresh_response_key"), 200, refreshRespJson -> + byte[] expectedRawUidIdentity = getRawUidFromRawDii(DiiType.Email, emailAddress, firstLevelSalt, + rotatingSalt123.currentSalt()); + byte[] expectedFirstLevelHashIdentity = TokenUtils.getFirstLevelHashFromRawDii(emailAddress, firstLevelSalt); + + assertAdvertisingTokenRefreshTokenRequests(firstAdvertisingTokenRequest, firstTokenRefreshRequest, clientSiteId, + expectedRawUidIdentity, + PrivacyBits.DEFAULT, + bodyJson, + expectedFirstLevelHashIdentity); + + + sendTokenRefresh(apiVersion, vertx, testContext, genRefreshToken, bodyJson.getString("refresh_response_key"), 200, refreshRespJson -> { assertEquals("success", refreshRespJson.getString("status")); JsonObject refreshBody = refreshRespJson.getJsonObject("body"); assertNotNull(refreshBody); - EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - - AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, refreshBody, IdentityType.Email); - - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenGenerated()); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenOptedOut()); - assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, refreshBody, DiiType.Email); String refreshTokenStringNew = refreshBody.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"); assertNotEquals(genRefreshToken, refreshTokenStringNew); - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, refreshTokenStringNew); - assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenInput.firstLevelHashIdentity.firstLevelHash); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, refreshTokenStringNew, DiiType.Email); - assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("identity_expires")), 10); - assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("refresh_expires")), 10); - assertEqualsClose(now.plusMillis(refreshIdentityAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("refresh_from")), 10); + // assert if the ad/refresh tokens from original token/generate is same as the ad/refresh tokens from token/refresh + assertAdvertisingTokenRefreshTokenRequests( + advertisingTokenRequest, + firstTokenRefreshRequest, + clientSiteId, + expectedRawUidIdentity, + PrivacyBits.DEFAULT, + bodyJson, + expectedFirstLevelHashIdentity); + assertAdvertisingTokenRefreshTokenRequests( + firstAdvertisingTokenRequest, + tokenRefreshRequest, + clientSiteId, + expectedRawUidIdentity, + PrivacyBits.DEFAULT, + bodyJson, + expectedFirstLevelHashIdentity); + assertAdvertisingTokenRefreshTokenRequests( + advertisingTokenRequest, + tokenRefreshRequest, + clientSiteId, + expectedRawUidIdentity, + PrivacyBits.DEFAULT, + bodyJson, + expectedFirstLevelHashIdentity); assertTokenStatusMetrics( clientSiteId, - apiVersion.equals("v1") ? TokenResponseStatsCollector.Endpoint.GenerateV1 : TokenResponseStatsCollector.Endpoint.GenerateV2, + TokenResponseStatsCollector.Endpoint.GenerateV2, TokenResponseStatsCollector.ResponseStatus.Success, TokenResponseStatsCollector.PlatformType.InApp); assertTokenStatusMetrics( clientSiteId, - apiVersion.equals("v1") ? TokenResponseStatsCollector.Endpoint.RefreshV1 : TokenResponseStatsCollector.Endpoint.RefreshV2, + TokenResponseStatsCollector.Endpoint.RefreshV2, TokenResponseStatsCollector.ResponseStatus.Success, TokenResponseStatsCollector.PlatformType.InApp); testContext.completeNow(); - }); - }, ClientVersionHeader, iosClientVersionHeaderValue); + }, additionalHeaders); + }, additionalHeaders); } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenGenerateThenRefreshSaltsExpired(String apiVersion, Vertx vertx, VertxTestContext testContext) { when(saltProviderSnapshot.getExpires()).thenReturn(Instant.now().minus(1, ChronoUnit.HOURS)); final int clientSiteId = 201; @@ -1356,6 +1753,8 @@ void tokenGenerateThenRefreshSaltsExpired(String apiVersion, Vertx vertx, VertxT setupSalts(); setupKeys(); + Map additionalHeaders = Map.of(ClientVersionHeader, androidClientVersionHeaderValue); + generateTokens(apiVersion, vertx, "email", emailAddress, genRespJson -> { assertEquals("success", genRespJson.getString("status")); JsonObject bodyJson = genRespJson.getJsonObject("body"); @@ -1365,25 +1764,25 @@ void tokenGenerateThenRefreshSaltsExpired(String apiVersion, Vertx vertx, VertxT when(this.optOutStore.getLatestEntry(any())).thenReturn(null); - sendTokenRefresh(apiVersion, vertx, ClientVersionHeader, androidClientVersionHeaderValue, testContext, genRefreshToken, bodyJson.getString("refresh_response_key"), 200, refreshRespJson -> + sendTokenRefresh(apiVersion, vertx, testContext, genRefreshToken, bodyJson.getString("refresh_response_key"), 200, refreshRespJson -> { assertEquals("success", refreshRespJson.getString("status")); JsonObject refreshBody = refreshRespJson.getJsonObject("body"); assertNotNull(refreshBody); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, refreshBody, IdentityType.Email); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, refreshBody, DiiType.Email); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenGenerated()); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenOptedOut()); - assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); + assertFalse(advertisingTokenRequest.privacyBits.isClientSideTokenGenerated()); + assertFalse(advertisingTokenRequest.privacyBits.isClientSideTokenOptedOut()); + assertEquals(clientSiteId, advertisingTokenRequest.sourcePublisher.siteId); + assertArrayEquals(getRawUidFromRawDii(DiiType.Email, emailAddress, firstLevelSalt, rotatingSalt123.currentSalt()), advertisingTokenRequest.rawUid.rawUid()); String refreshTokenStringNew = refreshBody.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"); assertNotEquals(genRefreshToken, refreshTokenStringNew); - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, refreshTokenStringNew); - assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenInput.firstLevelHashIdentity.firstLevelHash); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, refreshTokenStringNew, DiiType.Email); + assertEquals(clientSiteId, tokenRefreshRequest.sourcePublisher.siteId); + assertArrayEquals(TokenUtils.getFirstLevelHashFromRawDii(emailAddress, firstLevelSalt), tokenRefreshRequest.firstLevelHash.firstLevelHash()); assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("identity_expires")), 10); assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("refresh_expires")), 10); @@ -1391,20 +1790,20 @@ void tokenGenerateThenRefreshSaltsExpired(String apiVersion, Vertx vertx, VertxT assertTokenStatusMetrics( clientSiteId, - apiVersion.equals("v1") ? TokenResponseStatsCollector.Endpoint.GenerateV1 : TokenResponseStatsCollector.Endpoint.GenerateV2, + TokenResponseStatsCollector.Endpoint.GenerateV2, TokenResponseStatsCollector.ResponseStatus.Success, TokenResponseStatsCollector.PlatformType.InApp); assertTokenStatusMetrics( clientSiteId, - apiVersion.equals("v1") ? TokenResponseStatsCollector.Endpoint.RefreshV1 : TokenResponseStatsCollector.Endpoint.RefreshV2, + TokenResponseStatsCollector.Endpoint.RefreshV2, TokenResponseStatsCollector.ResponseStatus.Success, TokenResponseStatsCollector.PlatformType.InApp); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(true); testContext.completeNow(); - }); - }, ClientVersionHeader, androidClientVersionHeaderValue); + }, additionalHeaders); + }, additionalHeaders); } @Test @@ -1428,18 +1827,18 @@ void tokenGenerateThenRefreshNoActiveKey(Vertx vertx, VertxTestContext testConte String genRefreshToken = bodyJson.getString("refresh_token"); setupKeys(true); - sendTokenRefresh("v2", vertx, ClientVersionHeader, androidClientVersionHeaderValue, testContext, genRefreshToken, bodyJson.getString("refresh_response_key"), 500, refreshRespJson -> + sendTokenRefresh("v2", vertx, testContext, genRefreshToken, bodyJson.getString("refresh_response_key"), 500, refreshRespJson -> { assertFalse(refreshRespJson.containsKey("body")); assertEquals("No active encryption key available", refreshRespJson.getString("message")); testContext.completeNow(); - }); + }, Map.of(ClientVersionHeader, androidClientVersionHeaderValue)); }); } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenGenerateThenValidateWithEmail_Match(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String emailAddress = ValidateIdentityForEmail; @@ -1469,7 +1868,7 @@ void tokenGenerateThenValidateWithEmail_Match(String apiVersion, Vertx vertx, Ve } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenGenerateThenValidateWithEmailHash_Match(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.GENERATOR); @@ -1498,7 +1897,7 @@ void tokenGenerateThenValidateWithEmailHash_Match(String apiVersion, Vertx vertx } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenGenerateThenValidateWithBothEmailAndEmailHash(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String emailAddress = ValidateIdentityForEmail; @@ -1529,7 +1928,7 @@ void tokenGenerateThenValidateWithBothEmailAndEmailHash(String apiVersion, Vertx } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenGenerateUsingCustomSiteKey(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 4; final int clientKeysetId = 201; @@ -1550,20 +1949,20 @@ void tokenGenerateUsingCustomSiteKey(String apiVersion, Vertx vertx, VertxTestCo assertNotNull(body); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Email); - assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, DiiType.Email); + assertEquals(clientSiteId, advertisingTokenRequest.sourcePublisher.siteId); + assertArrayEquals(getRawUidFromRawDii(DiiType.Email, emailAddress, firstLevelSalt, rotatingSalt123.currentSalt()), advertisingTokenRequest.rawUid.rawUid()); - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token")); - assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenInput.firstLevelHashIdentity.firstLevelHash); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), DiiType.Email); + assertEquals(clientSiteId, tokenRefreshRequest.sourcePublisher.siteId); + assertArrayEquals(TokenUtils.getFirstLevelHashFromRawDii(emailAddress, firstLevelSalt), tokenRefreshRequest.firstLevelHash.firstLevelHash()); testContext.completeNow(); }); } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenGenerateSaltsExpired(String apiVersion, Vertx vertx, VertxTestContext testContext) { when(saltProviderSnapshot.getExpires()).thenReturn(Instant.now().minus(1, ChronoUnit.HOURS)); final int clientSiteId = 201; @@ -1584,16 +1983,18 @@ void tokenGenerateSaltsExpired(String apiVersion, Vertx vertx, VertxTestContext assertNotNull(body); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Email); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, DiiType.Email); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenGenerated()); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenOptedOut()); - assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); + assertTrue(advertisingTokenRequest.privacyBits.isLegacyBitSet()); + assertEquals(advertisingTokenRequest.privacyBits, PrivacyBits.DEFAULT); + assertFalse(advertisingTokenRequest.privacyBits.isClientSideTokenGenerated()); + assertFalse(advertisingTokenRequest.privacyBits.isClientSideTokenOptedOut()); + assertEquals(clientSiteId, advertisingTokenRequest.sourcePublisher.siteId); + assertArrayEquals(getRawUidFromRawDii(DiiType.Email, emailAddress, firstLevelSalt, rotatingSalt123.currentSalt()), advertisingTokenRequest.rawUid.rawUid()); - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token")); - assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenInput.firstLevelHashIdentity.firstLevelHash); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), DiiType.Email); + assertEquals(clientSiteId, tokenRefreshRequest.sourcePublisher.siteId); + assertArrayEquals(TokenUtils.getFirstLevelHashFromRawDii(emailAddress, firstLevelSalt), tokenRefreshRequest.firstLevelHash.firstLevelHash()); assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("identity_expires")), 10); assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_expires")), 10); @@ -1628,15 +2029,15 @@ void tokenGenerateNoActiveKey(Vertx vertx, VertxTestContext testContext) { } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenRefreshNoToken(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.GENERATOR); - sendTokenRefresh(apiVersion, vertx, null, null, testContext, "", "", 400, json -> { + sendTokenRefresh(apiVersion, vertx, testContext, "", "", 400, json -> { assertEquals("invalid_token", json.getString("status")); assertTokenStatusMetrics( clientSiteId, - apiVersion.equals("v1") ? TokenResponseStatsCollector.Endpoint.RefreshV1 : TokenResponseStatsCollector.Endpoint.RefreshV2, + TokenResponseStatsCollector.Endpoint.RefreshV2, TokenResponseStatsCollector.ResponseStatus.InvalidToken, TokenResponseStatsCollector.PlatformType.Other); testContext.completeNow(); @@ -1644,26 +2045,26 @@ void tokenRefreshNoToken(String apiVersion, Vertx vertx, VertxTestContext testCo } @ParameterizedTest - @CsvSource({"v1,asdf", "v2,asdf", "v1,invalidBase64%%%%", "v2,invalidBase64%%%%"}) + @CsvSource({"v2,asdf", "v2,invalidBase64%%%%"}) void tokenRefreshInvalidTokenAuthenticated(String apiVersion, String token, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.GENERATOR); - sendTokenRefresh(apiVersion, vertx, ORIGIN_HEADER, "example.com", testContext, token, "", 400, json -> { + sendTokenRefresh(apiVersion, vertx, testContext, token, "", 400, json -> { assertEquals("invalid_token", json.getString("status")); assertTokenStatusMetrics( clientSiteId, - apiVersion.equals("v1") ? TokenResponseStatsCollector.Endpoint.RefreshV1 : TokenResponseStatsCollector.Endpoint.RefreshV2, + TokenResponseStatsCollector.Endpoint.RefreshV2, TokenResponseStatsCollector.ResponseStatus.InvalidToken, TokenResponseStatsCollector.PlatformType.HasOriginHeader); testContext.completeNow(); - }); + }, Map.of(ORIGIN_HEADER, "https://example.com")); } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenRefreshInvalidTokenUnauthenticated(String apiVersion, Vertx vertx, VertxTestContext testContext) { - sendTokenRefresh(apiVersion, vertx, null, null, testContext, "abcd", "", 400, json -> { + sendTokenRefresh(apiVersion, vertx, testContext, "abcd", "", 400, json -> { assertEquals("error", json.getString("status")); testContext.completeNow(); }); @@ -1677,7 +2078,7 @@ private void generateRefreshToken(String apiVersion, Vertx vertx, String identit } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void captureDurationsBetweenRefresh(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.GENERATOR); @@ -1690,13 +2091,13 @@ void captureDurationsBetweenRefresh(String apiVersion, Vertx vertx, VertxTestCon sendTokenRefresh(apiVersion, vertx, testContext, refreshToken, bodyJson.getString("refresh_response_key"), 200, refreshRespJson -> { assertEquals("success", refreshRespJson.getString("status")); assertEquals(300, Metrics.globalRegistry - .get("uid2.token_refresh_duration_seconds") + .get("uid2_token_refresh_duration_seconds") .tag("api_contact", "test-contact") .tag("site_id", String.valueOf(clientSiteId)) .summary().mean()); assertEquals(1, Metrics.globalRegistry - .get("uid2.advertising_token_expired_on_refresh") + .get("uid2_advertising_token_expired_on_refresh_total") .tag("site_id", String.valueOf(clientSiteId)) .tag("is_expired", "false") .counter().count()); @@ -1707,7 +2108,7 @@ void captureDurationsBetweenRefresh(String apiVersion, Vertx vertx, VertxTestCon } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void captureExpiredAdvertisingTokenStatus(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.GENERATOR); @@ -1721,7 +2122,7 @@ void captureExpiredAdvertisingTokenStatus(String apiVersion, Vertx vertx, VertxT assertEquals("success", refreshRespJson.getString("status")); assertEquals(1, Metrics.globalRegistry - .get("uid2.advertising_token_expired_on_refresh") + .get("uid2_advertising_token_expired_on_refresh_total") .tag("site_id", String.valueOf(clientSiteId)) .tag("is_expired", "true") .counter().count()); @@ -1732,7 +2133,7 @@ void captureExpiredAdvertisingTokenStatus(String apiVersion, Vertx vertx, VertxT } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenRefreshExpiredTokenAuthenticated(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.GENERATOR); @@ -1745,14 +2146,14 @@ void tokenRefreshExpiredTokenAuthenticated(String apiVersion, Vertx vertx, Vertx sendTokenRefresh(apiVersion, vertx, testContext, refreshToken, bodyJson.getString("refresh_response_key"), 400, refreshRespJson -> { assertEquals("expired_token", refreshRespJson.getString("status")); assertNotNull(Metrics.globalRegistry - .get("uid2_refresh_token_received_count").counter()); + .get("uid2_refresh_token_received_count_total").counter()); testContext.completeNow(); }); }); } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenRefreshExpiredTokenUnauthenticated(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String emailAddress = "test@uid2.com"; @@ -1765,14 +2166,14 @@ void tokenRefreshExpiredTokenUnauthenticated(String apiVersion, Vertx vertx, Ver sendTokenRefresh(apiVersion, vertx, testContext, refreshToken, "", 400, refreshRespJson -> { assertEquals("error", refreshRespJson.getString("status")); assertNotNull(Metrics.globalRegistry - .get("uid2_refresh_token_received_count").counter()); + .get("uid2_refresh_token_received_count_total").counter()); testContext.completeNow(); }); }); } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenRefreshOptOut(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String emailAddress = "test@uid2.com"; @@ -1786,7 +2187,7 @@ void tokenRefreshOptOut(String apiVersion, Vertx vertx, VertxTestContext testCon assertEquals("optout", refreshRespJson.getString("status")); assertTokenStatusMetrics( clientSiteId, - apiVersion.equals("v1") ? TokenResponseStatsCollector.Endpoint.RefreshV1 : TokenResponseStatsCollector.Endpoint.RefreshV2, + TokenResponseStatsCollector.Endpoint.RefreshV2, TokenResponseStatsCollector.ResponseStatus.OptOut, TokenResponseStatsCollector.PlatformType.Other); testContext.completeNow(); @@ -1795,7 +2196,7 @@ void tokenRefreshOptOut(String apiVersion, Vertx vertx, VertxTestContext testCon } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenRefreshOptOutBeforeLogin(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String emailAddress = "test@uid2.com"; @@ -1815,49 +2216,10 @@ void tokenRefreshOptOutBeforeLogin(String apiVersion, Vertx vertx, VertxTestCont }); } - @Test - void v2HandleV1RefreshToken(Vertx vertx, VertxTestContext testContext) { - final int clientSiteId = 201; - fakeAuth(201, Role.GENERATOR); - final String emailAddress = "test@uid2.com"; - - generateRefreshToken("v1", vertx, "email", emailAddress, clientSiteId, genRespJson -> { - JsonObject bodyJson = genRespJson.getJsonObject("body"); - String refreshToken = bodyJson.getString("refresh_token"); - - sendTokenRefresh("v2", vertx, testContext, refreshToken, null, 200, refreshRespJson -> { - assertEquals("success", refreshRespJson.getString("status")); - - JsonObject refreshBodyJson = refreshRespJson.getJsonObject("body"); - assertNotNull(refreshBodyJson.getString("refresh_response_key")); - - decodeV2RefreshToken(refreshRespJson); - - testContext.completeNow(); - }); - }); - } - - @Test - void v1HandleV2RefreshToken(Vertx vertx, VertxTestContext testContext) { - final int clientSiteId = 201; - fakeAuth(201, Role.GENERATOR); - final String emailAddress = "test@uid2.com"; - - generateRefreshToken("v2", vertx, "email", emailAddress, clientSiteId, genRespJson -> { - JsonObject bodyJson = genRespJson.getJsonObject("body"); - String refreshToken = bodyJson.getString("refresh_token"); - - sendTokenRefresh("v1", vertx, testContext, refreshToken, null, 200, refreshRespJson -> { - assertEquals("success", refreshRespJson.getString("status")); - testContext.completeNow(); - }); - }); - } - @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) - void tokenValidateWithEmail_Mismatch(String apiVersion, Vertx vertx, VertxTestContext testContext) { + @CsvSource({"v2, text/plain", + "v2, application/octet-stream"}) + void tokenValidateWithEmail_Mismatch(String apiVersion, String contentType, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String emailAddress = ValidateIdentityForEmail; fakeAuth(clientSiteId, Role.GENERATOR); @@ -1873,11 +2235,12 @@ void tokenValidateWithEmail_Mismatch(String apiVersion, Vertx vertx, VertxTestCo assertEquals("success", respJson.getString("status")); testContext.completeNow(); - }); + }, + Map.of(HttpHeaders.CONTENT_TYPE.toString(), contentType)); } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenValidateWithEmailHash_Mismatch(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.GENERATOR); @@ -1896,121 +2259,8 @@ void tokenValidateWithEmailHash_Mismatch(String apiVersion, Vertx vertx, VertxTe }); } - @Test - void identityMapBothEmailAndHashSpecified(Vertx vertx, VertxTestContext testContext) { - final int clientSiteId = 201; - final String emailAddress = "test@uid2.com"; - final String emailHash = TokenUtils.getIdentityHashString(emailAddress); - fakeAuth(clientSiteId, Role.MAPPER); - setupSalts(); - setupKeys(); - get(vertx, "v1/identity/map?email=" + emailAddress + "&email_hash=" + urlEncode(emailHash), ar -> { - assertTrue(ar.succeeded()); - HttpResponse response = ar.result(); - assertEquals(400, response.statusCode()); - JsonObject json = response.bodyAsJsonObject(); - assertFalse(json.containsKey("body")); - assertEquals("client_error", json.getString("status")); - - testContext.completeNow(); - }); - } - - @Test - void identityMapNoEmailOrHashSpecified(Vertx vertx, VertxTestContext testContext) { - final int clientSiteId = 201; - fakeAuth(clientSiteId, Role.MAPPER); - setupSalts(); - setupKeys(); - get(vertx, "v1/identity/map", ar -> { - assertTrue(ar.succeeded()); - HttpResponse response = ar.result(); - assertEquals(400, response.statusCode()); - JsonObject json = response.bodyAsJsonObject(); - assertFalse(json.containsKey("body")); - assertEquals("client_error", json.getString("status")); - - testContext.completeNow(); - }); - } - - @Test - void identityMapForEmail(Vertx vertx, VertxTestContext testContext) { - final int clientSiteId = 201; - final String emailAddress = "test@uid2.com"; - fakeAuth(clientSiteId, Role.MAPPER); - setupSalts(); - setupKeys(); - get(vertx, "v1/identity/map?email=" + emailAddress, ar -> { - assertTrue(ar.succeeded()); - HttpResponse response = ar.result(); - assertEquals(200, response.statusCode()); - JsonObject json = response.bodyAsJsonObject(); - assertEquals("success", json.getString("status")); - JsonObject body = json.getJsonObject("body"); - assertNotNull(body); - - assertEquals(emailAddress, body.getString("identifier")); - assertFalse(body.getString("advertising_id").isEmpty()); - assertFalse(body.getString("bucket_id").isEmpty()); - - testContext.completeNow(); - }); - } - - @Test - void identityMapForSaltsExpired(Vertx vertx, VertxTestContext testContext) { - when(saltProviderSnapshot.getExpires()).thenReturn(Instant.now().minus(1, ChronoUnit.HOURS)); - final int clientSiteId = 201; - final String emailAddress = "test@uid2.com"; - fakeAuth(clientSiteId, Role.MAPPER); - setupSalts(); - setupKeys(); - get(vertx, "v1/identity/map?email=" + emailAddress, ar -> { - assertTrue(ar.succeeded()); - HttpResponse response = ar.result(); - assertEquals(200, response.statusCode()); - JsonObject json = response.bodyAsJsonObject(); - assertEquals("success", json.getString("status")); - JsonObject body = json.getJsonObject("body"); - assertNotNull(body); - - assertEquals(emailAddress, body.getString("identifier")); - assertFalse(body.getString("advertising_id").isEmpty()); - assertFalse(body.getString("bucket_id").isEmpty()); - - verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(true); - - testContext.completeNow(); - }); - } - - @Test - void identityMapForEmailHash(Vertx vertx, VertxTestContext testContext) { - final int clientSiteId = 201; - final String emailHash = TokenUtils.getIdentityHashString("test@uid2.com"); - fakeAuth(clientSiteId, Role.MAPPER); - setupSalts(); - setupKeys(); - get(vertx, "v1/identity/map?email_hash=" + urlEncode(emailHash), ar -> { - assertTrue(ar.succeeded()); - HttpResponse response = ar.result(); - assertEquals(200, response.statusCode()); - JsonObject json = response.bodyAsJsonObject(); - assertEquals("success", json.getString("status")); - JsonObject body = json.getJsonObject("body"); - assertNotNull(body); - - assertEquals(emailHash, body.getString("identifier")); - assertFalse(body.getString("advertising_id").isEmpty()); - assertFalse(body.getString("bucket_id").isEmpty()); - - testContext.completeNow(); - }); - } - @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void identityMapBatchBothEmailAndHashEmpty(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.MAPPER); @@ -2030,7 +2280,7 @@ void identityMapBatchBothEmailAndHashEmpty(String apiVersion, Vertx vertx, Vertx } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void identityMapBatchBothEmailAndHashSpecified(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.MAPPER); @@ -2044,7 +2294,7 @@ void identityMapBatchBothEmailAndHashSpecified(String apiVersion, Vertx vertx, V req.put("email_hash", emailHashes); emails.add("test1@uid2.com"); - emailHashes.add(TokenUtils.getIdentityHashString("test2@uid2.com")); + emailHashes.add(TokenUtils.getHashedDiiString("test2@uid2.com")); send(apiVersion, vertx, apiVersion + "/identity/map", false, null, req, 400, respJson -> { assertFalse(respJson.containsKey("body")); @@ -2054,7 +2304,7 @@ void identityMapBatchBothEmailAndHashSpecified(String apiVersion, Vertx vertx, V } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void identityMapBatchNoEmailOrHashSpecified(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.MAPPER); @@ -2072,7 +2322,7 @@ void identityMapBatchNoEmailOrHashSpecified(String apiVersion, Vertx vertx, Vert } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void identityMapSingleEmailProvided(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.MAPPER); @@ -2093,7 +2343,7 @@ void identityMapSingleEmailProvided(String apiVersion, Vertx vertx, VertxTestCon } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void identityMapSingleEmailHashProvided(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.MAPPER); @@ -2114,7 +2364,7 @@ void identityMapSingleEmailHashProvided(String apiVersion, Vertx vertx, VertxTes } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void identityMapSinglePhoneProvided(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.MAPPER); @@ -2135,7 +2385,7 @@ void identityMapSinglePhoneProvided(String apiVersion, Vertx vertx, VertxTestCon } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void identityMapSinglePhoneHashProvided(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.MAPPER); @@ -2156,7 +2406,7 @@ void identityMapSinglePhoneHashProvided(String apiVersion, Vertx vertx, VertxTes } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void identityMapBatchEmails(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.MAPPER); @@ -2172,7 +2422,7 @@ void identityMapBatchEmails(String apiVersion, Vertx vertx, VertxTestContext tes } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void identityMapBatchEmailHashes(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.MAPPER); @@ -2183,8 +2433,8 @@ void identityMapBatchEmailHashes(String apiVersion, Vertx vertx, VertxTestContex JsonArray hashes = new JsonArray(); req.put("email_hash", hashes); final String[] email_hashes = { - TokenUtils.getIdentityHashString("test1@uid2.com"), - TokenUtils.getIdentityHashString("test2@uid2.com"), + TokenUtils.getHashedDiiString("test1@uid2.com"), + TokenUtils.getHashedDiiString("test2@uid2.com"), }; for (String email_hash : email_hashes) { @@ -2198,7 +2448,7 @@ void identityMapBatchEmailHashes(String apiVersion, Vertx vertx, VertxTestContex } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void identityMapBatchEmailsOneEmailInvalid(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.MAPPER); @@ -2220,7 +2470,7 @@ void identityMapBatchEmailsOneEmailInvalid(String apiVersion, Vertx vertx, Vertx } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void identityMapBatchEmailsNoEmails(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.MAPPER); @@ -2238,7 +2488,7 @@ void identityMapBatchEmailsNoEmails(String apiVersion, Vertx vertx, VertxTestCon } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void identityMapBatchRequestTooLarge(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.MAPPER); @@ -2259,13 +2509,13 @@ void identityMapBatchRequestTooLarge(String apiVersion, Vertx vertx, VertxTestCo private static Stream optOutStatusRequestData() { List rawUIDS = Arrays.asList("RUQbFozFwnmPVjDx8VMkk9vJoNXUJImKnz2h9RfzzM24", - "qAmIGxqLk_RhOtm4f1nLlqYewqSma8fgvjEXYnQ3Jr0K", - "r3wW2uvJkwmeFcbUwSeM6BIpGF8tX38wtPfVc4wYyo71", - "e6SA-JVAXnvk8F1MUtzsMOyWuy5Xqe15rLAgqzSGiAbz"); + "qAmIGxqLk_RhOtm4f1nLlqYewqSma8fgvjEXYnQ3Jr0K", + "r3wW2uvJkwmeFcbUwSeM6BIpGF8tX38wtPfVc4wYyo71", + "e6SA-JVAXnvk8F1MUtzsMOyWuy5Xqe15rLAgqzSGiAbz"); Map optedOutIdsCase1 = new HashMap<>(); - optedOutIdsCase1.put(rawUIDS.get(0), Instant.now().minus(1, ChronoUnit.DAYS).getEpochSecond()); - optedOutIdsCase1.put(rawUIDS.get(1), Instant.now().minus(2, ChronoUnit.DAYS).getEpochSecond()); + optedOutIdsCase1.put(rawUIDS.get(0), Instant.now().minus(1, DAYS).getEpochSecond()); + optedOutIdsCase1.put(rawUIDS.get(1), Instant.now().minus(2, DAYS).getEpochSecond()); optedOutIdsCase1.put(rawUIDS.get(2), -1L); optedOutIdsCase1.put(rawUIDS.get(3), -1L); @@ -2273,10 +2523,10 @@ private static Stream optOutStatusRequestData() { optedOutIdsCase2.put(rawUIDS.get(2), -1L); optedOutIdsCase2.put(rawUIDS.get(3), -1L); return Stream.of( - Arguments.arguments(optedOutIdsCase1, 2, Role.MAPPER), - Arguments.arguments(optedOutIdsCase1, 2, Role.ID_READER), - Arguments.arguments(optedOutIdsCase1, 2, Role.SHARER), - Arguments.arguments(optedOutIdsCase2, 0, Role.MAPPER) + Arguments.arguments(optedOutIdsCase1, 2, Role.MAPPER), + Arguments.arguments(optedOutIdsCase1, 2, Role.ID_READER), + Arguments.arguments(optedOutIdsCase1, 2, Role.SHARER), + Arguments.arguments(optedOutIdsCase2, 0, Role.MAPPER) ); } @@ -2344,8 +2594,9 @@ void optOutStatusValidationError(JsonObject requestJson, String errorMsg, Vertx }); } - @Test - void optOutStatusUnauthorized(Vertx vertx, VertxTestContext testContext) { + @ParameterizedTest + @ValueSource(strings = {"text/plain", "application/octet-stream"}) + void optOutStatusUnauthorized(String contentType, Vertx vertx, VertxTestContext testContext) { fakeAuth(126, Role.GENERATOR); setupSalts(); setupKeys(); @@ -2353,11 +2604,12 @@ void optOutStatusUnauthorized(Vertx vertx, VertxTestContext testContext) { send("v2", vertx, "v2/optout/status", false, null, new JsonObject(), 401, respJson -> { assertEquals(com.uid2.shared.Const.ResponseStatus.Unauthorized, respJson.getString("status")); testContext.completeNow(); - }); + }, Map.of(HttpHeaders.CONTENT_TYPE.toString(), contentType)); } - @Test - void LogoutV2(Vertx vertx, VertxTestContext testContext) { + @ParameterizedTest + @ValueSource(strings = {"text/plain", "application/octet-stream"}) + void LogoutV2(String contentType, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.OPTOUT); setupSalts(); @@ -2367,16 +2619,17 @@ void LogoutV2(Vertx vertx, VertxTestContext testContext) { req.put("email", "test@uid2.com"); doAnswer(invocation -> { - Handler> handler = invocation.getArgument(2); + Handler> handler = invocation.getArgument(4); handler.handle(Future.succeededFuture(Instant.now())); return null; - }).when(this.optOutStore).addEntry(any(), any(), any()); + }).when(this.optOutStore).addEntry(any(), any(), eq("uid-trace-id"), eq("test-instance-id"), any()); send("v2", vertx, "v2/token/logout", false, null, req, 200, respJson -> { assertEquals("success", respJson.getString("status")); assertEquals("OK", respJson.getJsonObject("body").getString("optout")); testContext.completeNow(); - }); + }, Map.of(Audit.UID_TRACE_ID_HEADER, "uid-trace-id", + HttpHeaders.CONTENT_TYPE.toString(), contentType)); } @Test @@ -2391,10 +2644,10 @@ void LogoutV2SaltsExpired(Vertx vertx, VertxTestContext testContext) { req.put("email", "test@uid2.com"); doAnswer(invocation -> { - Handler> handler = invocation.getArgument(2); + Handler> handler = invocation.getArgument(4); handler.handle(Future.succeededFuture(Instant.now())); return null; - }).when(this.optOutStore).addEntry(any(), any(), any()); + }).when(this.optOutStore).addEntry(any(), any(), any(), any(), any()); send("v2", vertx, "v2/token/logout", false, null, req, 200, respJson -> { assertEquals("success", respJson.getString("status")); @@ -2407,11 +2660,11 @@ void LogoutV2SaltsExpired(Vertx vertx, VertxTestContext testContext) { } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenGenerateBothPhoneAndHashSpecified(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String phone = "+15555555555"; - final String phoneHash = TokenUtils.getIdentityHashString(phone); + final String phoneHash = TokenUtils.getHashedDiiString(phone); fakeAuth(clientSiteId, Role.GENERATOR); setupSalts(); setupKeys(); @@ -2430,7 +2683,7 @@ void tokenGenerateBothPhoneAndHashSpecified(String apiVersion, Vertx vertx, Vert } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenGenerateBothPhoneAndEmailSpecified(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String phone = "+15555555555"; @@ -2453,13 +2706,13 @@ void tokenGenerateBothPhoneAndEmailSpecified(String apiVersion, Vertx vertx, Ver } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenGenerateBothPhoneHashAndEmailHashSpecified(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String phone = "+15555555555"; - final String phoneHash = TokenUtils.getIdentityHashString(phone); + final String phoneHash = TokenUtils.getHashedDiiString(phone); final String emailAddress = "test@uid2.com"; - final String emailHash = TokenUtils.getIdentityHashString(emailAddress); + final String emailHash = TokenUtils.getHashedDiiString(emailAddress); fakeAuth(clientSiteId, Role.GENERATOR); setupSalts(); setupKeys(); @@ -2478,7 +2731,7 @@ void tokenGenerateBothPhoneHashAndEmailHashSpecified(String apiVersion, Vertx ve } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenGenerateForPhone(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String phone = "+15555555555"; @@ -2496,31 +2749,37 @@ void tokenGenerateForPhone(String apiVersion, Vertx vertx, VertxTestContext test assertNotNull(body); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Phone); - - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenGenerated()); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenOptedOut()); - assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentity(IdentityType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); - - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Phone); - assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(phone, firstLevelSalt), refreshTokenInput.firstLevelHashIdentity.firstLevelHash); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, DiiType.Phone); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), DiiType.Phone); - assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("identity_expires")), 10); - assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_expires")), 10); - assertEqualsClose(now.plusMillis(refreshIdentityAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_from")), 10); + assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, tokenRefreshRequest, clientSiteId, + getRawUidFromRawDii(DiiType.Phone, phone, firstLevelSalt, rotatingSalt123.currentSalt()), + PrivacyBits.DEFAULT, + body, + TokenUtils.getFirstLevelHashFromRawDii(phone, firstLevelSalt)); testContext.completeNow(); }); } + void verifyFirstLevelHashIdentityAndEstablishedAt(byte[] expectedFirstLevelHash, + TokenRefreshRequest tokenRefreshRequest, + JsonObject receivedJsonBody, + Instant expectedEstablishedTime) { + + assertArrayEquals(expectedFirstLevelHash, tokenRefreshRequest.firstLevelHash.firstLevelHash()); + assertEquals(expectedEstablishedTime, tokenRefreshRequest.firstLevelHash.establishedAt()); + assertTrue(tokenRefreshRequest.firstLevelHash.establishedAt().toEpochMilli() < receivedJsonBody.getLong("identity_expires") ); + assertTrue(tokenRefreshRequest.firstLevelHash.establishedAt().toEpochMilli() < receivedJsonBody.getLong("refresh_expires") ); + assertTrue(tokenRefreshRequest.firstLevelHash.establishedAt().toEpochMilli() < receivedJsonBody.getLong("refresh_from") ); + } + @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenGenerateForPhoneHash(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String phone = "+15555555555"; - final String phoneHash = TokenUtils.getIdentityHashString(phone); + final String phoneHash = TokenUtils.getHashedDiiString(phone); fakeAuth(clientSiteId, Role.GENERATOR); setupSalts(); setupKeys(); @@ -2535,27 +2794,22 @@ void tokenGenerateForPhoneHash(String apiVersion, Vertx vertx, VertxTestContext assertNotNull(body); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Phone); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, DiiType.Phone); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), DiiType.Phone); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenGenerated()); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenOptedOut()); - assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentity(IdentityType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); + assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, tokenRefreshRequest, clientSiteId, + getRawUidFromRawDii(DiiType.Phone, phone, firstLevelSalt, rotatingSalt123.currentSalt()), + PrivacyBits.DEFAULT, + body, + TokenUtils.getFirstLevelHashFromRawDii(phone, firstLevelSalt)); - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Phone); - assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(phone, firstLevelSalt), refreshTokenInput.firstLevelHashIdentity.firstLevelHash); - - assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("identity_expires")), 10); - assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_expires")), 10); - assertEqualsClose(now.plusMillis(refreshIdentityAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_from")), 10); testContext.completeNow(); }); } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenGenerateThenRefreshForPhone(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String phone = "+15555555555"; @@ -2568,33 +2822,72 @@ void tokenGenerateThenRefreshForPhone(String apiVersion, Vertx vertx, VertxTestC JsonObject bodyJson = genRespJson.getJsonObject("body"); assertNotNull(bodyJson); + EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); + AdvertisingTokenRequest firstAdvertisingTokenRequest = validateAndGetToken(encoder, bodyJson, + DiiType.Phone); String genRefreshToken = bodyJson.getString("refresh_token"); + TokenRefreshRequest firstTokenRefreshRequest = decodeRefreshToken(encoder, bodyJson.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), DiiType.Phone); when(this.optOutStore.getLatestEntry(any())).thenReturn(null); + byte[] expectedRawUidIdentity = getRawUidFromRawDii(DiiType.Phone, phone, firstLevelSalt, rotatingSalt123.currentSalt()); + byte[] expectedFirstLevelHashIdentity = TokenUtils.getFirstLevelHashFromRawDii(phone, firstLevelSalt); + + assertAdvertisingTokenRefreshTokenRequests(firstAdvertisingTokenRequest, firstTokenRefreshRequest, clientSiteId, + expectedRawUidIdentity, + PrivacyBits.DEFAULT, + bodyJson, + expectedFirstLevelHashIdentity); + sendTokenRefresh(apiVersion, vertx, testContext, genRefreshToken, bodyJson.getString("refresh_response_key"), 200, refreshRespJson -> { assertEquals("success", refreshRespJson.getString("status")); JsonObject refreshBody = refreshRespJson.getJsonObject("body"); assertNotNull(refreshBody); - EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - - AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, refreshBody, IdentityType.Phone); - - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenGenerated()); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenOptedOut()); - assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentity(IdentityType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, refreshBody, DiiType.Phone); String refreshTokenStringNew = refreshBody.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"); assertNotEquals(genRefreshToken, refreshTokenStringNew); - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, refreshTokenStringNew, IdentityType.Phone); - assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(phone, firstLevelSalt), refreshTokenInput.firstLevelHashIdentity.firstLevelHash); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, refreshTokenStringNew, DiiType.Phone); + + // assert if the ad/refresh tokens from original token/generate is same as the ad/refresh tokens from token/refresh + assertAdvertisingTokenRefreshTokenRequests( + advertisingTokenRequest, + firstTokenRefreshRequest, + clientSiteId, + expectedRawUidIdentity, + PrivacyBits.DEFAULT, + bodyJson, + expectedFirstLevelHashIdentity); + assertAdvertisingTokenRefreshTokenRequests( + firstAdvertisingTokenRequest, + tokenRefreshRequest, + clientSiteId, + expectedRawUidIdentity, + PrivacyBits.DEFAULT, + bodyJson, + expectedFirstLevelHashIdentity); + assertAdvertisingTokenRefreshTokenRequests( + advertisingTokenRequest, + tokenRefreshRequest, + clientSiteId, + expectedRawUidIdentity, + PrivacyBits.DEFAULT, + bodyJson, + expectedFirstLevelHashIdentity); - assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("identity_expires")), 10); - assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("refresh_expires")), 10); - assertEqualsClose(now.plusMillis(refreshIdentityAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("refresh_from")), 10); + assertTokenStatusMetrics( + clientSiteId, + TokenResponseStatsCollector.Endpoint.GenerateV2, + TokenResponseStatsCollector.ResponseStatus.Success, + //didn't set any specific header + TokenResponseStatsCollector.PlatformType.Other); + assertTokenStatusMetrics( + clientSiteId, + TokenResponseStatsCollector.Endpoint.RefreshV2, + TokenResponseStatsCollector.ResponseStatus.Success, + //didn't set any specific header + TokenResponseStatsCollector.PlatformType.Other); testContext.completeNow(); }); @@ -2602,7 +2895,7 @@ void tokenGenerateThenRefreshForPhone(String apiVersion, Vertx vertx, VertxTestC } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenGenerateThenValidateWithPhone_Match(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String phone = ValidateIdentityForPhone; @@ -2632,7 +2925,7 @@ void tokenGenerateThenValidateWithPhone_Match(String apiVersion, Vertx vertx, Ve } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenGenerateThenValidateSaltsExpired(String apiVersion, Vertx vertx, VertxTestContext testContext) { when(saltProviderSnapshot.getExpires()).thenReturn(Instant.now().minus(1, ChronoUnit.HOURS)); final int clientSiteId = 201; @@ -2665,7 +2958,7 @@ void tokenGenerateThenValidateSaltsExpired(String apiVersion, Vertx vertx, Vertx } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenGenerateThenValidateWithPhoneHash_Match(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String phoneHash = EncodingUtils.toBase64String(ValidateIdentityForPhoneHash); @@ -2695,7 +2988,7 @@ void tokenGenerateThenValidateWithPhoneHash_Match(String apiVersion, Vertx vertx } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void tokenGenerateThenValidateWithBothPhoneAndPhoneHash(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String phone = ValidateIdentityForPhone; @@ -2726,142 +3019,9 @@ void tokenGenerateThenValidateWithBothPhoneAndPhoneHash(String apiVersion, Vertx }); } - @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) - void tokenRefreshOptOutForPhone(String apiVersion, Vertx vertx, VertxTestContext testContext) { - final int clientSiteId = 201; - final String phone = "+15555555555"; - generateRefreshToken(apiVersion, vertx, "phone", phone, clientSiteId, genRespJson -> { - JsonObject bodyJson = genRespJson.getJsonObject("body"); - String refreshToken = bodyJson.getString("refresh_token"); - - when(this.optOutStore.getLatestEntry(any())).thenReturn(Instant.now()); - - get(vertx, "v1/token/refresh?refresh_token=" + urlEncode(refreshToken), testContext.succeeding(response -> testContext.verify(() -> { - assertEquals(200, response.statusCode()); - JsonObject json = response.bodyAsJsonObject(); - assertEquals("optout", json.getString("status")); - assertTokenStatusMetrics(clientSiteId, TokenResponseStatsCollector.Endpoint.RefreshV1, TokenResponseStatsCollector.ResponseStatus.OptOut, TokenResponseStatsCollector.PlatformType.Other); - - testContext.completeNow(); - }))); - }); - } - - @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) - void tokenRefreshOptOutBeforeLoginForPhone(String apiVersion, Vertx vertx, VertxTestContext testContext) { - final int clientSiteId = 201; - final String phone = "+15555555555"; - generateRefreshToken(apiVersion, vertx, "phone", phone, clientSiteId, genRespJson -> { - JsonObject bodyJson = genRespJson.getJsonObject("body"); - String refreshToken = bodyJson.getString("refresh_token"); - - when(this.optOutStore.getLatestEntry(any())).thenReturn(now.minusSeconds(10)); - - get(vertx, "v1/token/refresh?refresh_token=" + urlEncode(refreshToken), ar -> { - assertTrue(ar.succeeded()); - HttpResponse response = ar.result(); - assertEquals(200, response.statusCode()); - JsonObject json = response.bodyAsJsonObject(); - assertEquals("optout", json.getString("status")); - assertNull(json.getJsonObject("body")); - - testContext.completeNow(); - }); - }); - } - - @Test - void identityMapBothPhoneAndHashSpecified(Vertx vertx, VertxTestContext testContext) { - final int clientSiteId = 201; - final String phone = "+15555555555"; - final String phoneHash = TokenUtils.getIdentityHashString(phone); - fakeAuth(clientSiteId, Role.MAPPER); - setupSalts(); - setupKeys(); - get(vertx, "v1/identity/map?phone=" + urlEncode(phone) + "&phone_hash=" + urlEncode(phoneHash), ar -> { - assertTrue(ar.succeeded()); - HttpResponse response = ar.result(); - assertEquals(400, response.statusCode()); - JsonObject json = response.bodyAsJsonObject(); - assertFalse(json.containsKey("body")); - assertEquals("client_error", json.getString("status")); - - testContext.completeNow(); - }); - } - - @Test - void identityMapForPhone(Vertx vertx, VertxTestContext testContext) { - final int clientSiteId = 201; - final String phone = "+15555555555"; - fakeAuth(clientSiteId, Role.MAPPER); - setupSalts(); - setupKeys(); - get(vertx, "v1/identity/map?phone=" + urlEncode(phone), ar -> { - assertTrue(ar.succeeded()); - HttpResponse response = ar.result(); - assertEquals(200, response.statusCode()); - JsonObject json = response.bodyAsJsonObject(); - assertEquals("success", json.getString("status")); - JsonObject body = json.getJsonObject("body"); - assertNotNull(body); - - assertEquals(phone, body.getString("identifier")); - assertFalse(body.getString("advertising_id").isEmpty()); - assertFalse(body.getString("bucket_id").isEmpty()); - - testContext.completeNow(); - }); - } - - @Test - void identityMapForPhoneHash(Vertx vertx, VertxTestContext testContext) { - final int clientSiteId = 201; - final String phone = "+15555555555"; - final String phonneHash = TokenUtils.getIdentityHashString(phone); - fakeAuth(clientSiteId, Role.MAPPER); - setupSalts(); - setupKeys(); - get(vertx, "v1/identity/map?phone_hash=" + urlEncode(phonneHash), ar -> { - assertTrue(ar.succeeded()); - HttpResponse response = ar.result(); - assertEquals(200, response.statusCode()); - JsonObject json = response.bodyAsJsonObject(); - assertEquals("success", json.getString("status")); - JsonObject body = json.getJsonObject("body"); - assertNotNull(body); - - assertEquals(phonneHash, body.getString("identifier")); - assertFalse(body.getString("advertising_id").isEmpty()); - assertFalse(body.getString("bucket_id").isEmpty()); - - testContext.completeNow(); - }); - } - - @Test - void sendInformationToStatsCollector(Vertx vertx, VertxTestContext testContext) { - final int clientSiteId = 201; - final String emailAddress = "test@uid2.com"; - fakeAuth(clientSiteId, Role.GENERATOR); - setupSalts(); - setupKeys(); - - vertx.eventBus().consumer(Const.Config.StatsCollectorEventBus, message -> { - String expected = "{\"path\":\"/v1/token/generate\",\"referer\":null,\"apiContact\":null,\"siteId\":201}"; - assertSame(message.body().toString(), expected); - }); - - get(vertx, "v1/token/generate?email=" + emailAddress, ar -> { - verify(statsCollectorQueue, times(1)).enqueue(any(), any()); - testContext.completeNow(); - }); - } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void identityMapBatchBothPhoneAndHashEmpty(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.MAPPER); @@ -2881,7 +3041,7 @@ void identityMapBatchBothPhoneAndHashEmpty(String apiVersion, Vertx vertx, Vertx } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void identityMapBatchBothPhoneAndHashSpecified(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.MAPPER); @@ -2895,7 +3055,7 @@ void identityMapBatchBothPhoneAndHashSpecified(String apiVersion, Vertx vertx, V req.put("phone_hash", phoneHashes); phones.add("+15555555555"); - phoneHashes.add(TokenUtils.getIdentityHashString("+15555555555")); + phoneHashes.add(TokenUtils.getHashedDiiString("+15555555555")); send(apiVersion, vertx, apiVersion + "/identity/map", false, null, req, 400, respJson -> { assertFalse(respJson.containsKey("body")); @@ -2905,7 +3065,7 @@ void identityMapBatchBothPhoneAndHashSpecified(String apiVersion, Vertx vertx, V } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void identityMapBatchPhones(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.MAPPER); @@ -2926,7 +3086,7 @@ void identityMapBatchPhones(String apiVersion, Vertx vertx, VertxTestContext tes } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void identityMapBatchPhoneHashes(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.MAPPER); @@ -2937,8 +3097,8 @@ void identityMapBatchPhoneHashes(String apiVersion, Vertx vertx, VertxTestContex JsonArray hashes = new JsonArray(); req.put("phone_hash", hashes); final String[] email_hashes = { - TokenUtils.getIdentityHashString("+15555555555"), - TokenUtils.getIdentityHashString("+15555555556"), + TokenUtils.getHashedDiiString("+15555555555"), + TokenUtils.getHashedDiiString("+15555555556"), }; for (String email_hash : email_hashes) { @@ -2952,7 +3112,7 @@ void identityMapBatchPhoneHashes(String apiVersion, Vertx vertx, VertxTestContex } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void identityMapBatchPhonesOnePhoneInvalid(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.MAPPER); @@ -2974,7 +3134,7 @@ void identityMapBatchPhonesOnePhoneInvalid(String apiVersion, Vertx vertx, Vertx } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void identityMapBatchPhonesNoPhones(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.MAPPER); @@ -2992,7 +3152,7 @@ void identityMapBatchPhonesNoPhones(String apiVersion, Vertx vertx, VertxTestCon } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void identityMapBatchRequestTooLargeForPhone(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.MAPPER); @@ -3019,7 +3179,7 @@ void tokenGenerateRespectOptOutOption(String policyParameterKey, Vertx vertx, Ve setupKeys(); // the clock value shouldn't matter here - when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) + when(optOutStore.getLatestEntry(any(FirstLevelHash.class))) .thenReturn(now.minus(1, ChronoUnit.HOURS)); JsonObject req = new JsonObject(); @@ -3042,7 +3202,7 @@ void tokenGenerateRespectOptOutOption(String policyParameterKey, Vertx vertx, Ve } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void identityMapDefaultOption(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; fakeAuth(clientSiteId, Role.MAPPER); @@ -3050,7 +3210,7 @@ void identityMapDefaultOption(String apiVersion, Vertx vertx, VertxTestContext t setupKeys(); // the clock value shouldn't matter here - when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) + when(optOutStore.getLatestEntry(any(FirstLevelHash.class))) .thenReturn(now.minus(1, ChronoUnit.HOURS)); JsonObject req = new JsonObject(); @@ -3075,8 +3235,6 @@ void identityMapDefaultOption(String apiVersion, Vertx vertx, VertxTestContext t private static Stream versionAndPolicy() { return Stream.of( - Arguments.arguments("v1", "policy"), - Arguments.arguments("v1", "optout_check"), Arguments.arguments("v2", "policy"), Arguments.arguments("v2", "optout_check") ); @@ -3091,7 +3249,7 @@ void identityMapRespectOptOutOption(String apiVersion, String policyParameterKey setupKeys(); // the clock value shouldn't matter here - when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) + when(optOutStore.getLatestEntry(any(FirstLevelHash.class))) .thenReturn(now.minus(1, ChronoUnit.HOURS)); JsonObject req = new JsonObject(); @@ -3114,7 +3272,7 @@ void identityMapRespectOptOutOption(String apiVersion, String policyParameterKey } @ParameterizedTest - @ValueSource(strings = {"v1", "v2"}) + @ValueSource(strings = {"v2"}) void requestWithoutClientKeyOrReferer(String apiVersion, Vertx vertx, VertxTestContext testContext) { final String emailAddress = "test@uid2.com"; setupSalts(); @@ -3164,7 +3322,6 @@ private void postCstg(Vertx vertx, String endpoint, String httpOriginHeader, Jso } req.sendJsonObject(body, handler); } - private void sendCstg(Vertx vertx, String endpoint, String httpOriginHeader, JsonObject postPayload, SecretKey secretKey, int expectedHttpCode, VertxTestContext testContext, Handler handler) { postCstg(vertx, endpoint, httpOriginHeader, postPayload, testContext.succeeding(result -> testContext.verify(() -> { assertEquals(expectedHttpCode, result.statusCode()); @@ -3195,9 +3352,9 @@ private void setupCstgBackend(List domainNames, List appNames) when(siteProvider.getSite(clientSideTokenGenerateSiteId)).thenReturn(site); } - //if no identity is provided will get an error + //if no hashed dii is provided will get an error @Test - void cstgNoIdentityHashProvided(Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { + void cstgNoHashedDiiProvided(Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { setupCstgBackend("cstg.co.uk"); Tuple.Tuple2 data = createClientSideTokenGenerateRequestWithNoPayload(Instant.now().toEpochMilli()); sendCstg(vertx, @@ -3227,7 +3384,7 @@ void cstgNoIdentityHashProvided(Vertx vertx, VertxTestContext testContext) throw }) void cstgDomainNameCheckFails(String httpOrigin, Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { setupCstgBackend(); - Tuple.Tuple2 data = createClientSideTokenGenerateRequest(IdentityType.Email, "random@unifiedid.com", Instant.now().toEpochMilli()); + Tuple.Tuple2 data = createClientSideTokenGenerateRequest(DiiType.Email, "random@unifiedid.com", Instant.now().toEpochMilli()); sendCstg(vertx, "v2/token/client-generate", httpOrigin, @@ -3256,7 +3413,7 @@ void cstgDomainNameCheckFails(String httpOrigin, Vertx vertx, VertxTestContext t }) void cstgAppNameCheckFails(String appName, Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { setupCstgBackend(Collections.emptyList(), List.of("com.123.Game.App.android")); - Tuple.Tuple2 data = createClientSideTokenGenerateRequest(IdentityType.Email, "random@unifiedid.com", Instant.now().toEpochMilli(), appName); + Tuple.Tuple2 data = createClientSideTokenGenerateRequest(DiiType.Email, "random@unifiedid.com", Instant.now().toEpochMilli(), appName); sendCstg(vertx, "v2/token/client-generate", null, @@ -3291,7 +3448,7 @@ void cstgDomainNameCheckFailsAndLogInvalidHttpOrigin(String httpOrigin, Vertx ve this.uidOperatorVerticle.setLastInvalidOriginProcessTime(Instant.now().minusSeconds(3600)); setupCstgBackend(); - Tuple.Tuple2 data = createClientSideTokenGenerateRequest(IdentityType.Email, "random@unifiedid.com", Instant.now().toEpochMilli()); + Tuple.Tuple2 data = createClientSideTokenGenerateRequest(DiiType.Email, "random@unifiedid.com", Instant.now().toEpochMilli()); sendCstg(vertx, "v2/token/client-generate", httpOrigin, @@ -3322,7 +3479,7 @@ void cstgLogsInvalidAppName(String appName, Vertx vertx, VertxTestContext testCo this.uidOperatorVerticle.setLastInvalidOriginProcessTime(Instant.now().minusSeconds(3600)); setupCstgBackend(); - Tuple.Tuple2 data = createClientSideTokenGenerateRequest(IdentityType.Email, "random@unifiedid.com", Instant.now().toEpochMilli(), appName); + Tuple.Tuple2 data = createClientSideTokenGenerateRequest(DiiType.Email, "random@unifiedid.com", Instant.now().toEpochMilli(), appName); sendCstg(vertx, "v2/token/client-generate", null, @@ -3368,7 +3525,7 @@ void cstgDisabledAsUnauthorized(Vertx vertx, VertxTestContext testContext) throw requestJson.put("timestamp", timestamp); requestJson.put("subscription_id", subscriptionID); - Tuple.Tuple2 data = createClientSideTokenGenerateRequest(IdentityType.Email, "random@unifiedid.com", Instant.now().toEpochMilli(), null); + Tuple.Tuple2 data = createClientSideTokenGenerateRequest(DiiType.Email, "random@unifiedid.com", Instant.now().toEpochMilli(), null); sendCstg(vertx, "v2/token/client-generate", null, @@ -3406,7 +3563,7 @@ void cstgDomainNameCheckFailsAndLogSeveralInvalidHttpOrigin(String httpOrigin, V setupCstgBackend(); when(siteProvider.getSite(124)).thenReturn(new Site(124, "test2", true, new HashSet<>())); - Tuple.Tuple2 data = createClientSideTokenGenerateRequest(IdentityType.Email, "random@unifiedid.com", Instant.now().toEpochMilli()); + Tuple.Tuple2 data = createClientSideTokenGenerateRequest(DiiType.Email, "random@unifiedid.com", Instant.now().toEpochMilli()); sendCstg(vertx, "v2/token/client-generate", httpOrigin, @@ -3436,7 +3593,7 @@ void cstgDomainNameCheckFailsAndLogSeveralInvalidHttpOrigin(String httpOrigin, V }) void cstgDomainNameCheckPasses(String httpOrigin, Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { setupCstgBackend("cstg.co.uk", "cstg2.com", "localhost"); - Tuple.Tuple2 data = createClientSideTokenGenerateRequest(IdentityType.Email, "random@unifiedid.com", Instant.now().toEpochMilli()); + Tuple.Tuple2 data = createClientSideTokenGenerateRequest(DiiType.Email, "random@unifiedid.com", Instant.now().toEpochMilli()); sendCstg(vertx, "v2/token/client-generate", httpOrigin, @@ -3450,7 +3607,7 @@ void cstgDomainNameCheckPasses(String httpOrigin, Vertx vertx, VertxTestContext JsonObject refreshBody = respJson.getJsonObject("body"); assertNotNull(refreshBody); var encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - validateAndGetToken(encoder, refreshBody, IdentityType.Email); //to validate token version is correct + validateAndGetToken(encoder, refreshBody, DiiType.Email); //to validate token version is correct testContext.completeNow(); }); } @@ -3463,7 +3620,7 @@ void cstgDomainNameCheckPasses(String httpOrigin, Vertx vertx, VertxTestContext }) void cstgAppNameCheckPasses(String appName, Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { setupCstgBackend(Collections.emptyList(), List.of("com.123.Game.App.android", "123456789")); - Tuple.Tuple2 data = createClientSideTokenGenerateRequest(IdentityType.Email, "random@unifiedid.com", Instant.now().toEpochMilli(), appName); + Tuple.Tuple2 data = createClientSideTokenGenerateRequest(DiiType.Email, "random@unifiedid.com", Instant.now().toEpochMilli(), appName); sendCstg(vertx, "v2/token/client-generate", null, @@ -3477,7 +3634,7 @@ void cstgAppNameCheckPasses(String appName, Vertx vertx, VertxTestContext testCo JsonObject refreshBody = respJson.getJsonObject("body"); assertNotNull(refreshBody); var encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - validateAndGetToken(encoder, refreshBody, IdentityType.Email); //to validate token version is correct + validateAndGetToken(encoder, refreshBody, DiiType.Email); //to validate token version is correct assertTokenStatusMetrics( clientSideTokenGenerateSiteId, TokenResponseStatsCollector.Endpoint.ClientSideTokenGenerateV2, @@ -3492,15 +3649,15 @@ void cstgNoBody(Vertx vertx, VertxTestContext testContext) { setupCstgBackend("cstg.co.uk"); postCstg(vertx, - "v2/token/client-generate", - "https://cstg.co.uk", - null, - testContext.succeeding(result -> testContext.verify(() -> { - JsonObject response = result.bodyAsJsonObject(); - assertEquals("client_error", response.getString("status")); - assertEquals("json payload expected but not found", response.getString("message")); - testContext.completeNow(); - }))); + "v2/token/client-generate", + "https://cstg.co.uk", + null, + testContext.succeeding(result -> testContext.verify(() -> { + JsonObject response = result.bodyAsJsonObject(); + assertEquals("client_error", response.getString("status")); + assertEquals("json payload expected but not found", response.getString("message")); + testContext.completeNow(); + }))); } @Test @@ -3509,12 +3666,12 @@ void cstgForInvalidJsonPayloadReturns400(Vertx vertx, VertxTestContext testConte WebClient client = WebClient.create(vertx); client.postAbs(getUrlForEndpoint("v2/token/client-generate")) - .putHeader(ORIGIN_HEADER, "https://cstg.co.uk") - .putHeader("Content-Type", "application/json") - .sendBuffer(Buffer.buffer("not a valid json payload"), result -> testContext.verify(() -> { - assertEquals(400, result.result().statusCode()); - testContext.completeNow(); - })); + .putHeader(ORIGIN_HEADER, "https://cstg.co.uk") + .putHeader(HttpHeaders.CONTENT_TYPE.toString(), HttpMediaType.APPLICATION_JSON.getType()) + .sendBuffer(Buffer.buffer("not a valid json payload"), result -> testContext.verify(() -> { + assertEquals(400, result.result().statusCode()); + testContext.completeNow(); + })); } @ParameterizedTest @@ -3561,8 +3718,13 @@ void cstgMissingRequiredField(String testField, Vertx vertx, VertxTestContext te }); } - @Test - void cstgBadPublicKey(Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { + @ParameterizedTest + @ValueSource(strings = {"bad-key", clientKey}) + void cstgBadPublicKey(String publicKey, Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { + ListAppender logWatcher = new ListAppender<>(); + logWatcher.start(); + ((Logger) LoggerFactory.getLogger(UIDOperatorVerticle.class)).addAppender(logWatcher); + setupCstgBackend("cstg.co.uk"); String rawId = "random@unifiedid.com"; @@ -3584,7 +3746,7 @@ void cstgBadPublicKey(Vertx vertx, VertxTestContext testContext) throws NoSuchAl JsonObject requestJson = new JsonObject(); requestJson.put("payload", payload); requestJson.put("iv", EncodingUtils.toBase64String(iv)); - requestJson.put("public_key", "bad-key"); + requestJson.put("public_key", publicKey); requestJson.put("timestamp", timestamp); requestJson.put("subscription_id", clientSideTokenGenerateSubscriptionId); @@ -3596,6 +3758,9 @@ void cstgBadPublicKey(Vertx vertx, VertxTestContext testContext) throws NoSuchAl 400, testContext, respJson -> { + if (publicKey.equals(clientKey)) { // if client api key is passed in to cstg, we should log + Assertions.assertTrue(logWatcher.list.stream().anyMatch(l -> l.getFormattedMessage().contains("Client side key is an api key with api_key_id=key-id for site_id=1"))); + } assertEquals("client_error", respJson.getString("status")); assertEquals("bad public key", respJson.getString("message")); assertTokenStatusMetrics( @@ -3607,8 +3772,13 @@ void cstgBadPublicKey(Vertx vertx, VertxTestContext testContext) throws NoSuchAl }); } - @Test - void cstgBadSubscriptionId(Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { + @ParameterizedTest + @ValueSource(strings = {"bad", clientKey}) + void cstgBadSubscriptionId(String subscriptionId, Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { + ListAppender logWatcher = new ListAppender<>(); + logWatcher.start(); + ((Logger) LoggerFactory.getLogger(UIDOperatorVerticle.class)).addAppender(logWatcher); + setupCstgBackend("cstg.co.uk"); String rawId = "random@unifiedid.com"; @@ -3632,7 +3802,7 @@ void cstgBadSubscriptionId(Vertx vertx, VertxTestContext testContext) throws NoS requestJson.put("iv", EncodingUtils.toBase64String(iv)); requestJson.put("public_key", "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE92+xlW2eIrXsDzV4cSfldDKxLXHsMmjLIqpdwOqJ29pWTNnZMaY2ycZHFpxbp6UlQ6vVSpKwImTKr3uikm9yCw=="); requestJson.put("timestamp", timestamp); - requestJson.put("subscription_id", "bad"); + requestJson.put("subscription_id", subscriptionId); sendCstg(vertx, "v2/token/client-generate", @@ -3642,6 +3812,9 @@ void cstgBadSubscriptionId(Vertx vertx, VertxTestContext testContext) throws NoS 400, testContext, respJson -> { + if (subscriptionId.equals(clientKey)) { // if client api key is passed in to cstg, we should log + Assertions.assertTrue(logWatcher.list.stream().anyMatch(l -> l.getFormattedMessage().contains("Client side key is an api key with api_key_id=key-id for site_id=1"))); + } assertEquals("client_error", respJson.getString("status")); assertEquals("bad subscription_id", respJson.getString("message")); testContext.completeNow(); @@ -3944,24 +4117,24 @@ private Tuple.Tuple2 createClientSideTokenGenerateRequest return new Tuple.Tuple2<>(requestJson, secretKey); } - private Tuple.Tuple2 createClientSideTokenGenerateRequest(IdentityType identityType, String rawId, long timestamp) throws NoSuchAlgorithmException, InvalidKeyException { - return createClientSideTokenGenerateRequest(identityType, rawId, timestamp, null); + private Tuple.Tuple2 createClientSideTokenGenerateRequest(DiiType diiType, String rawId, long timestamp) throws NoSuchAlgorithmException, InvalidKeyException { + return createClientSideTokenGenerateRequest(diiType, rawId, timestamp, null); } - private Tuple.Tuple2 createClientSideTokenGenerateRequest(IdentityType identityType, String rawId, long timestamp, String appName) throws NoSuchAlgorithmException, InvalidKeyException { + private Tuple.Tuple2 createClientSideTokenGenerateRequest(DiiType diiType, String rawId, long timestamp, String appName) throws NoSuchAlgorithmException, InvalidKeyException { JsonObject identity = new JsonObject(); - if(identityType == IdentityType.Email) { + if(diiType == DiiType.Email) { identity.put("email_hash", getSha256(rawId)); } - else if(identityType == IdentityType.Phone) { + else if(diiType == DiiType.Phone) { identity.put("phone_hash", getSha256(rawId)); } else { //can't be other types assertFalse(true); } - + return createClientSideTokenGenerateRequestWithPayload(identity, timestamp, appName); } @@ -3976,17 +4149,17 @@ private Tuple.Tuple2 createClientSideTokenGenerateRequest "test@example.com,Email", "+61400000000,Phone" }) - void cstgUserOptsOutAfterTokenGenerate(String id, IdentityType identityType, Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { + void cstgUserOptsOutAfterTokenGenerate(String id, DiiType diiType, Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { setupCstgBackend("cstg.co.uk"); - final Tuple.Tuple2 data = createClientSideTokenGenerateRequest(identityType, id, Instant.now().toEpochMilli()); + final Tuple.Tuple2 data = createClientSideTokenGenerateRequest(diiType, id, Instant.now().toEpochMilli()); // When we generate the token the user hasn't opted out. - when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) + when(optOutStore.getLatestEntry(any(FirstLevelHash.class))) .thenReturn(null); final EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - final ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(FirstLevelHashIdentity.class); + final ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(FirstLevelHash.class); sendCstg(vertx, "v2/token/client-generate", @@ -3997,20 +4170,20 @@ void cstgUserOptsOutAfterTokenGenerate(String id, IdentityType identityType, Ver testContext, response -> { verify(optOutStore, times(1)).getLatestEntry(argumentCaptor.capture()); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(id, firstLevelSalt), - argumentCaptor.getValue().firstLevelHash); + assertArrayEquals(TokenUtils.getFirstLevelHashFromRawDii(id, firstLevelSalt), + argumentCaptor.getValue().firstLevelHash()); assertEquals("success", response.getString("status")); final JsonObject genBody = response.getJsonObject("body"); - final AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, genBody, identityType); - final RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, decodeV2RefreshToken(response), identityType); + final AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, genBody, diiType); + final TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, decodeV2RefreshToken(response), diiType); - assertAreClientSideGeneratedTokens(advertisingTokenInput, refreshTokenInput, clientSideTokenGenerateSiteId, identityType, id); + assertAreClientSideGeneratedTokens(advertisingTokenRequest, tokenRefreshRequest, clientSideTokenGenerateSiteId, diiType, id); // When we refresh the token the user has opted out. - when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) - .thenReturn(advertisingTokenInput.rawUidIdentity.establishedAt.plusSeconds(1)); + when(optOutStore.getLatestEntry(any(FirstLevelHash.class))) + .thenReturn(advertisingTokenRequest.establishedAt.plusSeconds(1)); sendTokenRefresh("v2", vertx, testContext, genBody.getString("refresh_token"), genBody.getString("refresh_response_key"), 200, refreshRespJson -> { assertEquals("optout", refreshRespJson.getString("status")); @@ -4030,19 +4203,19 @@ void cstgUserOptsOutAfterTokenGenerate(String id, IdentityType identityType, Ver "false,abc@abc.com,Email", "false,+61400000000,Phone", }) - void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id, IdentityType identityType, - Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { + void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id, DiiType diiType, + Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { setupCstgBackend("cstg.co.uk"); - Tuple.Tuple2 data = createClientSideTokenGenerateRequest(identityType, id, Instant.now().toEpochMilli()); + Tuple.Tuple2 data = createClientSideTokenGenerateRequest(diiType, id, Instant.now().toEpochMilli()); if(optOutExpected) { - when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) + when(optOutStore.getLatestEntry(any(FirstLevelHash.class))) .thenReturn(Instant.now().minus(1, ChronoUnit.HOURS)); } else { //not expectedOptedOut - when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) + when(optOutStore.getLatestEntry(any(FirstLevelHash.class))) .thenReturn(null); } @@ -4067,11 +4240,26 @@ void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id decodeV2RefreshToken(respJson); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, genBody, identityType); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, genBody, diiType); + + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, genBody.getString("decrypted_refresh_token"), diiType); + - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, genBody.getString("decrypted_refresh_token"), identityType); - assertAreClientSideGeneratedTokens(advertisingTokenInput, refreshTokenInput, clientSideTokenGenerateSiteId, identityType, id); + byte[] expectedRawUidIdentity = getRawUidFromRawDii(diiType, id, firstLevelSalt, rotatingSalt123.currentSalt()); + byte[] expectedFirstLevelHashIdentity = TokenUtils.getFirstLevelHashFromRawDii(id, firstLevelSalt); + + PrivacyBits expectedPrivacyBits = new PrivacyBits(); + expectedPrivacyBits.setLegacyBit(); + expectedPrivacyBits.setClientSideTokenGenerate(); + + assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, tokenRefreshRequest, + clientSideTokenGenerateSiteId, + expectedRawUidIdentity, + expectedPrivacyBits, + genBody, + expectedFirstLevelHashIdentity); + assertAreClientSideGeneratedTokens(advertisingTokenRequest, tokenRefreshRequest, clientSideTokenGenerateSiteId, diiType, id); assertEqualsClose(Instant.now().plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(genBody.getLong("identity_expires")), 10); assertEqualsClose(Instant.now().plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(genBody.getLong("refresh_expires")), 10); assertEqualsClose(Instant.now().plusMillis(refreshIdentityAfter.toMillis()), Instant.ofEpochMilli(genBody.getLong("refresh_from")), 10); @@ -4092,13 +4280,19 @@ void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id EncryptedTokenEncoder encoder2 = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); //make sure the new advertising token from refresh looks right - AdvertisingTokenInput adTokenFromRefresh = validateAndGetToken(encoder2, refreshBody, identityType); + AdvertisingTokenRequest adTokenFromRefresh = validateAndGetToken(encoder2, refreshBody, diiType); String refreshTokenStringNew = refreshBody.getString("decrypted_refresh_token"); assertNotEquals(genRefreshToken, refreshTokenStringNew); - RefreshTokenInput refreshTokenAfterRefreshSource = decodeRefreshToken(encoder, refreshTokenStringNew, identityType); + TokenRefreshRequest refreshTokenAfterRefreshSource = decodeRefreshToken(encoder, refreshTokenStringNew, diiType); - assertAreClientSideGeneratedTokens(adTokenFromRefresh, refreshTokenAfterRefreshSource, clientSideTokenGenerateSiteId, identityType, id); + assertAdvertisingTokenRefreshTokenRequests(adTokenFromRefresh, refreshTokenAfterRefreshSource, + clientSideTokenGenerateSiteId, + expectedRawUidIdentity, + expectedPrivacyBits, + genBody, + expectedFirstLevelHashIdentity); + assertAreClientSideGeneratedTokens(adTokenFromRefresh, refreshTokenAfterRefreshSource, clientSideTokenGenerateSiteId, diiType, id); assertEqualsClose(Instant.now().plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("identity_expires")), 10); assertEqualsClose(Instant.now().plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("refresh_expires")), 10); assertEqualsClose(Instant.now().plusMillis(refreshIdentityAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("refresh_from")), 10); @@ -4123,7 +4317,7 @@ void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id void cstgSaltsExpired(String httpOrigin, Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { when(saltProviderSnapshot.getExpires()).thenReturn(Instant.now().minus(1, ChronoUnit.HOURS)); setupCstgBackend("cstg.co.uk", "cstg2.com", "localhost"); - Tuple.Tuple2 data = createClientSideTokenGenerateRequest(IdentityType.Email, "random@unifiedid.com", Instant.now().toEpochMilli()); + Tuple.Tuple2 data = createClientSideTokenGenerateRequest(DiiType.Email, "random@unifiedid.com", Instant.now().toEpochMilli()); sendCstg(vertx, "v2/token/client-generate", httpOrigin, @@ -4137,7 +4331,7 @@ void cstgSaltsExpired(String httpOrigin, Vertx vertx, VertxTestContext testConte JsonObject refreshBody = respJson.getJsonObject("body"); assertNotNull(refreshBody); var encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - validateAndGetToken(encoder, refreshBody, IdentityType.Email); //to validate token version is correct + validateAndGetToken(encoder, refreshBody, DiiType.Email); //to validate token version is correct verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(true); @@ -4149,7 +4343,7 @@ void cstgSaltsExpired(String httpOrigin, Vertx vertx, VertxTestContext testConte void cstgNoActiveKey(Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { setupCstgBackend("cstg.co.uk"); setupKeys(true); - Tuple.Tuple2 data = createClientSideTokenGenerateRequest(IdentityType.Email, "random@unifiedid.com", Instant.now().toEpochMilli()); + Tuple.Tuple2 data = createClientSideTokenGenerateRequest(DiiType.Email, "random@unifiedid.com", Instant.now().toEpochMilli()); sendCstg(vertx, "v2/token/client-generate", "http://cstg.co.uk", @@ -4192,25 +4386,25 @@ void cstgInvalidInput(String identityType, String rawUID, Vertx vertx, VertxTest }); } - private void assertAreClientSideGeneratedTokens(AdvertisingTokenInput advertisingTokenInput, RefreshTokenInput refreshTokenInput, int siteId, IdentityType identityType, String identity) { - assertAreClientSideGeneratedTokens(advertisingTokenInput, - refreshTokenInput, + private void assertAreClientSideGeneratedTokens(AdvertisingTokenRequest advertisingTokenRequest, TokenRefreshRequest tokenRefreshRequest, int siteId, DiiType diiType, String identity) { + assertAreClientSideGeneratedTokens(advertisingTokenRequest, + tokenRefreshRequest, siteId, - identityType, + diiType, identity, false); } - private void assertAreClientSideGeneratedTokens(AdvertisingTokenInput advertisingTokenInput, RefreshTokenInput refreshTokenInput, int siteId, IdentityType identityType, String identity, boolean expectedOptOut) { - final PrivacyBits advertisingTokenPrivacyBits = PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits); - final PrivacyBits refreshTokenPrivacyBits = PrivacyBits.fromInt(refreshTokenInput.firstLevelHashIdentity.privacyBits); + private void assertAreClientSideGeneratedTokens(AdvertisingTokenRequest advertisingTokenRequest, TokenRefreshRequest tokenRefreshRequest, int siteId, DiiType diiType, String identity, boolean expectedOptOut) { + final PrivacyBits advertisingTokenPrivacyBits = advertisingTokenRequest.privacyBits; + final PrivacyBits refreshTokenPrivacyBits = tokenRefreshRequest.privacyBits; - final byte[] rawUid = getRawUidFromIdentity(identityType, + final byte[] rawUid = getRawUidFromRawDii(diiType, identity, firstLevelSalt, - rotatingSalt123.getSalt()); + rotatingSalt123.currentSalt()); - final byte[] firstLevelHash = TokenUtils.getFirstLevelHashFromIdentity(identity, firstLevelSalt); + final byte[] firstLevelHash = TokenUtils.getFirstLevelHashFromRawDii(identity, firstLevelSalt); assertAll( () -> assertTrue(advertisingTokenPrivacyBits.isClientSideTokenGenerated(), "Advertising token privacy bits CSTG flag is incorrect"), @@ -4219,11 +4413,11 @@ private void assertAreClientSideGeneratedTokens(AdvertisingTokenInput advertisin () -> assertTrue(refreshTokenPrivacyBits.isClientSideTokenGenerated(), "Refresh token privacy bits CSTG flag is incorrect"), () -> assertEquals(expectedOptOut, refreshTokenPrivacyBits.isClientSideTokenOptedOut(), "Refresh token privacy bits CSTG optout flag is incorrect"), - () -> assertEquals(siteId, advertisingTokenInput.sourcePublisher.siteId, "Advertising token site ID is incorrect"), - () -> assertEquals(siteId, refreshTokenInput.sourcePublisher.siteId, "Refresh token site ID is incorrect"), + () -> assertEquals(siteId, advertisingTokenRequest.sourcePublisher.siteId, "Advertising token site ID is incorrect"), + () -> assertEquals(siteId, tokenRefreshRequest.sourcePublisher.siteId, "Refresh token site ID is incorrect"), - () -> assertArrayEquals(rawUid, advertisingTokenInput.rawUidIdentity.rawUid, "Advertising token ID is incorrect"), - () -> assertArrayEquals(firstLevelHash, refreshTokenInput.firstLevelHashIdentity.firstLevelHash, "Refresh token ID is incorrect") + () -> assertArrayEquals(rawUid, advertisingTokenRequest.rawUid.rawUid(), "Advertising token ID is incorrect"), + () -> assertArrayEquals(firstLevelHash, tokenRefreshRequest.firstLevelHash.firstLevelHash(), "Refresh token ID is incorrect") ); } @@ -4397,7 +4591,7 @@ void getActiveKeyTest() { @ValueSource(strings = {"MultiKeysets", "AddKey", "RotateKey", "DisableActiveKey", "DisableDefaultKeyset"}) void tokenGenerateRotatingKeysets_GENERATOR(String testRun, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 101; - final String emailHash = TokenUtils.getIdentityHashString("test@uid2.com"); + final String emailHash = TokenUtils.getHashedDiiString("test@uid2.com"); fakeAuth(clientSiteId, Role.GENERATOR); MultipleKeysetsTests test = new MultipleKeysetsTests(); //To read these tests, open the MultipleKeysetsTests() constructor in another window so you can see the keyset contents and validate expectations @@ -4453,16 +4647,16 @@ void tokenGenerateRotatingKeysets_GENERATOR(String testRun, Vertx vertx, VertxTe assertNotNull(body); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Email); - assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, DiiType.Email); + assertEquals(clientSiteId, advertisingTokenRequest.sourcePublisher.siteId); //Uses a key from default keyset int clientKeyId; - if (advertisingTokenInput.version == TokenVersion.V3 || advertisingTokenInput.version == TokenVersion.V4) { + if (advertisingTokenRequest.version == TokenVersion.V3 || advertisingTokenRequest.version == TokenVersion.V4) { String advertisingTokenString = body.getString("advertising_token"); byte[] bytes = null; - if (advertisingTokenInput.version == TokenVersion.V3) { + if (advertisingTokenRequest.version == TokenVersion.V3) { bytes = EncodingUtils.fromBase64(advertisingTokenString); - } else if (advertisingTokenInput.version == TokenVersion.V4) { + } else if (advertisingTokenRequest.version == TokenVersion.V4) { bytes = Uid2Base64UrlCoder.decode(advertisingTokenString); //same as V3 but use Base64URL encoding } final Buffer b = Buffer.buffer(bytes); @@ -4472,7 +4666,7 @@ void tokenGenerateRotatingKeysets_GENERATOR(String testRun, Vertx vertx, VertxTe final Buffer masterPayload = Buffer.buffer(masterPayloadBytes); clientKeyId = masterPayload.getInt(29); } else { - clientKeyId = advertisingTokenInput.sourcePublisher.clientKeyId; + clientKeyId = advertisingTokenRequest.sourcePublisher.clientKeyId; } switch (testRun) { case "MultiKeysets": @@ -4496,8 +4690,9 @@ void tokenGenerateRotatingKeysets_GENERATOR(String testRun, Vertx vertx, VertxTe }); } - @Test - void keySharingKeysets_CorrectFiltering(Vertx vertx, VertxTestContext testContext) { + @ParameterizedTest + @ValueSource(strings = {"text/plain", "application/octet-stream"}) + void keySharingKeysets_CorrectFiltering(String contentType, Vertx vertx, VertxTestContext testContext) { //Call should return // all keys they have access in ACL // The master key -1 @@ -4541,7 +4736,7 @@ void keySharingKeysets_CorrectFiltering(Vertx vertx, VertxTestContext testContex System.out.println(respJson); checkEncryptionKeys(respJson, KeyDownloadEndpoint.SHARING, siteId, expectedKeys); testContext.completeNow(); - }); + }, Map.of(HttpHeaders.CONTENT_TYPE.toString(), contentType)); } private static Site defaultMockSite(int siteId, boolean includeDomainNames, boolean includeAppNames) { @@ -4655,20 +4850,20 @@ private static Stream testKeyDownloadEndpointKeysetsData_IDREADER() { Map emptySites = new HashMap<>(); return Stream.of( // Both domains and app names should be present in response - Arguments.of("true", KeyDownloadEndpoint.SHARING, mockSitesWithBoth, expectedSitesWithBoth), - Arguments.of("true", KeyDownloadEndpoint.BIDSTREAM, mockSitesWithBoth, expectedSitesWithBoth), + Arguments.of("true", KeyDownloadEndpoint.SHARING, mockSitesWithBoth, expectedSitesWithBoth), + Arguments.of("true", KeyDownloadEndpoint.BIDSTREAM, mockSitesWithBoth, expectedSitesWithBoth), - // only domains should be present in response - Arguments.of("false", KeyDownloadEndpoint.SHARING, mockSitesWithDomainsOnly, expectedSitesDomainsOnly), - Arguments.of("false", KeyDownloadEndpoint.BIDSTREAM, mockSitesWithDomainsOnly, expectedSitesDomainsOnly), + // only domains should be present in response + Arguments.of("false", KeyDownloadEndpoint.SHARING, mockSitesWithDomainsOnly, expectedSitesDomainsOnly), + Arguments.of("false", KeyDownloadEndpoint.BIDSTREAM, mockSitesWithDomainsOnly, expectedSitesDomainsOnly), - // only app names should be present in response - Arguments.of("true", KeyDownloadEndpoint.SHARING, mockSitesWithAppNamesOnly, expectedSitesWithAppNamesOnly), - Arguments.of("true", KeyDownloadEndpoint.BIDSTREAM, mockSitesWithAppNamesOnly, expectedSitesWithAppNamesOnly), + // only app names should be present in response + Arguments.of("true", KeyDownloadEndpoint.SHARING, mockSitesWithAppNamesOnly, expectedSitesWithAppNamesOnly), + Arguments.of("true", KeyDownloadEndpoint.BIDSTREAM, mockSitesWithAppNamesOnly, expectedSitesWithAppNamesOnly), - // None - Arguments.of("false", KeyDownloadEndpoint.SHARING, emptySites, emptySites), - Arguments.of("false", KeyDownloadEndpoint.BIDSTREAM, emptySites, emptySites) + // None + Arguments.of("false", KeyDownloadEndpoint.SHARING, emptySites, emptySites), + Arguments.of("false", KeyDownloadEndpoint.BIDSTREAM, emptySites, emptySites) ); } @@ -4745,10 +4940,10 @@ void keyDownloadEndpointKeysets_IDREADER(boolean provideAppNames, KeyDownloadEnd @Test void keySharingKeysets_SHARER_CustomMaxSharingLifetimeSeconds(Vertx vertx, VertxTestContext testContext) { - this.uidOperatorVerticle.setMaxSharingLifetimeSeconds(999999); + this.runtimeConfig = this.runtimeConfig.toBuilder().withMaxSharingLifetimeSeconds(999999).build(); keySharingKeysets_SHARER(true, true, vertx, testContext, 999999); } - + @ParameterizedTest @CsvSource({ "true, true", @@ -4766,7 +4961,7 @@ void keySharingKeysets_SHARER_defaultMaxSharingLifetimeSeconds(boolean provideSi // SHARER has no access to a keyset that is disabled - direct reject // SHARER has no access to a keyset with a missing allowed_sites - reject by sharing // SHARER has no access to a keyset with an empty allowed_sites - reject by sharing - // SHARER has no access to a keyset with an allowed_sites for other sites - reject by sharing + // SHARER has no access to a keyset with an allowed_sites for other sites - reject by sharing void keySharingKeysets_SHARER(boolean provideSiteDomainNames, boolean provideAppNames, Vertx vertx, VertxTestContext testContext, int expectedMaxSharingLifetimeSeconds) { if (!provideAppNames) { this.uidOperatorVerticle.setKeySharingEndpointProvideAppNames(false); @@ -5090,4 +5285,239 @@ void secureLinkValidationFailsReturnsIdentityError(Vertx vertx, VertxTestContext testContext.completeNow(); }); } + + @Test + void tokenGenerateRespectsConfigValues(Vertx vertx, VertxTestContext testContext) { + final int clientSiteId = 201; + final String emailAddress = "test@uid2.com"; + fakeAuth(clientSiteId, Role.GENERATOR); + setupSalts(); + setupKeys(); + + JsonObject v2Payload = new JsonObject(); + v2Payload.put("email", emailAddress); + + Duration newIdentityExpiresAfter = Duration.ofMinutes(20); + Duration newRefreshExpiresAfter = Duration.ofMinutes(30); + Duration newRefreshIdentityAfter = Duration.ofMinutes(10); + + this.runtimeConfig = this.runtimeConfig + .toBuilder() + .withIdentityTokenExpiresAfterSeconds((int) newIdentityExpiresAfter.toSeconds()) + .withRefreshTokenExpiresAfterSeconds((int) newRefreshExpiresAfter.toSeconds()) + .withRefreshIdentityTokenAfterSeconds((int) newRefreshIdentityAfter.toSeconds()) + .build(); + + sendTokenGenerate("v2", vertx, + null, v2Payload, 200, + respJson -> { + testContext.verify(() -> { + JsonObject body = respJson.getJsonObject("body"); + assertNotNull(body); + assertEquals(now.plusMillis(newIdentityExpiresAfter.toMillis()).toEpochMilli(), body.getLong("identity_expires")); + assertEquals(now.plusMillis(newRefreshExpiresAfter.toMillis()).toEpochMilli(), body.getLong("refresh_expires")); + assertEquals(now.plusMillis(newRefreshIdentityAfter.toMillis()).toEpochMilli(), body.getLong("refresh_from")); + }); + testContext.completeNow(); + }); + } + + @Test + void keySharingRespectsConfigValues(Vertx vertx, VertxTestContext testContext) { + int newSharingTokenExpiry = config.getInteger(Const.Config.SharingTokenExpiryProp) + 1; + int newMaxSharingLifetimeSeconds = config.getInteger(Const.Config.SharingTokenExpiryProp) + 1; + + this.runtimeConfig = this.runtimeConfig + .toBuilder() + .withSharingTokenExpirySeconds(newSharingTokenExpiry) + .withMaxBidstreamLifetimeSeconds(newMaxSharingLifetimeSeconds) + .build(); + + String apiVersion = "v2"; + int siteId = 5; + fakeAuth(siteId, Role.SHARER); + Keyset[] keysets = { + new Keyset(MasterKeysetId, MasterKeySiteId, "test", null, now.getEpochSecond(), true, true), + new Keyset(10, 5, "siteKeyset", null, now.getEpochSecond(), true, true), + }; + KeysetKey[] encryptionKeys = { + new KeysetKey(101, "master key".getBytes(), now, now, now.plusSeconds(10), MasterKeysetId), + new KeysetKey(102, "site key".getBytes(), now, now, now.plusSeconds(10), 10), + }; + MultipleKeysetsTests test = new MultipleKeysetsTests(Arrays.asList(keysets), Arrays.asList(encryptionKeys)); + setupSiteDomainAndAppNameMock(true, false, 101, 102, 103, 104, 105); + send(apiVersion, vertx, apiVersion + "/key/sharing", true, null, null, 200, respJson -> { + testContext.verify(() -> { + JsonObject body = respJson.getJsonObject("body"); + assertNotNull(body); + assertEquals(newSharingTokenExpiry, Integer.parseInt(body.getString("token_expiry_seconds"))); + assertEquals(newMaxSharingLifetimeSeconds + TOKEN_LIFETIME_TOLERANCE.toSeconds(), body.getLong(Const.Config.MaxSharingLifetimeProp)); + }); + testContext.completeNow(); + }); + } + + @ParameterizedTest + @ValueSource(strings = {"text/plain", "application/octet-stream"}) + void keyBidstreamRespectsConfigValues(String contentType, Vertx vertx, VertxTestContext testContext) { + int newMaxBidstreamLifetimeSeconds = 999999; + + this.runtimeConfig = this.runtimeConfig + .toBuilder() + .withMaxBidstreamLifetimeSeconds(newMaxBidstreamLifetimeSeconds) + .build(); + + final String apiVersion = "v2"; + final KeyDownloadEndpoint endpoint = KeyDownloadEndpoint.BIDSTREAM; + + final int clientSiteId = 101; + fakeAuth(clientSiteId, Role.ID_READER); + + // Required, sets up mock keys. + new MultipleKeysetsTests(); + + send(apiVersion, vertx, apiVersion + endpoint.getPath(), true, null, null, 200, respJson -> { + testContext.verify(() -> { + JsonObject body = respJson.getJsonObject("body"); + assertNotNull(body); + assertEquals(newMaxBidstreamLifetimeSeconds + TOKEN_LIFETIME_TOLERANCE.toSeconds(), body.getLong(Const.Config.MaxBidstreamLifetimeSecondsProp)); + }); + testContext.completeNow(); + }, Map.of(HttpHeaders.CONTENT_TYPE.toString(), contentType)); + } + + @Test + void tokenGenerateRespectsConfigValuesWithRemoteConfig(Vertx vertx, VertxTestContext testContext) { + final int clientSiteId = 201; + final String emailAddress = "test@uid2.com"; + fakeAuth(clientSiteId, Role.GENERATOR); + setupSalts(); + setupKeys(); + + JsonObject v2Payload = new JsonObject(); + v2Payload.put("email", emailAddress); + + Duration newIdentityExpiresAfter = Duration.ofMinutes(20); + Duration newRefreshExpiresAfter = Duration.ofMinutes(30); + Duration newRefreshIdentityAfter = Duration.ofMinutes(10); + + this.runtimeConfig = this.runtimeConfig + .toBuilder() + .withIdentityTokenExpiresAfterSeconds((int) newIdentityExpiresAfter.toSeconds()) + .withRefreshTokenExpiresAfterSeconds((int) newRefreshExpiresAfter.toSeconds()) + .withRefreshIdentityTokenAfterSeconds((int) newRefreshIdentityAfter.toSeconds()) + .build(); + + sendTokenGenerate("v2", vertx, + null, v2Payload, 200, + respJson -> { + testContext.verify(() -> { + JsonObject body = respJson.getJsonObject("body"); + assertNotNull(body); + assertEquals(now.plusMillis(newIdentityExpiresAfter.toMillis()).toEpochMilli(), body.getLong("identity_expires")); + assertEquals(now.plusMillis(newRefreshExpiresAfter.toMillis()).toEpochMilli(), body.getLong("refresh_expires")); + assertEquals(now.plusMillis(newRefreshIdentityAfter.toMillis()).toEpochMilli(), body.getLong("refresh_from")); + }); + testContext.completeNow(); + }); + } + + @Test + void keySharingRespectsConfigValuesWithRemoteConfig(Vertx vertx, VertxTestContext testContext) { + int newSharingTokenExpiry = config.getInteger(Const.Config.SharingTokenExpiryProp) + 1; + int newMaxSharingLifetimeSeconds = config.getInteger(Const.Config.SharingTokenExpiryProp) + 1; + + this.runtimeConfig = this.runtimeConfig + .toBuilder() + .withSharingTokenExpirySeconds(newSharingTokenExpiry) + .withMaxSharingLifetimeSeconds(newMaxSharingLifetimeSeconds) + .build(); + + String apiVersion = "v2"; + int siteId = 5; + fakeAuth(siteId, Role.SHARER); + Keyset[] keysets = { + new Keyset(MasterKeysetId, MasterKeySiteId, "test", null, now.getEpochSecond(), true, true), + new Keyset(10, 5, "siteKeyset", null, now.getEpochSecond(), true, true), + }; + KeysetKey[] encryptionKeys = { + new KeysetKey(101, "master key".getBytes(), now, now, now.plusSeconds(10), MasterKeysetId), + new KeysetKey(102, "site key".getBytes(), now, now, now.plusSeconds(10), 10), + }; + MultipleKeysetsTests test = new MultipleKeysetsTests(Arrays.asList(keysets), Arrays.asList(encryptionKeys)); + setupSiteDomainAndAppNameMock(true, false, 101, 102, 103, 104, 105); + send(apiVersion, vertx, apiVersion + "/key/sharing", true, null, null, 200, respJson -> { + testContext.verify(() -> { + JsonObject body = respJson.getJsonObject("body"); + assertNotNull(body); + assertEquals(newSharingTokenExpiry, Integer.parseInt(body.getString("token_expiry_seconds"))); + assertEquals(newMaxSharingLifetimeSeconds + TOKEN_LIFETIME_TOLERANCE.toSeconds(), body.getLong(Const.Config.MaxSharingLifetimeProp)); + }); + testContext.completeNow(); + }); + } + + @Test + void keyBidstreamRespectsConfigValuesWithRemoteConfig(Vertx vertx, VertxTestContext testContext) { + int newMaxBidstreamLifetimeSeconds = 999999; + + this.runtimeConfig = this.runtimeConfig + .toBuilder() + .withMaxBidstreamLifetimeSeconds(newMaxBidstreamLifetimeSeconds) + .build(); + + final String apiVersion = "v2"; + final KeyDownloadEndpoint endpoint = KeyDownloadEndpoint.BIDSTREAM; + + final int clientSiteId = 101; + fakeAuth(clientSiteId, Role.ID_READER); + + // Required, sets up mock keys. + new MultipleKeysetsTests(); + + send(apiVersion, vertx, apiVersion + endpoint.getPath(), true, null, null, 200, respJson -> { + testContext.verify(() -> { + JsonObject body = respJson.getJsonObject("body"); + assertNotNull(body); + assertEquals(newMaxBidstreamLifetimeSeconds + TOKEN_LIFETIME_TOLERANCE.toSeconds(), body.getLong(Const.Config.MaxBidstreamLifetimeSecondsProp)); + }); + testContext.completeNow(); + }); + } + + private void assertLastUpdatedHasMillis(JsonArray buckets) { + for (int i = 0; i < buckets.size(); i++) { + JsonObject bucket = buckets.getJsonObject(i); + String lastUpdated = bucket.getString("last_updated"); + // Verify pattern yyyy-MM-dd'T'HH:mm:ss.SSS + assertTrue(lastUpdated.matches("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}"), + "last_updated does not contain millisecond precision: " + lastUpdated); + } + } + + @ParameterizedTest + @ValueSource(strings = {"v2"}) + void identityBucketsAlwaysReturnMilliseconds(String apiVersion, Vertx vertx, VertxTestContext testContext) { + final int clientSiteId = 201; + fakeAuth(clientSiteId, Role.MAPPER); + setupSalts(); + + // SaltEntry with a lastUpdated that has 0 milliseconds + long lastUpdatedMillis = Instant.parse("2024-01-01T00:00:00Z").toEpochMilli(); + SaltEntry bucketEntry = new SaltEntry(456, "hashed456", lastUpdatedMillis, "salt456", null, null, null, null); + when(saltProviderSnapshot.getModifiedSince(any())).thenReturn(List.of(bucketEntry)); + + String sinceTimestamp = "2023-12-31T00:00:00"; // earlier timestamp + + boolean isV1 = apiVersion.equals("v1"); + String v1Param = isV1 ? "since_timestamp=" + sinceTimestamp : null; + JsonObject req = isV1 ? null : new JsonObject().put("since_timestamp", sinceTimestamp); + + send(apiVersion, vertx, apiVersion + "/identity/buckets", isV1, v1Param, req, 200, respJson -> { + JsonArray buckets = respJson.getJsonArray("body"); + assertFalse(buckets.isEmpty()); + assertLastUpdatedHasMillis(buckets); + testContext.completeNow(); + }); + } } diff --git a/src/test/java/com/uid2/operator/UidOperatorVerticleV4Test.java b/src/test/java/com/uid2/operator/UidOperatorVerticleV4Test.java deleted file mode 100644 index 7a040427e..000000000 --- a/src/test/java/com/uid2/operator/UidOperatorVerticleV4Test.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.uid2.operator; - -import com.uid2.shared.model.TokenVersion; - -import java.io.IOException; - -public class UidOperatorVerticleV4Test extends UIDOperatorVerticleTest { - public UidOperatorVerticleV4Test() throws IOException { - } - - @Override - protected TokenVersion getTokenVersion() {return TokenVersion.V4;} - -} diff --git a/src/test/java/com/uid2/operator/V2RequestUtilTest.java b/src/test/java/com/uid2/operator/V2RequestUtilTest.java index 008583cb8..1522e8633 100644 --- a/src/test/java/com/uid2/operator/V2RequestUtilTest.java +++ b/src/test/java/com/uid2/operator/V2RequestUtilTest.java @@ -1,30 +1,50 @@ package com.uid2.operator; -import com.uid2.operator.service.V2RequestUtil; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.LoggerContext; +import com.uid2.operator.model.identities.IdentityScope; +import com.uid2.operator.model.KeyManager; +import com.uid2.operator.service.V2RequestUtil; import com.uid2.shared.IClock; import com.uid2.shared.auth.ClientKey; +import com.uid2.shared.encryption.Random; +import com.uid2.shared.model.KeysetKey; import io.vertx.core.json.JsonObject; -import org.junit.Test; +import io.vertx.ext.web.RequestBody; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; import org.slf4j.LoggerFactory; import java.time.Instant; import java.util.Set; -import static org.junit.jupiter.api.Assertions.*; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrowsExactly; +import static org.mockito.Mockito.when; +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) public class V2RequestUtilTest { private static final String LOGGER_NAME = "com.uid2.operator.service.V2RequestUtil"; - private static MemoryAppender memoryAppender; - private IClock clock = mock(IClock.class); - private Instant mockNow = Instant.parse("2024-03-20T04:02:46.130Z"); + private static final Instant MOCK_NOW = Instant.parse("2024-03-20T04:02:46.130Z"); + + @Mock + private IClock clock; + @Mock + private KeyManager keyManager; + @Mock + private KeysetKey refreshKey; + private MemoryAppender memoryAppender; - public void setupMemoryAppender() { + private void setupMemoryAppender() { Logger logger = (Logger)LoggerFactory.getLogger(LOGGER_NAME); memoryAppender = new MemoryAppender(); memoryAppender.setContext((LoggerContext) LoggerFactory.getILoggerFactory()); @@ -33,20 +53,27 @@ public void setupMemoryAppender() { memoryAppender.start(); } + @BeforeEach + public void setup() { + setupMemoryAppender(); + } + @AfterEach - public void close() { + public void teardown() { memoryAppender.reset(); memoryAppender.stop(); } @Test public void testParseRequestWithExpectedJson() { - when(clock.now()).thenReturn(mockNow); + when(clock.now()).thenReturn(MOCK_NOW); + String testToken = "AdvertisingTokenmZ4dZgeuXXl6DhoXqbRXQbHlHhA96leN94U1uavZVspwKXlfWETZ3b%2FbesPFFvJxNLLySg4QEYHUAiyUrNncgnm7ppu0mi6wU2CW6hssiuEkKfstbo9XWgRUbWNTM%2BewMzXXM8G9j8Q%3D"; String testEmailHash = "LdhtUlMQ58ZZy5YUqGPRQw5xUMS5dXG5ocJHYJHbAKI="; JsonObject expectedPayload = new JsonObject(); expectedPayload.put("token", testToken); expectedPayload.put("email_hash", testEmailHash); + // The bodyString was encoded by below json: // { // "token": "AdvertisingTokenmZ4dZgeuXXl6DhoXqbRXQbHlHhA96leN94U1uavZVspwKXlfWETZ3b%2FbesPFFvJxNLLySg4QEYHUAiyUrNncgnm7ppu0mi6wU2CW6hssiuEkKfstbo9XWgRUbWNTM%2BewMzXXM8G9j8Q%3D", @@ -59,40 +86,47 @@ public void testParseRequestWithExpectedJson() { "YGdzZw9oM2RzBgB8THMyAEe408lvdfsTsGteaLAGayY=", "name", "contact", - mockNow, + MOCK_NOW, Set.of(), 113, false, "key-id" ); - V2RequestUtil.V2Request res = V2RequestUtil.parseRequest(bodyString, ck, clock); + V2RequestUtil.V2Request res = V2RequestUtil.parseRequestAsString(bodyString, ck, clock); + assertEquals(expectedPayload, res.payload); } @Test public void testParseRequestWithNullBody() { - when(clock.now()).thenReturn(mockNow); - V2RequestUtil.V2Request res = V2RequestUtil.parseRequest(null, null, clock); + when(clock.now()).thenReturn(MOCK_NOW); + + V2RequestUtil.V2Request res = V2RequestUtil.parseRequestAsString(null, null, clock); + assertEquals("Invalid body: Body is missing.", res.errorMessage); } @Test public void testParseRequestWithNonBase64Body() { - when(clock.now()).thenReturn(mockNow); - V2RequestUtil.V2Request res = V2RequestUtil.parseRequest("test string", null, clock); + when(clock.now()).thenReturn(MOCK_NOW); + + V2RequestUtil.V2Request res = V2RequestUtil.parseRequestAsString("test string", null, clock); + assertEquals("Invalid body: Body is not valid base64.", res.errorMessage); } @Test public void testParseRequestWithTooShortBody() { - when(clock.now()).thenReturn(mockNow); - V2RequestUtil.V2Request res = V2RequestUtil.parseRequest("dGVzdA==", null, clock); + when(clock.now()).thenReturn(MOCK_NOW); + + V2RequestUtil.V2Request res = V2RequestUtil.parseRequestAsString("dGVzdA==", null, clock); + assertEquals("Invalid body: Body too short. Check encryption method.", res.errorMessage); } @Test public void testParseRequestWithMalformedJson() { - setupMemoryAppender(); when(clock.now()).thenReturn(Instant.parse("2024-03-20T06:33:15.627Z")); + // The bodyString was encoded by below json: // { // "token": "AdvertisingTokenmZ4dZgeuXXl6DhoXqbRXQbHlHhA96leN94U1uavZVspwKXlfWETZ3b%2FbesPFFvJxNLLySg4QEYHUAiyUrNncgnm7ppu0mi6wU2CW6hssiuEkKfstbo9XWgRUbWNTM%2BewMzXXM8G9j8Q%3D", @@ -106,16 +140,43 @@ public void testParseRequestWithMalformedJson() { "YGdzZw9oM2RzBgB8THMyAEe408lvdfsTsGteaLAGayY=", "name", "contact", - mockNow, + MOCK_NOW, Set.of(), 113, false, "key-id" ); - V2RequestUtil.V2Request res = V2RequestUtil.parseRequest(bodyString, ck, clock); + V2RequestUtil.V2Request res = V2RequestUtil.parseRequestAsString(bodyString, ck, clock); + assertEquals("Invalid payload in body: Data is not valid json string.", res.errorMessage); assertThat(memoryAppender.countEventsForLogger(LOGGER_NAME)).isEqualTo(1); assertThat(memoryAppender.search("[ERROR] Invalid payload in body: Data is not valid json string.").size()).isEqualTo(1); assertThat(memoryAppender.checkNoThrowableLogged().size()).isEqualTo(1); } + + @Test + public void testHandleRefreshTokenInResponseBody() { + when(keyManager.getRefreshKey()).thenReturn(refreshKey); + when(refreshKey.getId()).thenReturn(Integer.MAX_VALUE); + when(refreshKey.getKeyBytes()).thenReturn(Random.getRandomKeyBytes()); + + String response = """ + { + "identity": { + "advertising_token": "A4AAABZBgXozOcvdoBLWXaJSltTRG27n1kFegS9IKt-wN8bUPIPKiUXu9gxOzB0CvYprD8-tJNJjYNUy_HegQ1DdWkHwTm9vz9C2PUPtWzZenVy3g5L3hrbD_c7GuA6M6suZAkQGgeRM-7ixjVK2iUKYs5fOgxqzAl21St-7Bm97mgUEoMmg37bW5-X9w3TVs6PAUgSF2DuQmmwVXeKIsmoQZA", + "refresh_token": "AAAAFkKfY/PfFkWOByfIqQpP/nWp70ULyurGFQU7CUs5VWWhSgvzFRqXBes5DBqn6GKtwgKH/dF1Cx6Id951RnumXMJ5Oebw4vxQSvtGMNroN1B6HuPZcZiMnvDaTKjCZSAMd6Rc61pZzaQQ7wDKNP9NHNIzRmp7oziVlnEkT/sTJFfZZQPMFjWNqPy2nR0CFg8Zxui5ac6Ix9KEIFXOPM2v1O3kUm5E6x8MJ4vRLclK3NtAbWE3imauSpGSVlqG12hQKEBfN5CbcGRtdQGzdZoWjl8adZQdovufwulg59o8yKrEVPpL7wmoQ5oBaG9GG+FZMx4ttzkS/UlW+uk5qxUopeCRsuOSD/zWAsDDPP+6/FFuIMj+ftASZ7gXVaDraWqD", + "identity_expires": 1728595268736, + "refresh_expires": 1731186368736, + "refresh_from": 1728594668736, + "refresh_response_key": "sMRiJivNZJ6msQSvZhsVooG2T/xXTigaFRBPFHCPGQQ=" + } + } + """; + JsonObject jsonBody = new JsonObject(response); + + IllegalArgumentException e = assertThrowsExactly( + IllegalArgumentException.class, + () -> V2RequestUtil.handleRefreshTokenInResponseBody(jsonBody, keyManager, IdentityScope.UID2)); + assertEquals("Generated refresh token's length=168 is not equal to=388", e.getMessage()); + } } diff --git a/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java b/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java index 1c25faecd..29521cee7 100644 --- a/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java +++ b/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java @@ -3,14 +3,19 @@ import com.uid2.operator.Const; import com.uid2.operator.Main; import com.uid2.operator.model.*; -import com.uid2.operator.model.userIdentity.FirstLevelHashIdentity; -import com.uid2.operator.model.userIdentity.HashedDiiIdentity; +import com.uid2.operator.model.identities.DiiType; +import com.uid2.operator.model.identities.FirstLevelHash; +import com.uid2.operator.model.identities.HashedDii; +import com.uid2.operator.model.identities.IdentityScope; import com.uid2.operator.service.EncryptedTokenEncoder; import com.uid2.operator.service.IUIDOperatorService; +import com.uid2.operator.service.ShutdownService; import com.uid2.operator.service.UIDOperatorService; import com.uid2.operator.store.CloudSyncOptOutStore; import com.uid2.operator.store.IOptOutStore; +import com.uid2.operator.vertx.OperatorShutdownHandler; import com.uid2.shared.Utils; +import com.uid2.shared.audit.UidInstanceIdProvider; import com.uid2.shared.auth.ClientKey; import com.uid2.shared.auth.Role; import com.uid2.shared.cloud.CloudStorageException; @@ -21,7 +26,7 @@ import com.uid2.shared.optout.OptOutHeap; import com.uid2.shared.optout.OptOutPartition; import com.uid2.shared.store.CloudPath; -import com.uid2.shared.store.RotatingSaltProvider; +import com.uid2.shared.store.salt.RotatingSaltProvider; import com.uid2.shared.store.reader.RotatingClientKeyProvider; import com.uid2.shared.store.reader.RotatingKeysetKeyStore; import com.uid2.shared.store.reader.RotatingKeysetProvider; @@ -36,6 +41,7 @@ import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.time.Clock; +import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -45,46 +51,42 @@ public class BenchmarkCommon { - static IUIDOperatorService createUidOperatorService() throws Exception { + final static int IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS = 600; + final static int REFRESH_TOKEN_EXPIRES_AFTER_SECONDS = 900; + final static int REFRESH_IDENTITY_TOKEN_AFTER_SECONDS = 300; + + static IUIDOperatorService createUidOperatorService() throws Exception { RotatingKeysetKeyStore keysetKeyStore = new RotatingKeysetKeyStore( new EmbeddedResourceStorage(Main.class), new GlobalScope(new CloudPath("/com.uid2.core/test/keyset_keys/metadata.json"))); keysetKeyStore.loadContent(); - RotatingKeysetProvider keysetProvider = new RotatingKeysetProvider( - new EmbeddedResourceStorage(Main.class), - new GlobalScope(new CloudPath("/com.uid2.core/test/keysets/metadata.json"))); - keysetProvider.loadContent(); + RotatingKeysetProvider keysetProvider = new RotatingKeysetProvider( + new EmbeddedResourceStorage(Main.class), + new GlobalScope(new CloudPath("/com.uid2.core/test/keysets/metadata.json"))); + keysetProvider.loadContent(); RotatingSaltProvider saltProvider = new RotatingSaltProvider( new EmbeddedResourceStorage(Main.class), "/com.uid2.core/test/salts/metadata.json"); saltProvider.loadContent(); - final int IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS = 600; - final int REFRESH_TOKEN_EXPIRES_AFTER_SECONDS = 900; - final int REFRESH_IDENTITY_TOKEN_AFTER_SECONDS = 300; - - final JsonObject config = new JsonObject(); - config.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); - config.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); - config.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); - final EncryptedTokenEncoder tokenEncoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); final List optOutPartitionFiles = new ArrayList<>(); final ICloudStorage optOutLocalStorage = make1mOptOutEntryStorage( saltProvider.getSnapshot(Instant.now()).getFirstLevelSalt(), /* out */ optOutPartitionFiles); final IOptOutStore optOutStore = new StaticOptOutStore(optOutLocalStorage, make1mOptOutEntryConfig(), optOutPartitionFiles); - + final OperatorShutdownHandler shutdownHandler = new OperatorShutdownHandler(Duration.ofHours(1), Duration.ofHours(1), Clock.systemUTC(), new ShutdownService()); return new UIDOperatorService( - config, optOutStore, saltProvider, tokenEncoder, Clock.systemUTC(), IdentityScope.UID2, - null + shutdownHandler::handleSaltRetrievalResponse, + false, + new UidInstanceIdProvider("test-instance", "id") ); } @@ -150,13 +152,12 @@ static ICloudStorage make1mOptOutEntryStorage(String salt, List out_gene return storage; } - static HashedDiiIdentity[] createHashedDiiIdentities() { - HashedDiiIdentity[] arr = new HashedDiiIdentity[65536]; + static HashedDii[] createHashedDiiIdentities() { + HashedDii[] arr = new HashedDii[65536]; for (int i = 0; i < 65536; i++) { final byte[] diiHash = new byte[33]; new Random().nextBytes(diiHash); - arr[i] = new HashedDiiIdentity(IdentityScope.UID2, IdentityType.Email, diiHash, 0, - Instant.now().minusSeconds(120), Instant.now().minusSeconds(60)); + arr[i] = new HashedDii(IdentityScope.UID2, DiiType.Email, diiHash); } return arr; } @@ -169,7 +170,7 @@ static SourcePublisher createSourcePublisher() throws Exception { for (ClientKey client : clients.getAll()) { if (client.hasRole(Role.GENERATOR)) { - return new SourcePublisher(client.getSiteId(), 0, 0); + return new SourcePublisher(client.getSiteId()); } } throw new IllegalStateException("embedded resource does not include any publisher key"); @@ -189,14 +190,14 @@ public StaticOptOutStore(ICloudStorage storage, JsonObject jsonConfig, Collectio } @Override - public Instant getLatestEntry(FirstLevelHashIdentity firstLevelHashIdentity) { - long epochSecond = this.snapshot.getOptOutTimestamp(firstLevelHashIdentity.firstLevelHash); + public Instant getLatestEntry(FirstLevelHash firstLevelHash) { + long epochSecond = this.snapshot.getOptOutTimestamp(firstLevelHash.firstLevelHash()); Instant instant = epochSecond > 0 ? Instant.ofEpochSecond(epochSecond) : null; return instant; } @Override - public void addEntry(FirstLevelHashIdentity firstLevelHashIdentity, byte[] advertisingId, Handler> handler) { + public void addEntry(FirstLevelHash firstLevelHash, byte[] advertisingId, String uidTraceId, String uidInstanceId, Handler> handler) { // noop } diff --git a/src/test/java/com/uid2/operator/benchmark/IdentityMapBenchmark.java b/src/test/java/com/uid2/operator/benchmark/IdentityMapBenchmark.java index 880875fc0..f4064244d 100644 --- a/src/test/java/com/uid2/operator/benchmark/IdentityMapBenchmark.java +++ b/src/test/java/com/uid2/operator/benchmark/IdentityMapBenchmark.java @@ -1,17 +1,32 @@ package com.uid2.operator.benchmark; import com.uid2.operator.model.*; -import com.uid2.operator.model.userIdentity.HashedDiiIdentity; +import com.uid2.operator.model.identities.HashedDii; import com.uid2.operator.service.IUIDOperatorService; +import com.uid2.operator.service.V2RequestUtil; +import com.uid2.operator.vertx.V2PayloadHandler; +import com.uid2.shared.InstantClock; +import com.uid2.shared.Utils; +import com.uid2.shared.auth.ClientKey; +import com.uid2.shared.auth.Role; +import com.uid2.shared.encryption.AesGcm; +import io.vertx.core.buffer.Buffer; +import io.vertx.core.json.JsonObject; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; import java.time.Instant; +import java.util.*; +import java.util.concurrent.TimeUnit; public class IdentityMapBenchmark { private static final IUIDOperatorService uidService; - private static final HashedDiiIdentity[] hashedDiiIdentities; + private static final HashedDii[] hashedDiiIdentities; private static int idx = 0; static { @@ -23,15 +38,140 @@ public class IdentityMapBenchmark { } } + @State(Scope.Thread) + public static class PayloadState { + @Param({"100", "1000", "10000"}) + int numRecords; + + Buffer payloadBinary; + String payloadNone; + + private static final ClientKey clientKey = new ClientKey( + "FIUDeEqA+O2hW7PXdAJI/NAnJleHX+6QU46avhal5Wy4th0ZEKvG5eoFk8CFvqxGkMFlHwGG+DWTU+ZdwiHdQg==", + "RvCHAXUo/1pR3Nun5HuzcRJvlC7vT6gZMmsGPzyOONA=", + "DzBzbjTJcYL0swDtFs2krRNu+g1Eokm2tBU4dEuD0Wk=", + "test", + Instant.now(), + Set.of(Role.MAPPER, Role.GENERATOR, Role.ID_READER, Role.SHARER, Role.OPTOUT), + 999, + "UID2-C-L-999-fCXrM" + ); + static Instant now = Instant.now(); + static byte[] nonce = com.uid2.shared.encryption.Random.getBytes(8); + private static final Random RANDOM = new Random(); + + + @Setup + public void setup() { + this.payloadBinary = Buffer.buffer(createEncryptedPayload(this.numRecords)); + this.payloadNone = Utils.toBase64String(createEncryptedPayload(this.numRecords)); + } + + private static String randomEmail() { + return "email_" + Math.abs(RANDOM.nextLong()) + "@example.com"; + } + + private static String randomPhoneNumber() { + // Phone numbers with 15 digits are technically valid but are not used in any country + return "+" + String.format("%015d", Math.abs(RANDOM.nextLong() % 1_000_000_000_000_000L)); + } + + private static String randomHash() { + // This isn't really a hashed DII but looks like one ot UID2 + byte[] randomBytes = new byte[32]; + RANDOM.nextBytes(randomBytes); + return Base64.getEncoder().encodeToString(randomBytes); + } + + private static JsonObject createDII(int numRecords) { + JsonObject dii = new JsonObject(); + List emails = new ArrayList<>(); + List phones = new ArrayList<>(); + List emailHashes = new ArrayList<>(); + List phoneHashes = new ArrayList<>(); + for (int i = 0; i < numRecords; i++) { + emails.add(randomEmail()); + phones.add(randomPhoneNumber()); + emailHashes.add(randomHash()); + phoneHashes.add(randomHash()); + } + dii.put("email", emails); + dii.put("email_hash", emailHashes); + dii.put("phones", phones); + dii.put("phone_hash", phoneHashes); + return dii; + } + + private static byte[] createEncryptedPayload(int numRecords) { + Buffer b = Buffer.buffer(); + b.appendLong(now.toEpochMilli()); + b.appendLong(new BigInteger(nonce).longValue()); + + b.appendBytes(createDII(numRecords).encode().getBytes(StandardCharsets.UTF_8)); + + Buffer bufBody = Buffer.buffer(); + bufBody.appendByte((byte) 1); + byte[] payload = b.getBytes(); + + bufBody.appendBytes(AesGcm.encrypt(payload, clientKey.getSecretBytes())); + + return bufBody.getBytes(); + } + } + @Benchmark @BenchmarkMode(Mode.Throughput) - public RawUidResponse IdentityMapRawThroughput() { + public IdentityMapResponseItem IdentityMapRawThroughput() { return uidService.map(hashedDiiIdentities[(idx++) & 65535], Instant.now()); } @Benchmark @BenchmarkMode(Mode.Throughput) - public RawUidResponse IdentityMapWithOptOutThroughput() { - return uidService.mapIdentity(new MapRequest(hashedDiiIdentities[(idx++) & 65535], OptoutCheckPolicy.RespectOptOut, Instant.now())); + public IdentityMapResponseItem IdentityMapWithOptOutThroughput() { + return uidService.mapHashedDii(new IdentityMapRequestItem(hashedDiiIdentities[(idx++) & 65535], OptoutCheckPolicy.RespectOptOut, Instant.now())); + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + @Fork(2) + @Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) + @Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) + public void decompressionBenchmarkingBinary(PayloadState state, Blackhole bh) { + var data = V2RequestUtil.parseRequestAsBuffer(state.payloadBinary, PayloadState.clientKey, new InstantClock()); + bh.consume(data); + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + @Fork(2) + @Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) + @Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) + public void decompressionBenchmarkingNone(PayloadState state, Blackhole bh) { + var data = V2RequestUtil.parseRequestAsString(state.payloadNone, PayloadState.clientKey, new InstantClock()); + bh.consume(data); + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + @Fork(2) + @Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) + @Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) + public void compressionBenchmarkingBinary(PayloadState state, Blackhole bh) { + var data = V2PayloadHandler.encryptResponse(PayloadState.nonce, PayloadState.createDII(state.numRecords), PayloadState.clientKey.getSecretBytes()); + bh.consume(data); + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + @Fork(2) + @Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) + @Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) + public void compressionBenchmarkingNone(PayloadState state, Blackhole bh) { + var data = V2PayloadHandler.encryptResponse(PayloadState.nonce, PayloadState.createDII(state.numRecords), PayloadState.clientKey.getSecretBytes()); + bh.consume(data); } } diff --git a/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java b/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java index e907f638c..5a376ea95 100644 --- a/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java +++ b/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java @@ -1,23 +1,24 @@ package com.uid2.operator.benchmark; import com.uid2.operator.model.*; -import com.uid2.operator.model.userIdentity.HashedDiiIdentity; +import com.uid2.operator.model.identities.HashedDii; import com.uid2.operator.service.EncryptedTokenEncoder; import com.uid2.operator.service.IUIDOperatorService; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Mode; +import java.time.Duration; import java.util.ArrayList; import java.util.List; public class TokenEndecBenchmark { private static final IUIDOperatorService uidService; - private static final HashedDiiIdentity[] hashedDiiIdentities; + private static final HashedDii[] hashedDiiIdentities; private static final SourcePublisher publisher; private static final EncryptedTokenEncoder encoder; - private static final IdentityResponse[] generatedTokens; + private static final TokenGenerateResponse[] generatedTokens; private static int idx = 0; static { @@ -35,32 +36,44 @@ public class TokenEndecBenchmark { } } - static IdentityResponse[] createAdvertisingTokens() { - List tokens = new ArrayList<>(); + static TokenGenerateResponse[] createAdvertisingTokens() { + List tokens = new ArrayList<>(); for (int i = 0; i < hashedDiiIdentities.length; i++) { tokens.add( - uidService.generateIdentity(new IdentityRequest( + uidService.generateIdentity(new TokenGenerateRequest( publisher, hashedDiiIdentities[i], - OptoutCheckPolicy.DoNotRespect))); + OptoutCheckPolicy.DoNotRespect), + Duration.ofSeconds(BenchmarkCommon.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(BenchmarkCommon.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(BenchmarkCommon.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)) + ); } - return tokens.toArray(new IdentityResponse[tokens.size()]); + return tokens.toArray(new TokenGenerateResponse[tokens.size()]); } @Benchmark @BenchmarkMode(Mode.Throughput) - public IdentityResponse TokenGenerationBenchmark() { - return uidService.generateIdentity(new IdentityRequest( + public TokenGenerateResponse TokenGenerationBenchmark() { + return uidService.generateIdentity(new TokenGenerateRequest( publisher, hashedDiiIdentities[(idx++) & 65535], - OptoutCheckPolicy.DoNotRespect)); + OptoutCheckPolicy.DoNotRespect), + Duration.ofSeconds(BenchmarkCommon.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(BenchmarkCommon.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(BenchmarkCommon.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS) + ); } @Benchmark @BenchmarkMode(Mode.Throughput) - public RefreshResponse TokenRefreshBenchmark() { + public TokenRefreshResponse TokenRefreshBenchmark() { return uidService.refreshIdentity( encoder.decodeRefreshToken( - generatedTokens[(idx++) & 65535].getRefreshToken())); + generatedTokens[(idx++) & 65535].getRefreshToken()), + Duration.ofSeconds(BenchmarkCommon.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(BenchmarkCommon.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(BenchmarkCommon.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS) + ); } } diff --git a/src/test/java/com/uid2/operator/service/ResponseUtilTest.java b/src/test/java/com/uid2/operator/service/ResponseUtilTest.java index 103dd73a6..77f848cc3 100644 --- a/src/test/java/com/uid2/operator/service/ResponseUtilTest.java +++ b/src/test/java/com/uid2/operator/service/ResponseUtilTest.java @@ -42,12 +42,13 @@ void tearDown() { @Test void logsErrorWithNoExtraDetails() { - ResponseUtil.Error("Some error status", 500, rc, "Some error message"); + ResponseUtil.LogErrorAndSendResponse("Some error status", 500, rc, "Some error message"); - String expected = "Error response to http request. {" + + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":null," + "\"siteId\":null," + + "\"path\":null," + "\"statusCode\":500," + "\"clientAddress\":null," + "\"message\":\"Some error message\"" + @@ -65,12 +66,13 @@ void logsErrorWithExtraDetailsFromAuthorizable() { when(mockAuthorizable.getSiteId()).thenReturn(10); when(rc.data().get("api-client")).thenReturn(mockAuthorizable); - ResponseUtil.Error("Some error status", 500, rc, "Some error message"); + ResponseUtil.LogErrorAndSendResponse("Some error status", 500, rc, "Some error message"); - String expected = "Error response to http request. {" + + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":\"Test Contract\"," + "\"siteId\":10," + + "\"path\":null," + "\"statusCode\":500," + "\"clientAddress\":null," + "\"message\":\"Some error message\"" + @@ -83,12 +85,13 @@ void logsErrorWithExtraDetailsFromAuthorizable() { void logsErrorWithSiteIdFromContext() { when(rc.get(Const.RoutingContextData.SiteId)).thenReturn(20); - ResponseUtil.Error("Some error status", 500, rc, "Some error message"); + ResponseUtil.LogErrorAndSendResponse("Some error status", 500, rc, "Some error message"); - String expected = "Error response to http request. {" + + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":null," + "\"siteId\":20," + + "\"path\":null," + "\"statusCode\":500," + "\"clientAddress\":null," + "\"message\":\"Some error message\"" + @@ -104,12 +107,13 @@ void logsErrorWithClientAddress() { when(rc.request().remoteAddress()).thenReturn(socket); - ResponseUtil.Error("Some error status", 500, rc, "Some error message"); + ResponseUtil.LogErrorAndSendResponse("Some error status", 500, rc, "Some error message"); - String expected = "Error response to http request. {" + + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":null," + "\"siteId\":null," + + "\"path\":null," + "\"statusCode\":500," + "\"clientAddress\":\"192.168.10.10\"," + "\"message\":\"Some error message\"" + @@ -124,11 +128,12 @@ void logsErrorWithServiceAndServiceLinkNames() { when(rc1.get(SecureLinkValidatorService.SERVICE_LINK_NAME, "")).thenReturn("TestLink1"); when(rc1.get(SecureLinkValidatorService.SERVICE_NAME, "")).thenReturn("TestService1"); - ResponseUtil.Error("Some error status", 500, rc1, "Some error message"); - String expected = "Error response to http request. {" + + ResponseUtil.LogErrorAndSendResponse("Some error status", 500, rc1, "Some error message"); + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":null," + "\"siteId\":null," + + "\"path\":null," + "\"statusCode\":500," + "\"clientAddress\":null," + "\"message\":\"Some error message\"," + @@ -144,9 +149,9 @@ void logsWarningWithOrigin() { when(request.getHeader("origin")).thenReturn("testOriginHeader"); when(rc.request()).thenReturn(request); - ResponseUtil.Warning("Some error status", 400, rc, "Some error message"); + ResponseUtil.LogInfoAndSendResponse("Some error status", 400, rc, "Some error message"); - String expected = "Warning response to http request. {" + + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":null," + "\"siteId\":null," + @@ -165,9 +170,9 @@ void logsWarningWithOriginNull() { when(request.getHeader("origin")).thenReturn(null); when(rc.request()).thenReturn(request); - ResponseUtil.Warning("Some error status", 400, rc, "Some error message"); + ResponseUtil.LogWarningAndSendResponse("Some error status", 400, rc, "Some error message"); - String expected = "Warning response to http request. {" + + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":null," + "\"siteId\":null," + @@ -185,9 +190,9 @@ void logsWarningWithReferer() { when(request.getHeader("referer")).thenReturn("testRefererHeader"); when(rc.request()).thenReturn(request); - ResponseUtil.Warning("Some error status", 400, rc, "Some error message"); + ResponseUtil.LogInfoAndSendResponse("Some error status", 400, rc, "Some error message"); - String expected = "Warning response to http request. {" + + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":null," + "\"siteId\":null," + @@ -206,9 +211,9 @@ void logsWarningWithRefererNull() { when(request.getHeader("referer")).thenReturn(null); when(rc.request()).thenReturn(request); - ResponseUtil.Warning("Some error status", 400, rc, "Some error message"); + ResponseUtil.LogWarningAndSendResponse("Some error status", 400, rc, "Some error message"); - String expected = "Warning response to http request. {" + + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":null," + "\"siteId\":null," + diff --git a/src/test/java/com/uid2/operator/service/SecureLinkValidatorServiceTest.java b/src/test/java/com/uid2/operator/service/SecureLinkValidatorServiceTest.java index 68decfb9f..9f8cd8a81 100644 --- a/src/test/java/com/uid2/operator/service/SecureLinkValidatorServiceTest.java +++ b/src/test/java/com/uid2/operator/service/SecureLinkValidatorServiceTest.java @@ -67,6 +67,18 @@ void validateRequest_linkIdNotFound_returnsFalse() { assertFalse(service.validateRequest(this.routingContext, requestJsonObject, Role.MAPPER)); } + @Test + void validateRequest_linkIdFoundLinkDisabled_returnsFalse() { + this.setClientKey(10); + JsonObject requestJsonObject = new JsonObject(); + requestJsonObject.put(SecureLinkValidatorService.LINK_ID, "999"); + + when(this.rotatingServiceLinkStore.getServiceLink(10, "999")).thenReturn(new ServiceLink("999", 10, 100, "testServiceLink", Set.of(Role.MAPPER), true)); + + SecureLinkValidatorService service = new SecureLinkValidatorService(this.rotatingServiceLinkStore, this.rotatingServiceStore); + assertFalse(service.validateRequest(this.routingContext, requestJsonObject, Role.MAPPER)); + } + @Test void validateRequest_roleNotInServiceLink_returnsFalse() { this.setClientKey(10); diff --git a/src/test/java/com/uid2/operator/service/TokenUtilsTest.java b/src/test/java/com/uid2/operator/service/TokenUtilsTest.java deleted file mode 100644 index 2fb7af1fd..000000000 --- a/src/test/java/com/uid2/operator/service/TokenUtilsTest.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.uid2.operator.service; - -import com.uid2.shared.cloud.CloudStorageException; -import org.junit.jupiter.api.Test; - -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import static com.uid2.operator.service.TokenUtils.getSiteIdsUsingV4Tokens; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -public class TokenUtilsTest { - Set siteIdsV4TokensSet = new HashSet<>(Arrays.asList(127, 128)); - @Test - void getSiteIdsUsingV4Tokens_multipleSiteIds() { - Set actualSiteIdsV4TokensSet = getSiteIdsUsingV4Tokens("127, 128"); - assertEquals(siteIdsV4TokensSet, actualSiteIdsV4TokensSet); - } - - @Test - void getSiteIdsUsingV4Tokens_oneSiteIds() { - Set actualSiteIdsV4TokensSet = getSiteIdsUsingV4Tokens("127"); - assertEquals(new HashSet<>(List.of(127)), actualSiteIdsV4TokensSet); - } - - @Test - void getSiteIdsUsingV4Tokens_emptyInput() { - Set actualSiteIdsV4TokensSet = getSiteIdsUsingV4Tokens(""); - assertEquals(new HashSet<>(), actualSiteIdsV4TokensSet); - } - - @Test - void getSiteIdsUsingV4Tokens_inputContainsSpaces() { - Set actualSiteIdsV4TokensSet = getSiteIdsUsingV4Tokens(" 127 ,128 "); - assertEquals(siteIdsV4TokensSet, actualSiteIdsV4TokensSet); - } - - @Test - void getSiteIdsUsingV4Tokens_inputContainsInvalidInteger() { - assertThrows(IllegalArgumentException.class, - () -> getSiteIdsUsingV4Tokens(" 1 27 ,128 ")); - } -} diff --git a/src/test/java/com/uid2/operator/store/RuntimeConfigTest.java b/src/test/java/com/uid2/operator/store/RuntimeConfigTest.java new file mode 100644 index 000000000..1c04b5f7f --- /dev/null +++ b/src/test/java/com/uid2/operator/store/RuntimeConfigTest.java @@ -0,0 +1,142 @@ +package com.uid2.operator.store; + +import io.vertx.core.json.JsonObject; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.stream.Stream; + +import static com.uid2.operator.Const.Config.*; +import static com.uid2.operator.service.UIDOperatorService.*; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.*; + +public class RuntimeConfigTest { + private final JsonObject validConfig = new JsonObject() + .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 259200) + .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 2592000) + .put(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, 3600) + .put(SharingTokenExpiryProp, 2592000); + + @Test + void validConfigDoesNotThrow() { + assertDoesNotThrow(this::mapToRuntimeConfig); + } + + @Test + void mapToRuntimeConfigPreservesValues() { + RuntimeConfig config = mapToRuntimeConfig(); + + assertEquals(validConfig.getInteger(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS), config.getIdentityTokenExpiresAfterSeconds()); + assertEquals(validConfig.getInteger(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), config.getRefreshTokenExpiresAfterSeconds()); + assertEquals(validConfig.getInteger(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), config.getRefreshIdentityTokenAfterSeconds()); + assertEquals(validConfig.getInteger(SharingTokenExpiryProp), config.getSharingTokenExpirySeconds()); + } + + @Test + void runtimeConfigUsesCorrectDefaultValues() { + RuntimeConfig config = mapToRuntimeConfig(); + + assertEquals(validConfig.getInteger(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS), config.getMaxBidstreamLifetimeSeconds()); + assertEquals(validConfig.getInteger(SharingTokenExpiryProp), config.getMaxSharingLifetimeSeconds()); + } + + @Test + void maxBidstreamLifetimeSecondsIsReturnedIfSet() { + int maxBidstreamLifetimeSeconds = validConfig.getInteger(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS) + 1; + validConfig.put(MaxBidstreamLifetimeSecondsProp, maxBidstreamLifetimeSeconds); + + RuntimeConfig config = mapToRuntimeConfig(); + + assertEquals(maxBidstreamLifetimeSeconds, config.getMaxBidstreamLifetimeSeconds()); + } + + @Test + void maxSharingLifetimeSecondsIsReturnedIfSet() { + validConfig.put(MaxSharingLifetimeProp, 123); + + RuntimeConfig config = mapToRuntimeConfig(); + + assertEquals(123, config.getMaxSharingLifetimeSeconds()); + } + + @Test + void extraPropertyDoesNotThrow() { + validConfig.put("some_new_property", 1); + + assertDoesNotThrow(this::mapToRuntimeConfig); + } + + @ParameterizedTest + @MethodSource("requiredFields") + void requiredFieldIsNullThrows(String propertyName) { + validConfig.putNull(propertyName); + + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, this::mapToRuntimeConfig); + + assertThat(ex.getMessage()).contains(String.format("%s is required", propertyName)); + } + + @ParameterizedTest + @MethodSource("requiredFields") + void requiredFieldIsMissingThrows(String propertyName) { + validConfig.remove(propertyName); + + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, this::mapToRuntimeConfig); + + assertThat(ex.getMessage()).contains(String.format("%s is required", propertyName)); + } + + private RuntimeConfig mapToRuntimeConfig() { + return validConfig.mapTo(RuntimeConfig.class); + } + + private static Stream requiredFields() { + return Stream.of( + Arguments.of(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS), + Arguments.of(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Arguments.of(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Arguments.of(SharingTokenExpiryProp) + ); + } + + @Test + void identityTokenExpiresAfterSecondsIsGreaterThanRefreshTokenExpiresAfterSecondsThrows() { + validConfig.put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 10); + validConfig.put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 5); + + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, this::mapToRuntimeConfig); + assertThat(ex.getMessage()).contains("refresh_token_expires_after_seconds (5) must be >= identity_token_expires_after_seconds (10)"); + } + + @Test + void refreshIdentityAfterSecondsIsGreaterThanIdentityTokenExpiresAfterSecondsThrows() { + validConfig.put(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, 6); + validConfig.put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 5); + + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, this::mapToRuntimeConfig); + assertThat(ex.getMessage()).contains("identity_token_expires_after_seconds (5) must be >= refresh_identity_token_after_seconds (6)"); + } + + @Test + void maxBidStreamLifetimeSecondsIsLessThanIdentityTokenExpiresAfterSecondsThrows() { + int newMaxBidStreamLifetimeSeconds = validConfig.getInteger(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS) - 1; + validConfig.put(MaxBidstreamLifetimeSecondsProp, newMaxBidStreamLifetimeSeconds); + + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, this::mapToRuntimeConfig); + assertThat(ex.getMessage()).contains(String.format("max_bidstream_lifetime_seconds (%d) must be >= identity_token_expires_after_seconds (%d)", newMaxBidStreamLifetimeSeconds, validConfig.getInteger(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS))); + } + + @Test + void toBuilderBuildReturnsEquivalentObject() { + RuntimeConfig config = mapToRuntimeConfig(); + + RuntimeConfig toBuilderBuild = config.toBuilder().build(); + + assertThat(toBuilderBuild) + .usingRecursiveComparison() + .isEqualTo(config); + } +} diff --git a/src/test/java/com/uid2/operator/util/DomainNameCheckUtilTest.java b/src/test/java/com/uid2/operator/util/DomainNameCheckUtilTest.java new file mode 100644 index 000000000..f1158f83d --- /dev/null +++ b/src/test/java/com/uid2/operator/util/DomainNameCheckUtilTest.java @@ -0,0 +1,73 @@ +package com.uid2.operator.util; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.Set; + +import static com.uid2.operator.util.DomainNameCheckUtil.isDomainNameAllowed; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class DomainNameCheckUtilTest { + @ParameterizedTest + @ValueSource(strings = { + "http://examplewebsite.com", + "https://examplewebsite.com", + "https://abc.examplewebsite.com:8080", + "https://abc.examplewebsite.com:8080/", + "https://abc.eXampleWebsIte.com:8080/", + "https://abc.exAmplewEbsite.com:8080/blahh/a.html", + + "http://e-wb.org", + "https://e-wb.org", + "https://abc.e-wb.org:8080", + "https://abc.e-wb.org:8080/", + "https://abc.e-Wb.org:8080/", + "https://abc.e-wb.org:8080/blahh/a.html", + + "http://aussiedomain.id.au", + "https://aussiedomain.id.au/head.html" + }) + void testDomainNameCheckSuccess(String origin) { + Set allowedDomainNamesForProd = Set.of("examplewebsite.com","e-wb.org","aussiedomain.id.au"); + + assertTrue(isDomainNameAllowed(origin, allowedDomainNamesForProd)); + } + + @ParameterizedTest + @ValueSource(strings = { + "http://localhost", + "https://localhost:8080/", + "https://abc.localhost:8080", + "https://abc.localhost:8080/", + "https://abc.locaLHost:8080/", + "https://abc.localhost:8080/blahh/a.html" + }) + void testLocalhostDomainNameCheck(String origin) { + Set allowedDomainNamesForTesting = Set.of("examplewebsite.com", "e-wb.org", "localhost"); + + assertTrue(isDomainNameAllowed(origin, allowedDomainNamesForTesting)); + } + + @ParameterizedTest + @ValueSource(strings = { + // Malformed URLs + "examplewebsite.com", + "examplewebsite.com:999999", + "abc:examplewebsite.com", + "/:$2231examplewebsite.com", + "/:$2231examplewebsite.com/23423/sfs.html", + + // Disallowed domain names + "http://boohoo.id.au", + "https://blah12.com", + "http://123.boohoo.id.au", + "https://456.blah12.com" + }) + void testDomainNameCheckFailure(String origin) { + Set allowedDomainNamesForProd = Set.of("examplewebsite.com", "e-wb.org", "aussiedomain.id.au"); + + assertFalse(isDomainNameAllowed(origin, allowedDomainNamesForProd)); + } +} diff --git a/src/test/java/com/uid2/operator/utilTests/IdentityMapResponseItemTest.java b/src/test/java/com/uid2/operator/utilTests/IdentityMapResponseItemTest.java new file mode 100644 index 000000000..e24539dc6 --- /dev/null +++ b/src/test/java/com/uid2/operator/utilTests/IdentityMapResponseItemTest.java @@ -0,0 +1,40 @@ +package com.uid2.operator.utilTests; + +import com.uid2.operator.model.IdentityMapResponseItem; +import com.uid2.operator.service.EncodingUtils; +import org.junit.Test; + +import java.security.NoSuchAlgorithmException; +import java.time.Instant; +import java.util.Arrays; + +import static org.junit.Assert.*; + + +public class IdentityMapResponseItemTest { + @Test + public void doRawUidResponseTest() throws NoSuchAlgorithmException { + assertEquals(IdentityMapResponseItem.OptoutIdentity.bucketId, ""); + assertTrue(IdentityMapResponseItem.OptoutIdentity.isOptedOut()); + + IdentityMapResponseItem optoutResponse = new IdentityMapResponseItem(new byte[33], null, null, null); + assertTrue(optoutResponse.isOptedOut()); + + byte[] rawUid = new byte[33]; + for(int i = 0; i < 33; i++) { + rawUid[i] = (byte) i; + } + + final Long expectedRefreshFrom = EncodingUtils.NowUTCMillis().toEpochMilli(); + byte[] expectedPreviousRawUid = new byte[33]; + for(int i = 0; i < 33; i++) { + rawUid[i] = (byte) 88; + } + + IdentityMapResponseItem generatedUid = new IdentityMapResponseItem(rawUid, "12345", expectedPreviousRawUid, expectedRefreshFrom); + assertFalse(generatedUid.isOptedOut()); + assertTrue(Arrays.equals(rawUid, generatedUid.rawUid)); + assertArrayEquals(expectedPreviousRawUid, generatedUid.previousRawUid); + assertEquals(expectedRefreshFrom, generatedUid.refreshFrom); + } +} diff --git a/src/test/java/com/uid2/operator/utilTests/PrivacyBitsTest.java b/src/test/java/com/uid2/operator/utilTests/PrivacyBitsTest.java new file mode 100644 index 000000000..91750bf87 --- /dev/null +++ b/src/test/java/com/uid2/operator/utilTests/PrivacyBitsTest.java @@ -0,0 +1,55 @@ +package com.uid2.operator.utilTests; + +import com.uid2.operator.util.PrivacyBits; +import org.junit.Test; +import java.security.NoSuchAlgorithmException; +import static org.junit.Assert.*; + + +public class PrivacyBitsTest { + @Test + public void doPrivacyBitsTest() throws NoSuchAlgorithmException { + assertEquals(PrivacyBits.DEFAULT.getAsInt(), 1); + PrivacyBits pb1 = new PrivacyBits(); + assertEquals(pb1.getAsInt(), 0); + assertEquals(pb1.hashCode(), 0); + assertNotEquals(pb1, PrivacyBits.fromInt(1)); + assertNotEquals(pb1, PrivacyBits.fromInt(121)); + assertFalse(pb1.isClientSideTokenGenerated()); + assertFalse(pb1.isClientSideTokenOptedOut()); + + pb1.setLegacyBit(); + assertEquals(pb1.getAsInt(), 0b1); + assertEquals(pb1.hashCode(), 0b1); + assertEquals(pb1, PrivacyBits.fromInt(1)); + assertNotEquals(pb1, PrivacyBits.fromInt(121)); + assertFalse(pb1.isClientSideTokenGenerated()); + assertFalse(pb1.isClientSideTokenOptedOut()); + + + pb1.setClientSideTokenGenerate(); + assertEquals(pb1.getAsInt(), 0b11); + assertEquals(pb1.hashCode(), 0b11); + assertEquals(pb1, PrivacyBits.fromInt(3)); + assertNotEquals(pb1, PrivacyBits.fromInt(121)); + assertTrue(pb1.isClientSideTokenGenerated()); + assertFalse(pb1.isClientSideTokenOptedOut()); + + + pb1.setClientSideTokenGenerateOptout(); + assertEquals(pb1.getAsInt(), 0b111); + assertEquals(pb1.hashCode(), 0b111); + assertEquals(pb1, PrivacyBits.fromInt(7)); + assertNotEquals(pb1, PrivacyBits.fromInt(121)); + assertTrue(pb1.isClientSideTokenGenerated()); + assertTrue(pb1.isClientSideTokenOptedOut()); + + PrivacyBits pb2 = new PrivacyBits(pb1); + assertEquals(pb2.getAsInt(), 0b111); + + PrivacyBits pb3 = PrivacyBits.fromInt(0b10110); + assertEquals(pb3.getAsInt(), 0b10110); + pb3.setLegacyBit(); + assertEquals(pb3.getAsInt(), 0b10111); + } +} diff --git a/src/test/java/com/uid2/operator/utilTests/TokenGenerateResponseTest.java b/src/test/java/com/uid2/operator/utilTests/TokenGenerateResponseTest.java new file mode 100644 index 000000000..c94c83c5c --- /dev/null +++ b/src/test/java/com/uid2/operator/utilTests/TokenGenerateResponseTest.java @@ -0,0 +1,45 @@ +package com.uid2.operator.utilTests; + +import com.uid2.operator.model.TokenGenerateResponse; +import com.uid2.shared.model.TokenVersion; +import io.vertx.core.json.JsonObject; +import org.junit.Test; + +import java.security.NoSuchAlgorithmException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import static org.junit.Assert.*; + + +public class TokenGenerateResponseTest { + @Test + public void doIdentityResponseTest() throws NoSuchAlgorithmException { + assertEquals(TokenGenerateResponse.OptOutResponse.getAdvertisingToken(), ""); + assertTrue(TokenGenerateResponse.OptOutResponse.isOptedOut()); + + TokenGenerateResponse nullAdTokenValue = new TokenGenerateResponse(null, TokenVersion.V4, "refreshToken", null,null,null); + assertTrue(nullAdTokenValue.isOptedOut()); + + Instant identityExpires = Instant.now(); + Instant refreshFrom = identityExpires.plus(5, ChronoUnit.MINUTES); + Instant refreshExpires = identityExpires.plus(10, ChronoUnit.MINUTES); + + + + TokenGenerateResponse response1 = new TokenGenerateResponse("adToken", TokenVersion.V3, "refreshToken", identityExpires + , refreshExpires, refreshFrom); + assertEquals(response1.getAdvertisingToken(), "adToken"); + assertEquals(response1.getAdvertisingTokenVersion(), TokenVersion.V3); + assertEquals(response1.getRefreshToken(), "refreshToken"); + assertEquals(response1.getIdentityExpires(), identityExpires); + assertEquals(response1.getRefreshExpires(), refreshExpires); + assertEquals(response1.getRefreshFrom(), refreshFrom); + + JsonObject jsonV1 = response1.toTokenGenerateResponseJson(); + assertEquals(jsonV1.getString("advertising_token"), response1.getAdvertisingToken()); + assertEquals(jsonV1.getString("refresh_token"), response1.getRefreshToken()); + assertEquals(jsonV1.getLong("refresh_expires").longValue(), response1.getRefreshExpires().toEpochMilli()); + assertEquals(jsonV1.getLong("refresh_from").longValue(), response1.getRefreshFrom().toEpochMilli()); + } +} diff --git a/static/js/euid-sdk-1.0.0.js b/static/js/euid-sdk-1.0.0.js index d89f0a712..fe4d85622 100644 --- a/static/js/euid-sdk-1.0.0.js +++ b/static/js/euid-sdk-1.0.0.js @@ -341,4 +341,4 @@ window.__euid = new EUID(); if (typeof exports !== 'undefined') { exports.EUID = EUID; exports.window = window; -} +} \ No newline at end of file diff --git a/static/js/openid-sdk-1.0.js b/static/js/openid-sdk-1.0.js deleted file mode 100644 index cc5811c09..000000000 --- a/static/js/openid-sdk-1.0.js +++ /dev/null @@ -1,150 +0,0 @@ -var __openId = { - - init : function(opts) { - this.opts = opts; - if (!this.opts["events"]) { - this.opts["events"] = {} - } - if (this.opts["events"]["init"]) { - this.printDebug("Calling init callback"); - this.opts["events"]["init"](this); - } - if (this.opts["start"]) { - if (this.opts["identity"]) { - this.setIdentity(this.opts["identity"]); - } else if (this.opts["email"]) { - this.startVerificationFlow(); - } else { - if (!this.detectFromUrl()) { - this.refreshIfNeededIdentity(); - return; - } - } - - this.establishIdentity(); - } - - }, - - getTDID : function() { - var cookie = Cookies.get("__open_id") - if (cookie) { - var payload = JSON.parse(decodeURIComponent(cookie)); - return payload["tdid"]; - } - }, - - detectFromUrl : function() { - const urlParams = new URLSearchParams(window.location.search); - const payload = urlParams.get("__oidt"); - console.log("Payload = "); - console.log(payload); - - if (payload && payload != "") { - this.setIdentity(payload); - return true; - } else { - return false; - } - }, - - sendCode : function() { - - $("#verification-entry").show(); - - }, - verifyCode: function() { - - console.log("Submit Value"); - - var email = this.opts["email"]; - var setIdentitfy = this.setIdentity; - var establish = this.establishIdentity; - - $.ajax({ - url: "https://www.openid2.com:444/identity/verification/submit?email="+email+ - "&privacy_bits=1&code=1234&token=abasca", - }) - .done(function( data ) { - var d = JSON.stringify(data); - - window.__openId.opts["identity"] = d; - window.__openId.setIdentity(); - window.__openId.establishIdentity(); - }); - - }, - startVerificationFlow : function() { - $("#open-id-container").show(); - $("#verification-email").val(this.opts["email"]); - $("#send-code").on("click", this.sendCode); - $("#verify-code").on("click", function() { window.__openId.verifyCode() }); - }, - - setIdentity : function(tokens) { - Cookies.set("__open_id", tokens); - }, - - - establishIdentity : function() { - var cookie = Cookies.get("__open_id") - if (cookie) { - var payload = JSON.parse(decodeURIComponent(cookie)); - console.log("Cookie Payload = "); - console.log(cookie); - this.opts["events"]["established"](payload["advertisement_token"]); - } else { - console.log("here"); - if (this.opts["events"]["not_established"]) { - this.opts["events"]["not_established"](); - } - } - }, - - disconnect : function() { - Cookies.remove("__open_id"); - this.establishIdentity(); - }, - - needsRereshing : function(paylod) { - var refreshToken = paylod["refresh_token"]; - // FIXME TODO check for Reresh and continue the Lifecycle - return true; - }, - - refreshIfNeededIdentity : function() { - var cookie = Cookies.get("__open_id") - if (cookie) { - var payload = JSON.parse(decodeURIComponent(cookie)); - console.log("Cookie Payload = "); - console.log(cookie); - if (this.needsRereshing(payload)) { - - $.ajax({ - url: "https://www.openid2.com:444/token/refresh?refresh_token="+encodeURIComponent(payload["refresh_token"]) - }) - .done(function( data ) { - console.log("Token = "); - console.log(data); - if (data && data["advertisement_token"] && data["advertisement_token"] != "") { - var d = encodeURIComponent(JSON.stringify(data)); - window.__openId.setIdentity(d); - } else { - window.__openId.disconnect(); - } - window.__openId.establishIdentity(); - }); - } - } else { - window.__openId.establishIdentity(); - } - this.printDebug("Reresh Token here"); - }, - - printDebug : function(m) { - console.log("__open_id: " + m); - - } -} -window.__openId = __openId; -console.log("OepnID SDK Loaded"); diff --git a/static/js/uid2-esp-0.0.1a.js b/static/js/uid2-esp-0.0.1a.js deleted file mode 100644 index 0cac4edef..000000000 --- a/static/js/uid2-esp-0.0.1a.js +++ /dev/null @@ -1,20 +0,0 @@ -function __esp_getUID2Async(cb) { - return new Promise(function(cb) { - if (window.__uid2 && window.__uid2.getAdvertisingToken) { - cb(__uid2.getAdvertisingToken()); - } else { - throw new "UID2 SDK not present"; - } - }); -} - -if (typeof (googletag) !== "undefined" && googletag) { - - googletag.encryptedSignalProviders.push({ - id: 'uidapi.com', - collectorFunction: () => { - return __esp_getUID2Async().then((signals) => signals); - } - }); - -} \ No newline at end of file diff --git a/static/js/uid2-sdk-0.0.1a-source.ts b/static/js/uid2-sdk-0.0.1a-source.ts deleted file mode 100644 index c94526e9b..000000000 --- a/static/js/uid2-sdk-0.0.1a-source.ts +++ /dev/null @@ -1,97 +0,0 @@ -class UID2 { - - - - public init = (opts : object) => { - const identity = opts["identity"]; - if (identity) { - this.setIdentity(identity); - } else { - this.refreshIfNeeded(); - } - - } - - public refreshIfNeeded = () => { - - const identity = this.getIdentity(); - if (identity) { - const url = "https://prod.uidapi.com/token/refresh?refresh_token="+encodeURIComponent(identity["refresh_token"]); - const req = new XMLHttpRequest(); - req.overrideMimeType("application/json"); - var cb = this.handleRefreshResponse; - req.open("GET", url, false); - req.onload = function() { - cb(req.responseText); - } - req.send(); - - - - } - } - - private handleRefreshResponse = (body: string) => { - this.setIdentity(body); - } - - public getIdentity = () => { - const payload = this.getCookie("__uid_2"); - if (payload) { - return JSON.parse(payload); - } - } - - public getAdvertisingToken = () => { - const identity = this.getIdentity(); - if (identity) { - return identity["advertisement_token"]; - } - } - - public setIdentity = (value: object) => { - var payload; - if (typeof(value) === "object") { - payload = JSON.stringify(value); - } else { - payload = value; - } - this.setCookie("__uid_2", payload); - - } - - public setCookie = (name: string, value: string) => { - var days = 7; - var date = new Date(); - date.setTime(date.getTime()+(days*24*60*60*1000)); - - document.cookie=name + "=" + encodeURIComponent(value) +" ;path=/;expires="+date.toUTCString(); - - - } - public getCookie = (name: string) => { - const docCookie = document.cookie; - if (docCookie) { - var payload = docCookie.split('; ').find(row => row.startsWith(name)); - if (payload) { - return decodeURIComponent(payload.split('=')[1]) - } - } else { - return undefined; - } - } - - public removeCookie = (name: string) => { - document.cookie=name+"=;path=/;expires=Tue, 1 Jan 1980 23:59:59 GMT"; - } - - public disconnect = () { - this.removeCookie("__uid_2"); -} - - -} - -window.__uid2 = new UID2(); - - diff --git a/static/js/uid2-sdk-0.0.1a.js b/static/js/uid2-sdk-0.0.1a.js deleted file mode 100644 index c6920ad11..000000000 --- a/static/js/uid2-sdk-0.0.1a.js +++ /dev/null @@ -1,77 +0,0 @@ -class UID2 { - constructor() { - this.init = (opts) => { - const identity = opts["identity"]; - if (identity) { - this.setIdentity(identity); - } - else { - this.refreshIfNeeded(); - } - }; - this.refreshIfNeeded = () => { - const identity = this.getIdentity(); - if (identity) { - const url = "https://prod.uidapi.com/token/refresh?refresh_token=" + encodeURIComponent(identity["refresh_token"]); - const req = new XMLHttpRequest(); - req.overrideMimeType("application/json"); - var cb = this.handleRefreshResponse; - req.open("GET", url, false); - req.onload = function () { - cb(req.responseText); - }; - req.send(); - } - }; - this.handleRefreshResponse = (body) => { - this.setIdentity(body); - }; - this.getIdentity = () => { - const payload = this.getCookie("__uid_2"); - if (payload) { - return JSON.parse(payload); - } - }; - this.getAdvertisingToken = () => { - const identity = this.getIdentity(); - if (identity) { - return identity["advertisement_token"]; - } - }; - this.setIdentity = (value) => { - var payload; - if (typeof (value) === "object") { - payload = JSON.stringify(value); - } - else { - payload = value; - } - this.setCookie("__uid_2", payload); - }; - this.setCookie = (name, value) => { - var days = 7; - var date = new Date(); - date.setTime(date.getTime() + (days * 24 * 60 * 60 * 1000)); - document.cookie = name + "=" + encodeURIComponent(value) + " ;path=/;expires=" + date.toUTCString(); - }; - this.getCookie = (name) => { - const docCookie = document.cookie; - if (docCookie) { - var payload = docCookie.split('; ').find(row => row.startsWith(name)); - if (payload) { - return decodeURIComponent(payload.split('=')[1]); - } - } - else { - return undefined; - } - }; - this.removeCookie = (name) => { - document.cookie = name + "=;path=/;expires=Tue, 1 Jan 1980 23:59:59 GMT"; - }; - this.disconnect = () => { - this.removeCookie("__uid_2"); - }; - } -} -window.__uid2 = new UID2(); diff --git a/static/js/uid2-sdk-0.0.1b.js b/static/js/uid2-sdk-0.0.1b.js deleted file mode 100644 index 712704d91..000000000 --- a/static/js/uid2-sdk-0.0.1b.js +++ /dev/null @@ -1,98 +0,0 @@ -function __esp_getUID2Async(cb) { - return new Promise(function(cb) { - if (window.__uid2 && window.__uid2.getAdvertisingToken) { - cb(__uid2.getAdvertisingToken()); - } else { - throw new "UID2 SDK not present"; - } - }); -} - -if (typeof (googletag) !== "undefined" && googletag && googletag.encryptedSignalProviders) { - - googletag.encryptedSignalProviders.push({ - id: 'uidapi.com', - collectorFunction: () => { - return __esp_getUID2Async().then((signals) => signals); - } - }); - -} - -class UID2 { - constructor() { - this.init = (opts) => { - const identity = opts["identity"]; - if (identity) { - this.setIdentity(identity); - } - else { - this.refreshIfNeeded(); - } - }; - this.refreshIfNeeded = () => { - const identity = this.getIdentity(); - if (identity) { - const url = "https://prod.uidapi.com/token/refresh?refresh_token=" + encodeURIComponent(identity["refresh_token"]); - const req = new XMLHttpRequest(); - req.overrideMimeType("application/json"); - var cb = this.handleRefreshResponse; - req.open("GET", url, false); - req.onload = function () { - cb(req.responseText); - }; - req.send(); - } - }; - this.handleRefreshResponse = (body) => { - this.setIdentity(body); - }; - this.getIdentity = () => { - const payload = this.getCookie("__uid_2"); - if (payload) { - return JSON.parse(payload); - } - }; - this.getAdvertisingToken = () => { - const identity = this.getIdentity(); - if (identity) { - return identity["advertisement_token"]; - } - }; - this.setIdentity = (value) => { - var payload; - if (typeof (value) === "object") { - payload = JSON.stringify(value); - } - else { - payload = value; - } - this.setCookie("__uid_2", payload); - }; - this.setCookie = (name, value) => { - var days = 7; - var date = new Date(); - date.setTime(date.getTime() + (days * 24 * 60 * 60 * 1000)); - document.cookie = name + "=" + encodeURIComponent(value) + " ;path=/;expires=" + date.toUTCString(); - }; - this.getCookie = (name) => { - const docCookie = document.cookie; - if (docCookie) { - var payload = docCookie.split('; ').find(row => row.startsWith(name)); - if (payload) { - return decodeURIComponent(payload.split('=')[1]); - } - } - else { - return undefined; - } - }; - this.removeCookie = (name) => { - document.cookie = name + "=;path=/;expires=Tue, 1 Jan 1980 23:59:59 GMT"; - }; - this.disconnect = () => { - this.removeCookie("__uid_2"); - }; - } -} -window.__uid2 = new UID2(); diff --git a/static/js/uid2-sdk-1.0.0.js b/static/js/uid2-sdk-1.0.0.js deleted file mode 100644 index 542ab4447..000000000 --- a/static/js/uid2-sdk-1.0.0.js +++ /dev/null @@ -1,338 +0,0 @@ - -class UID2 { - static get VERSION() { - return "1.0.2"; - } - static get COOKIE_NAME() { - return "__uid_2"; - } - static get DEFAULT_REFRESH_RETRY_PERIOD_MS() { - return 5000; - } - - static setupGoogleTag() { - if (!window.googletag) { - window.googletag = {}; - } - if (!googletag.encryptedSignalProviders) { - googletag.encryptedSignalProviders = []; - } - googletag.encryptedSignalProviders.push({ - id: "uidapi.com", - collectorFunction: () => { - if (window.__uid2 && window.__uid2.getAdvertisingTokenAsync) { - return __uid2.getAdvertisingTokenAsync(); - } else { - return Promise.reject(new Error("UID2 SDK not present")); - } - }, - }); - } - - constructor() { - // PUBLIC METHODS - - this.init = (opts) => { - if (_initCalled) { - throw new TypeError('Calling init() more than once is not allowed'); - } - - if (typeof opts !== 'object' || opts === null) { - throw new TypeError('opts must be an object'); - } else if (typeof opts.callback !== 'function') { - throw new TypeError('opts.callback must be a function'); - } else if (typeof opts.refreshRetryPeriod !== 'undefined') { - if (typeof opts.refreshRetryPeriod !== 'number') - throw new TypeError('opts.refreshRetryPeriod must be a number'); - else if (opts.refreshRetryPeriod < 1000) - throw new RangeError('opts.refreshRetryPeriod must be >= 1000'); - } - - _initCalled = true; - _opts = opts; - applyIdentity(_opts.identity ? _opts.identity : loadIdentity()); - }; - this.getAdvertisingToken = () => { - return _identity && !temporarilyUnavailable() ? _identity.advertising_token : undefined; - }; - this.getAdvertisingTokenAsync = () => { - if (!initialised()) { - return new Promise((resolve, reject) => { - _promises.push({ resolve: resolve, reject: reject }); - }); - } else if (_identity) { - return temporarilyUnavailable() - ? Promise.reject(new Error('temporarily unavailable')) - : Promise.resolve(_identity.advertising_token); - } else { - return Promise.reject(new Error('identity not available')); - } - }; - this.isLoginRequired = () => { - return initialised() ? !_identity : undefined; - }; - this.disconnect = () => { - this.abort(); - removeCookie(UID2.COOKIE_NAME); - _identity = undefined; - _lastStatus = UID2.IdentityStatus.INVALID; - - const promises = _promises; - _promises = []; - promises.forEach(p => p.reject(new Error("disconnect()"))); - }; - this.abort = () => { - _initCalled = true; - if (typeof _refreshTimerId !== 'undefined') { - clearTimeout(_refreshTimerId); - _refreshTimerId = undefined; - } - if (_refreshReq) { - _refreshReq.abort(); - _refreshReq = undefined; - } - }; - - // PRIVATE STATE - - let _initCalled = false; - let _opts; - let _identity; - let _lastStatus; - let _refreshTimerId; - let _refreshReq; - let _promises = []; - - // PRIVATE METHODS - - const initialised = () => typeof _lastStatus !== 'undefined'; - const temporarilyUnavailable = () => _lastStatus === UID2.IdentityStatus.EXPIRED; - - const getOptionOrDefault = (value, defaultValue) => { - return typeof value === 'undefined' ? defaultValue : value; - }; - - const setCookie = (name, identity) => { - const value = JSON.stringify(identity); - const expires = new Date(identity.refresh_expires); - const path = getOptionOrDefault(_opts.cookiePath, "/"); - let cookie = name + "=" + encodeURIComponent(value) + " ;path=" + path + ";expires=" + expires.toUTCString(); - if (typeof _opts.cookieDomain !== 'undefined') { - cookie += ";domain=" + _opts.cookieDomain; - } - document.cookie = cookie; - }; - const removeCookie = (name) => { - document.cookie = name + "=;expires=Tue, 1 Jan 1980 23:59:59 GMT"; - }; - const getCookie = (name) => { - const docCookie = document.cookie; - if (docCookie) { - const payload = docCookie.split('; ').find(row => row.startsWith(name+'=')); - if (payload) { - return decodeURIComponent(payload.split('=')[1]); - } - } - }; - - const updateStatus = (status, statusText) => { - _lastStatus = status; - - const promises = _promises; - _promises = []; - - const advertisingToken = this.getAdvertisingToken(); - - const result = { - advertisingToken: advertisingToken, - advertising_token: advertisingToken, - status: status, - statusText: statusText - }; - _opts.callback(result); - - if (advertisingToken) { - promises.forEach(p => p.resolve(advertisingToken)); - } else { - promises.forEach(p => p.reject(new Error(statusText))); - } - }; - const setValidIdentity = (identity, status, statusText) => { - _identity = identity; - setCookie(UID2.COOKIE_NAME, identity); - setRefreshTimer(); - updateStatus(status, statusText); - }; - const setFailedIdentity = (status, statusText) => { - _identity = undefined; - this.abort(); - removeCookie(UID2.COOKIE_NAME); - updateStatus(status, statusText); - }; - const checkIdentity = (identity) => { - if (!identity.advertising_token) { - throw new InvalidIdentityError("advertising_token is not available or is not valid"); - } else if (!identity.refresh_token) { - throw new InvalidIdentityError("refresh_token is not available or is not valid"); - } - }; - const tryCheckIdentity = (identity) => { - try { - checkIdentity(identity); - return true; - } catch (err) { - if (err instanceof InvalidIdentityError) { - setFailedIdentity(UID2.IdentityStatus.INVALID, err.message); - return false; - } else { - throw err; - } - } - }; - const setIdentity = (identity, status, statusText) => { - if (tryCheckIdentity(identity)) { - setValidIdentity(identity, status, statusText); - } - }; - const loadIdentity = () => { - const payload = getCookie(UID2.COOKIE_NAME); - if (payload) { - return JSON.parse(payload); - } - }; - - const enrichIdentity = (identity, now) => { - return { - refresh_from: now, - refresh_expires: now + 7 * 86400 * 1000, // 7 days - identity_expires: now + 4 * 3600 * 1000, // 4 hours - ...identity, - }; - }; - const applyIdentity = (identity) => { - if (!identity) { - setFailedIdentity(UID2.IdentityStatus.NO_IDENTITY, "Identity not available"); - return; - } - - if (!tryCheckIdentity(identity)) { - // failed identity already set - return; - } - - const now = Date.now(); - identity = enrichIdentity(identity, now); - if (identity.refresh_expires < now) { - setFailedIdentity(UID2.IdentityStatus.REFRESH_EXPIRED, "Identity expired, refresh expired"); - return; - } - if (identity.refresh_from <= now) { - refreshToken(identity); - return; - } - - if (typeof _identity === 'undefined') { - setIdentity(identity, UID2.IdentityStatus.ESTABLISHED, "Identity established"); - } else if (identity.advertising_token !== _identity.advertising_token) { - // identity must have been refreshed from another tab - setIdentity(identity, UID2.IdentityStatus.REFRESH, "Identity refreshed"); - } else { - setRefreshTimer(); - } - } - const refreshToken = (identity) => { - const baseUrl = getOptionOrDefault(_opts.baseUrl, "https://prod.uidapi.com"); - const url = baseUrl + "/v1/token/refresh?refresh_token=" + encodeURIComponent(identity.refresh_token); - const req = new XMLHttpRequest(); - _refreshReq = req; - req.overrideMimeType("application/json"); - req.open("GET", url, true); - req.setRequestHeader('X-UID2-Client-Version', 'uid2-sdk-' + UID2.VERSION); - req.onreadystatechange = () => { - _refreshReq = undefined; - if (req.readyState !== req.DONE) return; - try { - const response = JSON.parse(req.responseText); - if (!checkResponseStatus(identity, response)) return; - checkIdentity(response.body); - setIdentity(response.body, UID2.IdentityStatus.REFRESHED, "Identity refreshed"); - } catch (err) { - handleRefreshFailure(identity, err.message); - } - }; - req.send(); - }; - const checkResponseStatus = (identity, response) => { - if (typeof response !== 'object' || response === null) { - throw new TypeError("refresh response is not an object"); - } - if (response.status === "optout") { - setFailedIdentity(UID2.IdentityStatus.OPTOUT, "User opted out"); - return false; - } else if (response.status === "expired_token") { - setFailedIdentity(UID2.IdentityStatus.REFRESH_EXPIRED, "Refresh token expired"); - return false; - } else if (response.status === "success") { - if (typeof response.body === 'object' && response.body !== null) { - return true; - } - throw new TypeError("refresh response object does not have a body"); - } else { - throw new TypeError("unexpected response status: " + response.status); - } - }; - const handleRefreshFailure = (identity, errorMessage) => { - const now = Date.now(); - if (identity.refresh_expires <= now) { - setFailedIdentity(UID2.IdentityStatus.REFRESH_EXPIRED, "Refresh expired; token refresh failed: " + errorMessage); - } else if (identity.identity_expires <= now && !temporarilyUnavailable()) { - setValidIdentity(identity, UID2.IdentityStatus.EXPIRED, "Token refresh failed for expired identity: " + errorMessage); - } else if (initialised()) { - setRefreshTimer(); // silently retry later - } else { - setIdentity(identity, UID2.IdentityStatus.ESTABLISHED, "Identity established; token refresh failed: " + errorMessage) - } - }; - const setRefreshTimer = () => { - const timeout = getOptionOrDefault(_opts.refreshRetryPeriod, UID2.DEFAULT_REFRESH_RETRY_PERIOD_MS); - _refreshTimerId = setTimeout(() => { - if (this.isLoginRequired()) return; - applyIdentity(loadIdentity()); - }, timeout); - }; - - // PRIVATE ERRORS - - class InvalidIdentityError extends Error { - constructor(message) { - super(message); - this.name = "InvalidIdentityError"; - } - } - } -} - -(function (UID2) { - let IdentityStatus; // enum - (function (IdentityStatus) { - // identity available - IdentityStatus[IdentityStatus["ESTABLISHED"] = 0] = "ESTABLISHED"; - IdentityStatus[IdentityStatus["REFRESHED"] = 1] = "REFRESHED"; - // identity temporarily not available - IdentityStatus[IdentityStatus["EXPIRED"] = 100] = "EXPIRED"; - // identity not available - IdentityStatus[IdentityStatus["NO_IDENTITY"] = -1] = "NO_IDENTITY"; - IdentityStatus[IdentityStatus["INVALID"] = -2] = "INVALID"; - IdentityStatus[IdentityStatus["REFRESH_EXPIRED"] = -3] = "REFRESH_EXPIRED"; - IdentityStatus[IdentityStatus["OPTOUT"] = -4] = "OPTOUT"; - })(IdentityStatus = UID2.IdentityStatus || (UID2.IdentityStatus = {})); -})(UID2 || (UID2 = {})); - -window.__uid2 = new UID2(); - -UID2.setupGoogleTag(); - -if (typeof exports !== 'undefined') { - exports.UID2 = UID2; - exports.window = window; -} diff --git a/version.json b/version.json index 16e102dfa..0deb4c095 100644 --- a/version.json +++ b/version.json @@ -1 +1 @@ -{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.40", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } } +{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.56", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } }