feat: Add legacy logging with pilot, diracx

Author: Robin-Van-de-Merghel

.github/workflows/integration.yml

@@ -38,7 +38,7 @@ jobs:
           - TEST_NAME: "Backward Compatibility"
             ARGS: CLIENT_INSTALLATION_BRANCH=rel-v8r0 PILOT_INSTALLATION_BRANCH=rel-v8r0
           - TEST_NAME: "Test DiracX latest"
-            ARGS: TEST_DIRACX=Yes
+            ARGS: TEST_DIRACX=Yes --diracx-dist-dir $GITHUB_WORKSPACE/diracx-dist
 
     steps:
     - uses: actions/checkout@v4
@@ -57,7 +57,22 @@ jobs:
             packaging \
             pyyaml \
             requests \
-            typer
+            typer \
+            build
+    - name: Building wheels
+      run: |
+        # Clone diracx
+        git clone https://github.com/Robin-Van-de-Merghel/diracx.git $GITHUB_WORKSPACE/diracx
+
+        # Create dist dir
+        mkdir -p $GITHUB_WORKSPACE/diracx-dist
+
+        # Building diracx
+        for pkg_dir in $GITHUB_WORKSPACE/diracx/diracx-* $GITHUB_WORKSPACE/diracx; do
+          echo "Building $pkg_dir"
+          python -m build --outdir "$GITHUB_WORKSPACE/diracx-dist" $pkg_dir
+        done
+
     - name: Prepare environment
       run: ./integration_tests.py prepare-environment ${{ matrix.ARGS }}
     - name: Install server

tests/.dirac-ci-config.yaml

@@ -4,7 +4,7 @@ config:
   CLIENT_UPLOAD_BASE64: SSBsaWtlIHBpenphIQo=
   CLIENT_UPLOAD_LFN: LFN:/vo/test_lfn.txt
   CLIENT_UPLOAD_FILE: test_lfn.txt
-  PILOT_INSTALLATION_COMMAND: dirac-pilot.py --modules /home/dirac/LocalRepo/ALTERNATIVE_MODULES/DIRAC -M 2 -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --pilotUUID=whatever12345 --CVMFS_locations=/home/dirac/ -o diracInstallOnly --wnVO=vo --debug
+  PILOT_INSTALLATION_COMMAND: dirac-pilot.py --modules /home/dirac/LocalRepo/ALTERNATIVE_MODULES/DIRAC -M 2 -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --pilotUUID=whatever12345 --CVMFS_locations=/home/dirac/ -o diracInstallOnly --wnVO=vo --debug --diracx_URL=http://diracx:8000/ -z
   PILOT_JSON: "{
         \"timestamp\": \"2023-02-13T14:34:26.725499\",
         \"CEs\": {
@@ -37,7 +37,7 @@ config:
                 \"https://server:9135/Configuration/Server\"
         ]
 }"
-  PILOT_DOWNLOAD_COMMAND: "git clone --single-branch --branch master https://github.com/DIRACGrid/Pilot.git && mv Pilot/Pilot/*.py . && rm -rf Pilot"
+  PILOT_DOWNLOAD_COMMAND: "git clone --single-branch --branch robin-add-pilot-logging https://github.com/Robin-Van-de-Merghel/Pilot.git && mv Pilot/Pilot/*.py . && rm -rf Pilot"
 
 # List of feature variables which must be passed when preparing
 required-feature-flags: []

tests/CI/docker-compose.yml

@@ -269,7 +269,6 @@ services:
       - DIRACX_DB_URL_JOBLOGGINGDB=mysql+aiomysql://Dirac:Dirac@mysql/JobLoggingDB
       - DIRACX_DB_URL_SANDBOXMETADATADB=mysql+aiomysql://Dirac:Dirac@mysql/SandboxMetadataDB
       - DIRACX_DB_URL_PILOTAGENTSDB=mysql+aiomysql://Dirac:Dirac@mysql/PilotAgentsDB
-      - 'DIRACX_OS_DB_PILOTLOGSDB={"sqlalchemy_dsn": "mysql+aiomysql://Dirac:Dirac@mysql/PilotLogsDB"}'
       - DIRACX_DB_URL_TASKQUEUEDB=mysql+aiomysql://Dirac:Dirac@mysql/TaskQueueDB
       - DIRACX_SERVICE_AUTH_TOKEN_KEYSTORE=file:///keystore/jwks.json
       - DIRACX_SERVICE_AUTH_TOKEN_ISSUER=http://diracx:8000
@@ -278,11 +277,11 @@ services:
       - DIRACX_SERVICE_AUTH_STATE_KEY=uSNPPtZ1EbC5np13zOwmWJ84Duix753Hejzk/u/MQE4=
       # Obtained with echo 'InsecureChangeMe' | base64 -d | openssl sha256
       - DIRACX_LEGACY_EXCHANGE_HASHED_API_KEY=07cddf6948d316ac9d186544dc3120c4c6697d8f994619665985c0a5bf76265a
+      - 'DIRACX_OS_DB_PILOTLOGSDB={"hosts": "elastic:changeme@opensearch:9200", "use_ssl": false, "verify_certs": false}'
       - 'DIRACX_OS_DB_JOBPARAMETERSDB={"hosts": "elastic:changeme@opensearch:9200", "use_ssl": false, "verify_certs": false}'
       - DIRACX_SANDBOX_STORE_BUCKET_NAME=sandboxes
       - DIRACX_SANDBOX_STORE_AUTO_CREATE_BUCKET=true
       - 'DIRACX_SANDBOX_STORE_S3_CLIENT_KWARGS={"endpoint_url": "http://s3-direct:9090", "aws_access_key_id": "console", "aws_secret_access_key": "console123"}'
-
     ports:
       - 8000:8000
     depends_on:

tests/CI/run_pilot.sh

@@ -29,6 +29,8 @@ touch /home/dirac/etc/grid-security/vomses/vomses
 # Copy over the pilot proxy
 cp /ca/certs/pilot_proxy /tmp/x509up_u$UID
 
+export X509_USER_PROXY=/tmp/x509up_u$UID
+
 eval "${PILOT_DOWNLOAD_COMMAND}"
 
 echo "${PILOT_JSON}" > pilot.json