Added test data for integration tests

Author: NJLangley

.github/workflows/test_azure_devtest_labs_integration.yml

@@ -36,6 +36,14 @@ jobs:
     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
     - uses: actions/checkout@v2
 
+    - name: Find test files
+      run: |
+        find ./DataPipelineTools.Functions.Tests/TestData -type f -name ".keepFolder"|while IFS=/ read filename; do
+          echo "$filename"
+          relativePath=$(echo "$filename" | sed -r 's/\.\/DataPipelineTools\.Functions\.Tests//g')
+          echo "$relativePath"
+        done
+
 
     # Build and run the unit tests
     - name: Setup .NET
@@ -160,6 +168,7 @@ jobs:
           echo "Deployment Outputs"
           #echo "::set-output name=STORAGE_ACCOUNTCONNECTION_STRING::$(echo $DEPLOYMENTOUTPUT | jq --raw-output '.storageAccountConnectionString.value')"
           echo "::set-output name=STORAGE_ACCOUNT_NAME::$(echo $DEPLOYMENTOUTPUT | jq --raw-output '.storageAccountName.value')"
+          echo "::set-output name=STORAGE_CONTAINER_NAME::$(echo $DEPLOYMENTOUTPUT | jq --raw-output '.storageContainerName.value')"
           echo "::set-output name=FUNCTIONS_APP_NAME::$(echo $DEPLOYMENTOUTPUT | jq --raw-output '.functionsAppName.value')"
           echo "::set-output name=FUNCTIONS_APP_URI::$(echo $DEPLOYMENTOUTPUT | jq --raw-output '.functionsAppUri.value')"
           echo "::set-output name=KEY_VAULT_NAME::$(echo $DEPLOYMENTOUTPUT | jq --raw-output '.keyVaultName.value')"
@@ -180,6 +189,7 @@ jobs:
         #echo "FUNCTIONS_APP_KEY:                        ${{ steps.create-devtest-labs-environment.outputs.FUNCTIONS_APP_KEY }}"
         #echo "STORAGE_ACCOUNTCONNECTION_STRING:         ${{ steps.create-devtest-labs-environment.outputs.STORAGE_ACCOUNTCONNECTION_STRING }}"
         echo "STORAGE_ACCOUNT_NAME:                     ${{ steps.create-devtest-labs-environment.outputs.STORAGE_ACCOUNT_NAME }}"
+        echo "STORAGE_CONTAINER_NAME:                   ${{ steps.create-devtest-labs-environment.outputs.STORAGE_CONTAINER_NAME }}"
         echo "KEY_VAULT_NAME:                           ${{ steps.create-devtest-labs-environment.outputs.KEY_VAULT_NAME }}"
         echo 'RUN_SETTINGS:                             ${{ steps.create-devtest-labs-environment.outputs.RUN_SETTINGS }}'
     
@@ -229,11 +239,22 @@ jobs:
       uses: kheiakiyama/[email protected]
       with:
         version: 'v10'
+        creds: '${{ secrets.AZURE_DEV_TEST_LABS_CREDENTIALS }}'
 
     - name: Copy files to Azure Data Lake using AZCopy
       run: |
-        ls -ltA
-#azcopy_v10 --source {SOURCE} --destination {DEST} --dest-key ${{ secrets.STORAGE_KEY }} --recursive --set-content-type
+
+        STORAGE_ACCOUNT_NAME="${{ steps.create-devtest-labs-environment.outputs.STORAGE_ACCOUNT_NAME }}"
+        STORAGE_CONTAINER_NAME="${{ steps.create-devtest-labs-environment.outputs.STORAGE_CONTAINER_NAME }}"
+        # The exclude patter for the .keepDirectory files is not working, so workaround below if to find these files and remove them from ADLS
+        azcopy copy "./DataPipelineTools.Functions.Tests/TestData" "https://$STORAGE_ACCOUNT_NAME.dfs.core.windows.net/$STORAGE_CONTAINER_NAME" --recursive=true --exclude-pattern=".keepDirectory"
+
+        # Remove the .keepDirectory files, these are just used to allow us to add empty folders into the sample data in Git
+        find ./DataPipelineTools.Functions.Tests/TestData -type f -name ".keepFolder"|while IFS=/ read FILENAME; do
+          RELATIVE_PATH=$(echo "$FILENAME" | sed -r 's/\.\/DataPipelineTools\.Functions\.Tests\///g')
+          azcopy rm "https://$STORAGE_ACCOUNT_NAME.dfs.core.windows.net/$STORAGE_CONTAINER_NAME/$RELATIVE_PATH"
+        done
+
     
     - name: 'Deploy Azure Functions to Lab Environment using RBAC'
       uses: Azure/functions-action@v1

Azure-DevTestLab/Environments/sqlcollaborative_AzureDataPipelineTools/azuredeploy.json

@@ -537,6 +537,11 @@
             "type": "string",
             "value": "[variables('adlsStorageAccountName')]"
         },
+        "StorageContainerName": {
+            "type": "string",
+            "value": "[parameters('adlsStorageAccountContainerName')]"
+        },
+
         "keyVaultName": {
             "type": "string",
             "value": "[variables('keyVaultName')]"