From 20226bc89552b288c252fd9632c97dbae869f355 Mon Sep 17 00:00:00 2001 From: Rob Bos Date: Mon, 27 May 2024 17:51:01 +0000 Subject: [PATCH 1/3] setup --- .devcontainer/Dockerfile | 25 +++++++++ .devcontainer/devcontainer-add.json | 13 +++++ .devcontainer/devcontainer.json | 45 ++++++++++++++++ .devcontainer/postCreateCommand.ps1 | 1 + .gitignore | 2 + .vscode/launch.json | 2 +- .../GitHubActionsNews/Program.cs | 46 ++++++++++------ .../GitHubActionsNews/appsettings.json | 10 ++-- devcontainer.json | 52 ------------------- 9 files changed, 121 insertions(+), 75 deletions(-) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer-add.json create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/postCreateCommand.ps1 delete mode 100644 devcontainer.json diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..af269a2 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,25 @@ +FROM mcr.microsoft.com/devcontainers/base:focal + +# Install Docker +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install --no-install-recommends apt-transport-https ca-certificates curl software-properties-common gnupg2 \ + && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \ + && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \ + && apt-get update \ + && apt-get -y install --no-install-recommends docker-ce docker-ce-cli containerd.io + +# Install Node.js using apt-get +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install --no-install-recommends nodejs npm + +# # Install Google Chrome +RUN wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ + dpkg -i google-chrome-stable_current_amd64.deb || true \ + && apt-get install -yf \ + && rm google-chrome-stable_current_amd64.deb + +# # Install ChromeDriver +# RUN wget -N http://chromedriver.storage.googleapis.com/$(curl -sS chromedriver.storage.googleapis.com/LATEST_RELEASE)/chromedriver_linux64.zip \ +# && unzip chromedriver_linux64.zip -d /usr/local/bin \ +# && rm chromedriver_linux64.zip \ +# && chmod +x /usr/local/bin/chromedriver \ No newline at end of file diff --git a/.devcontainer/devcontainer-add.json b/.devcontainer/devcontainer-add.json new file mode 100644 index 0000000..ab926e4 --- /dev/null +++ b/.devcontainer/devcontainer-add.json @@ -0,0 +1,13 @@ +# start of a Dockerfile for this codespace + +# Install Google Chrome +wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb +sudo dpkg -i google-chrome-stable_current_amd64.deb +sudo apt-get install -f + +# Install ChromeDriver +wget https://chromedriver.storage.googleapis.com/2.41/chromedriver_linux64.zip +unzip chromedriver_linux64.zip +sudo mv chromedriver /usr/bin/chromedriver +sudo chown root:root /usr/bin/chromedriver +sudo chmod +x /usr/bin/chromedriver \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..6ad70d5 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,45 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/ubuntu +{ + "name": "devcontainer", + // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile + "dockerFile": "Dockerfile", + "features": { + "ghcr.io/devcontainers/features/github-cli:1": {}, + "ghcr.io/devcontainers/features/azure-cli:1": {}, + "ghcr.io/devcontainers-contrib/features/powershell:1": {}, + "ghcr.io/devcontainers/features/node:1": { + "version": "lts" + } + }, + "customizations": { + "vscode": { + "extensions": [ + "GitHub.copilot", + "ms-vscode.azurecli", + "ms-vscode.PowerShell" + ], + "settings": { + "terminal.integrated.shell.linux": "/usr/local/lib/pwsh", + "powershell.powerShellAdditionalExePaths": { + "pwsh": "/usr/local/lib/pwsh/pwsh" + }, + "editor.trimAutoWhitespace": true, + "files.trimTrailingWhitespace": true + } + } + }, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "pwsh -Command { ./.devcontainer/postCreateCommand.ps1; }", + + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "root" + + // Run the dev container as privileged to be able to run docker commands + "runArgs": ["--init", "--privileged"], + "overrideCommand": false +} diff --git a/.devcontainer/postCreateCommand.ps1 b/.devcontainer/postCreateCommand.ps1 new file mode 100644 index 0000000..fa99219 --- /dev/null +++ b/.devcontainer/postCreateCommand.ps1 @@ -0,0 +1 @@ +echo "Hello devcontainer" \ No newline at end of file diff --git a/.gitignore b/.gitignore index 93b7dfd..820a364 100644 --- a/.gitignore +++ b/.gitignore @@ -331,3 +331,5 @@ ASALocalRun/ # Custom files to ignore *.secrets.json + +google-chrome-stable_current_amd64.deb diff --git a/.vscode/launch.json b/.vscode/launch.json index 03068c0..efaa277 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -37,7 +37,7 @@ "preLaunchTask": "build", // If you have changed target frameworks, make sure to update the program path. "program": "${workspaceFolder}/AzDoExtensionNews/GitHubActionsNews/bin/Debug/net7.0/GitHubActionsNews.dll", - "args": ["apprise"], + "args": ["test"], "cwd": "${workspaceFolder}/AzDoExtensionNews/AzDoExtensionNews", // For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console "console": "internalConsole", diff --git a/AzDoExtensionNews/GitHubActionsNews/Program.cs b/AzDoExtensionNews/GitHubActionsNews/Program.cs index 9ad7de5..618b27e 100644 --- a/AzDoExtensionNews/GitHubActionsNews/Program.cs +++ b/AzDoExtensionNews/GitHubActionsNews/Program.cs @@ -26,9 +26,10 @@ static async Task Main(string[] args) { Log.Message("Please add a parameter to the run:"); Log.Message(" all = run through each action result in the list"); - Log.Message(" One or more comma separated letters = run through each action result that matches the search string"); + Log.Message(" one or more comma separated letters = run through each action result that matches the search string"); Log.Message(" consolidate = download all previous result files and consolidate to 1 file"); Log.Message(" verify = verify all actions in the storage account for overlap"); + Log.Message(" test = run for a single test action to debug"); } Configuration.LoadSettings(); @@ -64,10 +65,21 @@ private static void RunTest() var driver = GetDriver(); try { - driver.Navigate().GoToUrl("https://github.com/marketplace/actions/glo-parse-card-links"); - var version = ActionPageInteraction.GetVersionFromAction(driver); - var url = ActionPageInteraction.GetRepoFromAction(driver); - Log.Message($"Found version [{version}] and url [${url}]"); + // configure for testing either a single action or a search page + var runSingleActionTest = false; + if (runSingleActionTest) { + // run for a single action page + driver.Navigate().GoToUrl("https://github.com/marketplace/actions/glo-parse-card-links"); + var version = ActionPageInteraction.GetVersionFromAction(driver); + var url = ActionPageInteraction.GetRepoFromAction(driver); + Log.Message($"Found version [{version}] and url [${url}]"); + } + else { + // run fo a search page + var twoLetterQuery = "ca"; + var queriedGitHubMarketplaceUrl = $"{GitHubMarketplaceUrl}&query={twoLetterQuery}"; + var actions = GetAllActions(queriedGitHubMarketplaceUrl); + } } finally { @@ -271,10 +283,9 @@ private static List ScrapeGitHubMarketPlace(string searchUrl) private static ChromeDriver GetDriver() { var chromeOptions = new ChromeOptions(); - if (!Debugger.IsAttached) - { - chromeOptions.AddArguments("headless"); - } + chromeOptions.AddArguments("headless"); // Run Chrome in headless mode + chromeOptions.AddArguments("--no-sandbox"); // Bypass OS security model + chromeOptions.AddArguments("--disable-dev-shm-usage"); // Overcome limited resource problems var driver = new ChromeDriver(chromeOptions); return driver; } @@ -350,12 +361,13 @@ private static WebDriverWait ScrollPaginatorIntoView(IWebDriver driver) var actions = new Actions(driver); var waitForElement = new OpenQA.Selenium.Support.UI.WebDriverWait(driver, TimeSpan.FromSeconds(1)); + var elementName = "TablePaginationSteps"; try { - waitForElement.Until(ExpectedConditions.ElementExists(By.ClassName("paginate-container"))); - waitForElement.Until(ExpectedConditions.ElementIsVisible(By.ClassName("paginate-container"))); - waitForElement.Until(ExpectedConditions.ElementToBeClickable(By.ClassName("paginate-container"))); - var paginator = driver.FindElement(By.ClassName("paginate-container")); + waitForElement.Until(ExpectedConditions.ElementExists(By.ClassName(elementName))); + waitForElement.Until(ExpectedConditions.ElementIsVisible(By.ClassName(elementName))); + waitForElement.Until(ExpectedConditions.ElementToBeClickable(By.ClassName(elementName))); + var paginator = driver.FindElement(By.ClassName(elementName)); actions.MoveToElement(paginator); actions.Perform(); } @@ -363,10 +375,10 @@ private static WebDriverWait ScrollPaginatorIntoView(IWebDriver driver) { // wait some time and retry Thread.Sleep(1000); - waitForElement.Until(ExpectedConditions.ElementExists(By.ClassName("paginate-container"))); - waitForElement.Until(ExpectedConditions.ElementIsVisible(By.ClassName("paginate-container"))); - waitForElement.Until(ExpectedConditions.ElementToBeClickable(By.ClassName("paginate-container"))); - var paginator = driver.FindElement(By.ClassName("paginate-container")); + waitForElement.Until(ExpectedConditions.ElementExists(By.ClassName(elementName))); + waitForElement.Until(ExpectedConditions.ElementIsVisible(By.ClassName(elementName))); + waitForElement.Until(ExpectedConditions.ElementToBeClickable(By.ClassName(elementName))); + var paginator = driver.FindElement(By.ClassName(elementName)); actions.MoveToElement(paginator); actions.Perform(); } diff --git a/AzDoExtensionNews/GitHubActionsNews/appsettings.json b/AzDoExtensionNews/GitHubActionsNews/appsettings.json index b4bfecb..385b25a 100644 --- a/AzDoExtensionNews/GitHubActionsNews/appsettings.json +++ b/AzDoExtensionNews/GitHubActionsNews/appsettings.json @@ -1,8 +1,8 @@ { "name": "AzDoExtensionsNews", - "TwitterConsumerAPIKey": "", - "TwitterConsumerAPISecretKey": "", - "TwitterAccessToken": "", - "TwitterAccessTokenSecret": "", - "BlobStorageConnectionString": "" + "TwitterConsumerAPIKey": "TwitterConsumerAPIKey", + "TwitterConsumerAPISecretKey": "TwitterConsumerAPISecretKey", + "TwitterAccessToken": "TwitterAccessToken", + "TwitterAccessTokenSecret": "TwitterAccessTokenSecret", + "BlobStorageConnectionString": "BlobStorageConnectionString" } \ No newline at end of file diff --git a/devcontainer.json b/devcontainer.json deleted file mode 100644 index f6d6935..0000000 --- a/devcontainer.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "name": "C# (.NET Core)", - "build": { - "dockerfile": "Dockerfile", - "args": { - // Update 'VARIANT' to pick a .NET Core version. Rebuild the container if - // it already exists to update. Example variants: 2.1-bionic, 3.1-bionic - "VARIANT": "3.1-bionic", - // Options - "INSTALL_NODE": "true", - "NODE_VERSION": "lts/*", - "INSTALL_AZURE_CLI": "false", - "UPGRADE_PACKAGES": "false" - } - }, - - // Set *default* container specific settings.json values on container create. - "settings": { - "terminal.integrated.shell.linux": "/bin/bash" - }, - - // Add the IDs of extensions you want installed when the container is created. - "extensions": [ - "ms-dotnettools.csharp" - ], - - // Use 'forwardPorts' to make a list of ports inside the container available locally. - // "forwardPorts": [5000, 5001], - - // [Optional] To reuse of your local HTTPS dev cert, first export it locally using this command: - // * Windows PowerShell: - // dotnet dev-certs https --trust; dotnet dev-certs https -ep "$env:USERPROFILE/.aspnet/https/aspnetapp.pfx" -p "SecurePwdGoesHere" - // * macOS/Linux terminal: - // dotnet dev-certs https --trust; dotnet dev-certs https -ep "${HOME}/.aspnet/https/aspnetapp.pfx" -p "SecurePwdGoesHere" - // - // Next, after running the command above, uncomment lines in the 'mounts' and 'remoteEnv' lines below, - // and open / rebuild the container so the settings take effect. - // - "mounts": [ - // "source=${env:HOME}${env:USERPROFILE}/.aspnet/https,target=/home/vscode/.aspnet/https,type=bind" - ], - "remoteEnv": { - // "ASPNETCORE_Kestrel__Certificates__Default__Password": "SecurePwdGoesHere", - // "ASPNETCORE_Kestrel__Certificates__Default__Path": "/home/vscode/.aspnet/https/aspnetapp.pfx", - } - - // Use 'postCreateCommand' to run commands after the container is created. - // "postCreateCommand": "dotnet restore", - - // Uncomment to connect as a non-root user. See https://aka.ms/vscode-remote/containers/non-root. - // "remoteUser": "vscode" -} From b277a1f0a93780ff67e93c34619f3755deb4d72b Mon Sep 17 00:00:00 2001 From: Rob Bos Date: Mon, 27 May 2024 20:07:03 +0200 Subject: [PATCH 2/3] new config --- .devcontainer/Dockerfile | 8 ++++---- .devcontainer/devcontainer.json | 6 +++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index af269a2..b153cfe 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -19,7 +19,7 @@ RUN wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.d && rm google-chrome-stable_current_amd64.deb # # Install ChromeDriver -# RUN wget -N http://chromedriver.storage.googleapis.com/$(curl -sS chromedriver.storage.googleapis.com/LATEST_RELEASE)/chromedriver_linux64.zip \ -# && unzip chromedriver_linux64.zip -d /usr/local/bin \ -# && rm chromedriver_linux64.zip \ -# && chmod +x /usr/local/bin/chromedriver \ No newline at end of file +RUN wget -N http://chromedriver.storage.googleapis.com/$(curl -sS chromedriver.storage.googleapis.com/LATEST_RELEASE)/chromedriver_linux64.zip \ + && unzip chromedriver_linux64.zip -d /usr/local/bin \ + && rm chromedriver_linux64.zip \ + && chmod +x /usr/local/bin/chromedriver \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 6ad70d5..b6def37 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -3,7 +3,11 @@ { "name": "devcontainer", // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile - "dockerFile": "Dockerfile", + //"dockerFile": "Dockerfile", + "build": { + // Path is relative to the devcontainer.json file. + "dockerfile": "Dockerfile" + }, "features": { "ghcr.io/devcontainers/features/github-cli:1": {}, "ghcr.io/devcontainers/features/azure-cli:1": {}, From 0d10a02d91e7510b92f5d3b569d781b2d9bff677 Mon Sep 17 00:00:00 2001 From: Rob Bos Date: Thu, 20 Jun 2024 06:48:07 +0000 Subject: [PATCH 3/3] start of running in a codespace (still fails) --- .devcontainer/Dockerfile | 7 +++---- .devcontainer/devcontainer.json | 12 ++++++------ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index b153cfe..65eb9cf 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -12,10 +12,9 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ && apt-get -y install --no-install-recommends nodejs npm -# # Install Google Chrome -RUN wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ - dpkg -i google-chrome-stable_current_amd64.deb || true \ - && apt-get install -yf \ +# Install Google Chrome +RUN wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb \ + && apt-get install -y ./google-chrome-stable_current_amd64.deb \ && rm google-chrome-stable_current_amd64.deb # # Install ChromeDriver diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b6def37..5a5a1e9 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -9,12 +9,12 @@ "dockerfile": "Dockerfile" }, "features": { - "ghcr.io/devcontainers/features/github-cli:1": {}, - "ghcr.io/devcontainers/features/azure-cli:1": {}, - "ghcr.io/devcontainers-contrib/features/powershell:1": {}, - "ghcr.io/devcontainers/features/node:1": { - "version": "lts" - } + // "ghcr.io/devcontainers/features/github-cli:1": {}, + // "ghcr.io/devcontainers/features/azure-cli:1": {}, + // "ghcr.io/devcontainers-contrib/features/powershell:1": {}, + // "ghcr.io/devcontainers/features/node:1": { + // "version": "lts" + // } }, "customizations": { "vscode": {