From 9b4921301d540087d62c2149e50f8225591738c6 Mon Sep 17 00:00:00 2001 From: czarte Date: Tue, 20 May 2025 16:29:41 +0200 Subject: [PATCH 01/12] added dbip_download github workflow --- .github/workflows/dbip_download.yml | 52 +++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .github/workflows/dbip_download.yml diff --git a/.github/workflows/dbip_download.yml b/.github/workflows/dbip_download.yml new file mode 100644 index 000000000..eec549b9b --- /dev/null +++ b/.github/workflows/dbip_download.yml @@ -0,0 +1,52 @@ +name: Download DBIP data and generate dbip_country.rs + +on: + schedule: + - cron: "0 0 3 * *" # Runs at midnight on the 3rd of every month + +env: + TEMP_DIR: '/tmp' + YEAR_MONTH: '01-9999' + +permissions: + contents: write + +jobs: + generate: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + + - name: Download DBIP data in MMDB format + run: | + cd ip_country + export YEAR_MONTH=$(date +%Y-%m) + echo "TEMP_DIR=$(mktemp -d)" >> $GITHUB_ENV + echo "YEAR_MONTH=$YEAR_MONTH" >> $GITHUB_ENV + mkdir -p dbip-data + curl -L -o dbip-data/dbip-country-lite.mmdb.gz "https://download.db-ip.com/free/dbip-country-lite-$YEAR_MONTH.mmdb.gz" + gunzip dbip-data/dbip-country-lite.mmdb.gz + + - name: Generate Rust source file + run: | + cd ip_country + cargo run < "dbip-data/dbip-country-lite.mmdb" > "$TEMP_DIR"/dbip_country.rs + ls "$TEMP_DIR" + + - name: Commit and push generated file + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git checkout -B generated-source + git rm -rf . + mkdir ip_country/src + mv "${TEMP_DIR}"/dbip_country.rs ip_country/src/dbip_country.rs + git add ip_country/src/dbip_country.rs + git commit -m "Update generated dbip_country ${YEAR_MONTH} Rust source file" + git push -u origin HEAD \ No newline at end of file From 48403a7db903a5d235ce42132660965f0c287316 Mon Sep 17 00:00:00 2001 From: czarte Date: Wed, 21 May 2025 13:54:13 +0200 Subject: [PATCH 02/12] change to run every day on midnight for QA --- .github/workflows/dbip_download.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dbip_download.yml b/.github/workflows/dbip_download.yml index eec549b9b..b6a55977a 100644 --- a/.github/workflows/dbip_download.yml +++ b/.github/workflows/dbip_download.yml @@ -2,7 +2,7 @@ name: Download DBIP data and generate dbip_country.rs on: schedule: - - cron: "0 0 3 * *" # Runs at midnight on the 3rd of every month + - cron: "0 0 * * *" # Runs at midnight on the 3rd of every month env: TEMP_DIR: '/tmp' From 285a3dfe9890230c030737f63fe74be6605750b3 Mon Sep 17 00:00:00 2001 From: czarte Date: Thu, 22 May 2025 12:09:43 +0200 Subject: [PATCH 03/12] necesary changes to be able build dbip_country.rs on master --- .github/workflows/dbip_download.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dbip_download.yml b/.github/workflows/dbip_download.yml index b6a55977a..ddff4f01e 100644 --- a/.github/workflows/dbip_download.yml +++ b/.github/workflows/dbip_download.yml @@ -25,17 +25,18 @@ jobs: - name: Download DBIP data in MMDB format run: | - cd ip_country export YEAR_MONTH=$(date +%Y-%m) echo "TEMP_DIR=$(mktemp -d)" >> $GITHUB_ENV echo "YEAR_MONTH=$YEAR_MONTH" >> $GITHUB_ENV + git clone https://github.com/MASQ-Project/DBIP-Experiment.git dbipexperiment + cd dbipexperiment/ip_country mkdir -p dbip-data curl -L -o dbip-data/dbip-country-lite.mmdb.gz "https://download.db-ip.com/free/dbip-country-lite-$YEAR_MONTH.mmdb.gz" gunzip dbip-data/dbip-country-lite.mmdb.gz - name: Generate Rust source file run: | - cd ip_country + cd dbipexperiment/ip_country cargo run < "dbip-data/dbip-country-lite.mmdb" > "$TEMP_DIR"/dbip_country.rs ls "$TEMP_DIR" @@ -45,6 +46,7 @@ jobs: git config user.email "github-actions[bot]@users.noreply.github.com" git checkout -B generated-source git rm -rf . + mkdir ip_country mkdir ip_country/src mv "${TEMP_DIR}"/dbip_country.rs ip_country/src/dbip_country.rs git add ip_country/src/dbip_country.rs From f7363a754acdc3dbe0b2626d5a8a997530b646b7 Mon Sep 17 00:00:00 2001 From: czarte Date: Thu, 22 May 2025 12:50:02 +0200 Subject: [PATCH 04/12] -p for generating ip_country/src --- .github/workflows/dbip_download.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/dbip_download.yml b/.github/workflows/dbip_download.yml index ddff4f01e..2d7f27435 100644 --- a/.github/workflows/dbip_download.yml +++ b/.github/workflows/dbip_download.yml @@ -46,8 +46,7 @@ jobs: git config user.email "github-actions[bot]@users.noreply.github.com" git checkout -B generated-source git rm -rf . - mkdir ip_country - mkdir ip_country/src + mkdir -p ip_country/src mv "${TEMP_DIR}"/dbip_country.rs ip_country/src/dbip_country.rs git add ip_country/src/dbip_country.rs git commit -m "Update generated dbip_country ${YEAR_MONTH} Rust source file" From 99873a586469d2175374ac331fbfc4fc8516427c Mon Sep 17 00:00:00 2001 From: czarte Date: Mon, 26 May 2025 13:02:11 +0200 Subject: [PATCH 05/12] todo in dbip_download.yaml --- .github/workflows/dbip_download.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/dbip_download.yml b/.github/workflows/dbip_download.yml index 2d7f27435..7d477351a 100644 --- a/.github/workflows/dbip_download.yml +++ b/.github/workflows/dbip_download.yml @@ -46,6 +46,7 @@ jobs: git config user.email "github-actions[bot]@users.noreply.github.com" git checkout -B generated-source git rm -rf . +# make conditional checkout -b if branch do not exists and switch if exists mkdir -p ip_country/src mv "${TEMP_DIR}"/dbip_country.rs ip_country/src/dbip_country.rs git add ip_country/src/dbip_country.rs From 047c388b794d62b35676b37c06bb490531d07f21 Mon Sep 17 00:00:00 2001 From: czarte Date: Tue, 27 May 2025 13:14:48 +0200 Subject: [PATCH 06/12] condition for new branch generated-source --- .github/workflows/dbip_download.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dbip_download.yml b/.github/workflows/dbip_download.yml index 7d477351a..964b18552 100644 --- a/.github/workflows/dbip_download.yml +++ b/.github/workflows/dbip_download.yml @@ -42,12 +42,18 @@ jobs: - name: Commit and push generated file run: | + git ls-remote --heads | grep generated-source + BRANCH=$? git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - git checkout -B generated-source - git rm -rf . -# make conditional checkout -b if branch do not exists and switch if exists - mkdir -p ip_country/src + if [[ BRANCH ]] + then + git checkout -B generated-source + git rm -rf . + mkdir -p ip_country/src + else + git switch generated-source + fi; mv "${TEMP_DIR}"/dbip_country.rs ip_country/src/dbip_country.rs git add ip_country/src/dbip_country.rs git commit -m "Update generated dbip_country ${YEAR_MONTH} Rust source file" From b221ba49bde75ceb2852bab1ac0884591faf152f Mon Sep 17 00:00:00 2001 From: czarte Date: Tue, 27 May 2025 13:54:47 +0200 Subject: [PATCH 07/12] fixing condition for generated-source branch --- .github/workflows/dbip_download.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dbip_download.yml b/.github/workflows/dbip_download.yml index 964b18552..b49584c99 100644 --- a/.github/workflows/dbip_download.yml +++ b/.github/workflows/dbip_download.yml @@ -42,18 +42,16 @@ jobs: - name: Commit and push generated file run: | - git ls-remote --heads | grep generated-source - BRANCH=$? git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - if [[ BRANCH ]] + if [[ "$(git ls-remote --heads 2>/dev/null | grep generated-sources)" == "" ]] then git checkout -B generated-source git rm -rf . mkdir -p ip_country/src else - git switch generated-source - fi; + git checkout generated-source + fi mv "${TEMP_DIR}"/dbip_country.rs ip_country/src/dbip_country.rs git add ip_country/src/dbip_country.rs git commit -m "Update generated dbip_country ${YEAR_MONTH} Rust source file" From 081e1e8a948619d3827721562e7780a3e0f0296d Mon Sep 17 00:00:00 2001 From: czarte Date: Mon, 2 Jun 2025 12:02:33 +0200 Subject: [PATCH 08/12] fix generated-source branch name in condition --- .github/workflows/dbip_download.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dbip_download.yml b/.github/workflows/dbip_download.yml index b49584c99..28d43f501 100644 --- a/.github/workflows/dbip_download.yml +++ b/.github/workflows/dbip_download.yml @@ -44,7 +44,7 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - if [[ "$(git ls-remote --heads 2>/dev/null | grep generated-sources)" == "" ]] + if [[ "$(git ls-remote --heads 2>/dev/null | grep generated-source)" == "" ]] then git checkout -B generated-source git rm -rf . From 812129be6eec908d89d6904fb5b5044a8feab867 Mon Sep 17 00:00:00 2001 From: czarte Date: Mon, 2 Jun 2025 13:44:13 +0200 Subject: [PATCH 09/12] increse standard timeout milis const --- multinode_integration_tests/src/masq_real_node.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multinode_integration_tests/src/masq_real_node.rs b/multinode_integration_tests/src/masq_real_node.rs index d9dc2c6da..df4541e5d 100644 --- a/multinode_integration_tests/src/masq_real_node.rs +++ b/multinode_integration_tests/src/masq_real_node.rs @@ -38,7 +38,7 @@ use std::thread; use std::time::Duration; pub const DATA_DIRECTORY: &str = "/node_root/home"; -pub const STANDARD_CLIENT_TIMEOUT_MILLIS: u64 = 1000; +pub const STANDARD_CLIENT_TIMEOUT_MILLIS: u64 = 2000; #[derive(Clone, Debug, PartialEq, Eq)] pub struct Firewall { From eb7638a344a362b9dfc42d835c61bbd91da48fe1 Mon Sep 17 00:00:00 2001 From: czarte Date: Tue, 3 Jun 2025 12:10:01 +0200 Subject: [PATCH 10/12] manual trigger and debugging flags to runs --- .github/workflows/dbip_download.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dbip_download.yml b/.github/workflows/dbip_download.yml index 28d43f501..832ab5ae2 100644 --- a/.github/workflows/dbip_download.yml +++ b/.github/workflows/dbip_download.yml @@ -1,8 +1,9 @@ name: Download DBIP data and generate dbip_country.rs on: - schedule: - - cron: "0 0 * * *" # Runs at midnight on the 3rd of every month + workflow_dispatch: +# schedule: +# - cron: "0 0 * * *" # Runs at midnight on the 3rd of every month env: TEMP_DIR: '/tmp' @@ -25,6 +26,7 @@ jobs: - name: Download DBIP data in MMDB format run: | + set -xv export YEAR_MONTH=$(date +%Y-%m) echo "TEMP_DIR=$(mktemp -d)" >> $GITHUB_ENV echo "YEAR_MONTH=$YEAR_MONTH" >> $GITHUB_ENV @@ -36,12 +38,14 @@ jobs: - name: Generate Rust source file run: | + set -xv cd dbipexperiment/ip_country cargo run < "dbip-data/dbip-country-lite.mmdb" > "$TEMP_DIR"/dbip_country.rs ls "$TEMP_DIR" - name: Commit and push generated file run: | + set -xv git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" if [[ "$(git ls-remote --heads 2>/dev/null | grep generated-source)" == "" ]] From c290555eac69e7e03343d600be0e5822a62e7c67 Mon Sep 17 00:00:00 2001 From: czarte Date: Tue, 3 Jun 2025 12:13:36 +0200 Subject: [PATCH 11/12] use variable to store git ls-remote command --- .github/workflows/dbip_download.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dbip_download.yml b/.github/workflows/dbip_download.yml index 832ab5ae2..edfa48c20 100644 --- a/.github/workflows/dbip_download.yml +++ b/.github/workflows/dbip_download.yml @@ -48,7 +48,8 @@ jobs: set -xv git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - if [[ "$(git ls-remote --heads 2>/dev/null | grep generated-source)" == "" ]] + BRANCH=$(git ls-remote --heads 2>/dev/null | grep generated-source) + if [[ "$BRANCH" == "" ]] then git checkout -B generated-source git rm -rf . From e3e4f3e506161a6ee70b8b271c9bd8e84abbba12 Mon Sep 17 00:00:00 2001 From: czarte Date: Wed, 4 Jun 2025 11:28:42 +0200 Subject: [PATCH 12/12] add git fetch before checkout --- .github/workflows/dbip_download.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/dbip_download.yml b/.github/workflows/dbip_download.yml index edfa48c20..1b7d626f7 100644 --- a/.github/workflows/dbip_download.yml +++ b/.github/workflows/dbip_download.yml @@ -26,7 +26,6 @@ jobs: - name: Download DBIP data in MMDB format run: | - set -xv export YEAR_MONTH=$(date +%Y-%m) echo "TEMP_DIR=$(mktemp -d)" >> $GITHUB_ENV echo "YEAR_MONTH=$YEAR_MONTH" >> $GITHUB_ENV @@ -38,7 +37,6 @@ jobs: - name: Generate Rust source file run: | - set -xv cd dbipexperiment/ip_country cargo run < "dbip-data/dbip-country-lite.mmdb" > "$TEMP_DIR"/dbip_country.rs ls "$TEMP_DIR" @@ -48,6 +46,7 @@ jobs: set -xv git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" + git fetch BRANCH=$(git ls-remote --heads 2>/dev/null | grep generated-source) if [[ "$BRANCH" == "" ]] then