Skip to content

Commit 63bd58f

Browse files
adrianprelipceanadrianprelipcean
authored
Merge pull request #8 from express-rate-limit/feature/database-ci
Feature/database ci
2 parents 475a1c9 + f9b2693 commit 63bd58f

37 files changed

+912
-104
lines changed

‎.github/workflows/cicd.yml

Lines changed: 86 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ on:
55
pull_request:
66

77
jobs:
8-
ci:
9-
name: Continuous Integration
8+
ci-node:
9+
name: Continuous Integration Node
1010
runs-on: ubuntu-latest
1111
strategy:
1212
matrix:
@@ -20,10 +20,93 @@ jobs:
2020
- run: npm ci
2121
- run: npm test
2222
- run: npm run lint
23+
ci-db-linting:
24+
name: Continuous Integration Database - Linting
25+
runs-on: ubuntu-latest
26+
strategy:
27+
matrix:
28+
python-version: ["3.10"]
29+
steps:
30+
- uses: actions/checkout@v3
31+
with:
32+
fetch-depth: 0
33+
- name: Set up Python ${{ matrix.python-version }}
34+
uses: actions/setup-python@v3
35+
with:
36+
python-version: ${{ matrix.python-version }}
37+
- name: Cache requirements
38+
uses: actions/cache@v3
39+
with:
40+
path: ~/.cache/pip
41+
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
42+
restore-keys: |
43+
${{ runner.os }}-pip-
44+
- name: Install requirements
45+
if: steps.cache.outputs.cache-hit != 'true'
46+
run: |
47+
python -m pip install --upgrade pip
48+
pip install -r db/linting/requirements.txt
49+
- name: Analysing the SQL code
50+
run: |
51+
bash db/linting/lint.sh
52+
ci-db-testing:
53+
name: Continuous Integration Database - Testing
54+
runs-on: ubuntu-latest
55+
services:
56+
postgres:
57+
image: acprdev/postgres-ci:15.2
58+
env:
59+
DATABASE: test_postgres
60+
HOST: localhost
61+
PORT: 5432
62+
USER: postgres
63+
PASSWORD: postgres
64+
POSTGRES_PASSWORD: postgres
65+
POSTGRES_USER: postgres
66+
options: >-
67+
--health-cmd pg_isready
68+
--health-interval 10s
69+
--health-timeout 5s
70+
--health-retries 5
71+
ports:
72+
- 5432:5432
73+
74+
steps:
75+
- uses: actions/checkout@v3
76+
- name: Setup Perl
77+
id: perl
78+
uses: shogo82148/actions-setup-perl@v1
79+
with:
80+
perl-version: '5.34'
81+
install-modules-with: cpm
82+
- name: Cache CPAN Modules
83+
uses: actions/cache@v3
84+
with:
85+
path: local
86+
key: perl-${{ steps.perl.outputs.perl-hash }}
87+
- name: Install pg_prove
88+
run: |
89+
cpm install TAP::Parser::SourceHandler::pgTAP
90+
env:
91+
SHELL: /bin/bash
92+
- name: Initialize database and run tests
93+
run: |
94+
bash db/cli/init_db.sh
95+
bash db/cli/apply_all_migrations.sh
96+
bash db/cli/run_tests.sh
97+
env:
98+
DATABASE: test_postgres
99+
HOST: localhost
100+
PORT: 5432
101+
USER: postgres
102+
PASSWORD: postgres
103+
POSTGRES_PASSWORD: postgres
104+
POSTGRES_USER: postgres
105+
PGPASSWORD: postgres
23106
cd:
24107
name: Continuous Deployment
25108
runs-on: ubuntu-latest
26-
needs: ci
109+
needs: [ci-node,ci-db-linting,ci-db-testing]
27110
if: startsWith(github.ref, 'refs/tags/v')
28111
steps:
29112
- name: Checkout the repository

‎.gitignore

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,6 @@ node_modules/
22

33
local_folder/
44

5-
dist/
5+
dist/
6+
7+
.venv/

‎changelog.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
66
and this project adheres to
77
[Semantic Versioning](https://semver.org/spec/v2.0.0.html).
88

9+
## [1.3.0](https://github.com/adrianprelipcean/express-rate-limit-postgresql/releases/tag/v1.3.0)
10+
11+
### Changed
12+
13+
- Stores call stored procedures instead of raw SQL
14+
915
## [1.2.0](https://github.com/adrianprelipcean/express-rate-limit-postgresql/releases/tag/v1.2.0)
1016

1117
### Changed

‎contributing.md

Lines changed: 46 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -68,14 +68,40 @@ rate-limit-postgresql
6868
│ ├── tsconfig.cjs.json
6969
│ └── tsconfig.esm.json
7070
├── contributing.md
71+
├── db
72+
│ ├── cli
73+
│ │ ├── apply_all_migrations.sh
74+
│ │ ├── init_db.sh
75+
│ │ └── run_tests.sh
76+
│ ├── linting
77+
│ │ ├── lint.sh
78+
│ │ └── requirements.txt
79+
│ └── tests
80+
│ ├── performance
81+
│ │ ├── aggregated_ip.test.sql
82+
│ │ └── individual_ip.test.sql
83+
│ └── unit
84+
│ ├── agg_decrement.test.sql
85+
│ ├── agg_increment.test.sql
86+
│ ├── agg_reset_key.test.sql
87+
│ ├── agg_reset_session.test.sql
88+
│ ├── ind_decrement.test.sql
89+
│ ├── ind_increment.test.sql
90+
│ ├── ind_reset_key.test.sql
91+
│ ├── ind_reset_session.test.sql
92+
│ ├── session_reset.test.sql
93+
│ └── session_select.test.sql
7194
├── license.md
7295
├── package.json
7396
├── package-lock.json
7497
├── readme.md
7598
├── source
7699
│ ├── index.ts
77100
│ ├── migrations
78-
│ │ └── 1-init.sql
101+
│ │ ├── 1-init.sql
102+
│ │ ├── 2-add-db-functions-agg.sql
103+
│ │ ├── 3-add-db-functions-ind.sql
104+
│ │ └── 4-add-db-functions-sessions.sql
79105
│ ├── models
80106
│ │ └── session.ts
81107
│ ├── stores
@@ -92,11 +118,12 @@ rate-limit-postgresql
92118
│ │ └── store_individual_ip.spec.ts
93119
│ └── util
94120
│ └── session_handler.spec.ts
95-
└── third_party_licenses
96-
├── dev_detailed.json
97-
├── dev_summary.txt
98-
├── production_detailed.json
99-
└── production_summary.txt
121+
├── third_party_licenses
122+
│ ├── dev_detailed.json
123+
│ ├── dev_summary.txt
124+
│ ├── production_detailed.json
125+
│ └── production_summary.txt
126+
└── tsconfig.json
100127
```
101128

102129
> The content of `third_party_licenses` is auto-generated.
@@ -120,10 +147,22 @@ applied, etc.
120147

121148
- `tsconfig.*.json`: The Typescript configuration files for this project.
122149

150+
#### `db/`
151+
152+
- `db/cli/*.sh`: Bash scripts used by the database part of the CI pipeline
153+
- `db/linting/lint.sh`: Bash scripts used for linting SQL files as part of the
154+
database CI pipeline
155+
- `db/linting/.sqlfluff`: Configuration file for the `sqlfluff` linter
156+
- `db/linting/requirements.txt`: Python dependencies for the `sqlfluff` linter
157+
- `db/tests/performance/*.test.sql`: Performance tests for the stored procedures
158+
(using `pg_tap`)
159+
- `db/cli/unit/.test.sql`: Unit tests for the stored procedures (using `pg_tap`)
160+
123161
#### `source/`
124162

125163
- `source/migrations/*.sql`: Database migrations that are applied by
126-
`postgres-migrations` `source/models/*.ts`: Relevant types (e.g., Session)
164+
`postgres-migrations`
165+
- `source/models/*.ts`: Relevant types (e.g., Session)
127166
- `source/util/*.ts`: The centralized business logic for handling session
128167
validity (`session_handler.ts`) and migration handling
129168
(`migration_handler.ts`)

‎db/cli/apply_all_migrations.sh

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
echo 'Applying migrations'
2+
for f in `ls -v source/migrations/*.sql`;
3+
do
4+
if test -f "$f"; then
5+
psql -U $USER -h $HOST -p $PORT -d $DATABASE -v "ON_ERROR_STOP=1" -f "$f"
6+
echo 'Applied migration -' $f
7+
fi
8+
done

‎db/cli/create_migration.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
now=$(TZ=UTC date -R)
2+
count=$(ls source/migrations/ | wc -l)
3+
((count++))
4+
echo '--Migration generated '$now > source/migrations/$count-$1.sql

‎db/cli/init_db.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
createdb --username=$USER -h $HOST -p $PORT $DATABASE

‎db/cli/run_tests.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
set -e
2+
3+
psql --quiet -U $USER -h $HOST -p $PORT -d $DATABASE -c 'CREATE EXTENSION IF NOT EXISTS pgtap;'
4+
5+
pg_prove -U $USER -h $HOST -p $PORT -d $DATABASE db/tests/performance/*.test.sql
6+
7+
pg_prove -U $USER -h $HOST -p $PORT -d $DATABASE db/tests/unit/*.test.sql

‎db/linting/.sqlfluff

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
[sqlfluff]
2+
large_file_skip_char_limit = 0
3+
large_file_skip_byte_limit = 0
4+
exclude_rules = L016,LT05,RF05

‎db/linting/lint.sh

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
echo 'Linting migrations folder'
2+
sqlfluff lint source/migrations/ --dialect postgres --ignore parsing --config db/linting/.sqlfluff
3+
4+
echo 'Linting tests folder'
5+
sqlfluff lint db/tests/**/ --dialect postgres --ignore parsing --config db/linting/.sqlfluff

0 commit comments

Comments
 (0)