Skip to content

Commit a5bb67f

Browse files
curquizacurquiza
authored
Update CI with integration tests (#77)
1 parent 50581a1 commit a5bb67f

File tree

7 files changed

+127
-36
lines changed

7 files changed

+127
-36
lines changed

‎.github/workflows/check.yml‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
name: Check
2+
3+
on:
4+
pull_request:
5+
push:
6+
# trying and staging branches are for BORS config
7+
branches:
8+
- trying
9+
- staging
10+
- master
11+
12+
jobs:
13+
linter:
14+
name: linter
15+
runs-on: ubuntu-18.04
16+
steps:
17+
- uses: actions/checkout@v2
18+
- name: Set up Python 3.8
19+
uses: actions/setup-python@v1
20+
with:
21+
python-version: 3.8
22+
- name: Install pipenv
23+
uses: dschep/install-pipenv-action@v1
24+
- name: Install dependencies
25+
run: pipenv install --dev
26+
- name: Linter with pylint
27+
run: pipenv run pylint scraper
28+
29+
unit_tests:
30+
name: unit-tests
31+
runs-on: ubuntu-18.04
32+
steps:
33+
- uses: actions/checkout@v2
34+
- name: Set up Python 3.8
35+
uses: actions/setup-python@v1
36+
with:
37+
python-version: 3.8
38+
- name: Install pipenv
39+
uses: dschep/install-pipenv-action@v1
40+
- name: Install dependencies
41+
run: pipenv install --dev
42+
- name: Run tests
43+
run: pipenv run pytest ./scraper/src -k "not _browser"
44+
45+
integration_tests:
46+
name: integration-tests
47+
runs-on: ubuntu-18.04
48+
steps:
49+
- uses: actions/checkout@v2
50+
- name: Set up Python 3.8
51+
uses: actions/setup-python@v1
52+
with:
53+
python-version: 3.8
54+
- name: Install pipenv
55+
uses: dschep/install-pipenv-action@v1
56+
- name: Install dependencies for production only
57+
run: pipenv install
58+
- name: MeiliSearch setup with Docker
59+
run: docker run -d -p 7700:7700 getmeili/meilisearch:latest ./meilisearch --no-analytics=true --master-key=masterKey
60+
- name: Run the scraper with basic config
61+
run: pipenv run ./docs_scraper $CONFIG_FILE
62+
env:
63+
MEILISEARCH_HOST_URL: 'http://127.0.0.1:7700'
64+
MEILISEARCH_API_KEY: 'masterKey'
65+
CONFIG_FILE: 'scraper/src/tests/config_files_examples/docs-basics.config.json'
66+
- name: Run the scraper with custom config
67+
run: pipenv run ./docs_scraper $CONFIG_FILE
68+
env:
69+
MEILISEARCH_HOST_URL: 'http://127.0.0.1:7700'
70+
MEILISEARCH_API_KEY: 'masterKey'
71+
CONFIG_FILE: 'scraper/src/tests/config_files_examples/docs-custom.config.json'
72+
73+
image_build:
74+
name: image-build
75+
runs-on: ubuntu-18.04
76+
steps:
77+
- uses: actions/checkout@v2
78+
- name: Build Docker image
79+
run: docker build .

‎.github/workflows/test-lint.yml‎

Lines changed: 0 additions & 32 deletions
This file was deleted.

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,4 @@ configs/public
2121
configs/private
2222

2323
# config file
24-
*config*.json
24+
config.json

‎README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
</h4>
1515

1616
<p align="center">
17-
<a href="https://github.com/meilisearch/docs-searchbar.js/actions"><img src="https://github.com/meilisearch/docs-searchbar.js/workflows/Tests-Lint/badge.svg" alt="Test"></a>
17+
<a href="https://github.com/meilisearch/docs-searchbar.js/actions"><img src="https://github.com/meilisearch/docs-searchbar.js/workflows/Check/badge.svg" alt="Test"></a>
1818
<a href="https://github.com/meilisearch/docs-scraper/blob/master/LICENSE"><img src="https://img.shields.io/badge/license-MIT-informational" alt="License"></a>
1919
<a href="https://slack.meilisearch.com"><img src="https://img.shields.io/badge/slack-MeiliSearch-blue.svg?logo=slack" alt="Slack"></a>
2020
<a href="https://app.bors.tech/repositories/28977"><img src="https://bors.tech/images/badge_small.svg" alt="Bors enabled"></a>
@@ -478,7 +478,7 @@ In case of Google Cloud Identity-Aware Proxy, please specify these env variables
478478
- `IAP_AUTH_CLIENT_ID` - # pick [client ID of the application](https://console.cloud.google.com/apis/credentials) you are connecting to
479479
- `IAP_AUTH_SERVICE_ACCOUNT_JSON` - # generate in [Actions](https://console.cloud.google.com/iam-admin/serviceaccounts) -> Create key -> JSON
480480

481-
#### Keycloak Access: Identity and Access Management
481+
#### Keycloak Access: Identity and Access Management <!-- omit in TOC -->
482482

483483
If you need to scrape site protected by [Keycloak](https://github.com/keycloak/keycloak) (Gatekeeper), you have to provide a valid access token.
484484

‎bors.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
status = ["docs-scraper-tests"]
1+
status = ['unit-tests', 'integration-tests', 'linter', 'image-build']
22
# 1 hour timeout
33
timeout-sec = 3600

‎scraper/src/tests/config_files_examples/docs-basics.config.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"index_uid": "docs",
3+
"start_urls": ["https://docs.meilisearch.com"],
4+
"selectors": {
5+
"lvl0": {
6+
"selector": ".sidebar-heading.open",
7+
"global": true,
8+
"default_value": "Documentation"
9+
},
10+
"lvl1": ".theme-default-content h1",
11+
"lvl2": ".theme-default-content h2",
12+
"lvl3": ".theme-default-content h3",
13+
"lvl4": ".theme-default-content h4",
14+
"lvl5": ".theme-default-content h5",
15+
"text": ".theme-default-content p, .theme-default-content li"
16+
}
17+
}

‎scraper/src/tests/config_files_examples/docs-custom.config.json‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{
2+
"index_uid": "docs",
3+
"sitemap_urls": ["https://docs.meilisearch.com/sitemap.xml"],
4+
"start_urls": ["https://docs.meilisearch.com"],
5+
"selectors": {
6+
"lvl0": {
7+
"selector": ".sidebar-heading.open",
8+
"global": true,
9+
"default_value": "Documentation"
10+
},
11+
"lvl1": ".theme-default-content h1",
12+
"lvl2": ".theme-default-content h2",
13+
"lvl3": ".theme-default-content h3",
14+
"lvl4": ".theme-default-content h4",
15+
"lvl5": ".theme-default-content h5",
16+
"text": ".theme-default-content p, .theme-default-content li"
17+
},
18+
"strip_chars": " .,;:#",
19+
"scrap_start_urls": true,
20+
"custom_settings": {
21+
"synonyms": {
22+
"relevancy": ["relevant", "relevance"],
23+
"relevant": ["relevancy", "relevance"],
24+
"relevance": ["relevancy", "relevant"]
25+
}
26+
}
27+
}

0 commit comments

Comments
 (0)