Skip to content

Commit add18c3

Browse files
ckiosidisckiosidis
authored
Merge branch 'master' into add-meltwater-to-users
2 parents 25d2218 + 9fd49f3 commit add18c3

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+499
-338
lines changed

‎.coveragerc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ omit =
1111
*/bin/luigidc
1212
hadoop_test.py
1313
minicluster.py
14+
1415
[run]
1516
parallel=True
1617
concurrency=multiprocessing

‎.github/CODEOWNERS‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
# These owners will be the default owners for everything in
66
# the repo. Unless a later match takes precedence,
7-
* @dlstadther @Tarrasch @honnix
7+
* @dlstadther @Tarrasch @spotify/dataex
88

99
# Specific files, directories, paths, or file types can be
1010
# assigned more specificially.

‎.github/workflows/pythonbuild.yml‎

Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,201 @@
1+
name: Build
2+
3+
on:
4+
push:
5+
branches:
6+
- master
7+
pull_request:
8+
9+
jobs:
10+
core:
11+
runs-on: ubuntu-20.04
12+
13+
strategy:
14+
matrix:
15+
include:
16+
- python-version: 3.6
17+
tox-env: py36-core
18+
- python-version: 3.7
19+
tox-env: py37-core
20+
- python-version: 3.8
21+
tox-env: py38-core
22+
- python-version: 3.9
23+
tox-env: py39-core
24+
25+
steps:
26+
- uses: actions/checkout@v2
27+
- name: Set up Python ${{ matrix.python-version }}
28+
uses: actions/setup-python@v2
29+
with:
30+
python-version: ${{ matrix.python-version }}
31+
- name: Cache pip
32+
uses: actions/cache@v2
33+
with:
34+
# This path is specific to Ubuntu
35+
path: ~/.cache/pip
36+
# Look to see if there is a cache hit for the corresponding requirements files
37+
key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('requirements{0}.txt', matrix.spark-version-suffix))) }}
38+
- name: Install dependencies
39+
run: |
40+
python -m pip install --upgrade pip 'tox<3.0'
41+
- name: Setup MySQL DB
42+
run: |
43+
sudo /etc/init.d/mysql start
44+
mysql -e 'create database IF NOT EXISTS luigi_test;' -uroot -proot || true
45+
mysql -e 'create user 'travis'@'localhost';' -uroot -proot || true
46+
mysql -e 'grant all privileges ON *.* TO 'travis'@'localhost';' -uroot -proot || true
47+
- name: Build
48+
env:
49+
TOXENV: ${{ matrix.tox-env }}
50+
run: tox
51+
- name: Codecov
52+
env:
53+
COVERAGE_PROCESS_START: .coveragerc
54+
run: |
55+
pip install codecov
56+
codecov -e ${{ matrix.tox-env }}
57+
58+
postgres:
59+
runs-on: ubuntu-20.04
60+
services:
61+
postgres:
62+
image: postgres
63+
env:
64+
POSTGRES_USER: postgres
65+
POSTGRES_PASSWORD: postgres
66+
POSTGRES_DB: postgres
67+
ports:
68+
- 5432:5432
69+
# Set health checks to wait until postgres has started
70+
options: >-
71+
--health-cmd pg_isready
72+
--health-interval 10s
73+
--health-timeout 5s
74+
--health-retries 5
75+
76+
strategy:
77+
matrix:
78+
include:
79+
- python-version: 3.6
80+
tox-env: py36-postgres
81+
- python-version: 3.7
82+
tox-env: py37-postgres
83+
- python-version: 3.8
84+
tox-env: py38-postgres
85+
- python-version: 3.9
86+
tox-env: py39-postgres
87+
88+
steps:
89+
- uses: actions/checkout@v2
90+
- name: Set up Python ${{ matrix.python-version }}
91+
uses: actions/setup-python@v2
92+
with:
93+
python-version: ${{ matrix.python-version }}
94+
- name: Cache pip
95+
uses: actions/cache@v2
96+
with:
97+
# This path is specific to Ubuntu
98+
path: ~/.cache/pip
99+
# Look to see if there is a cache hit for the corresponding requirements files
100+
key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('requirements{0}.txt', matrix.spark-version-suffix))) }}
101+
- name: Install dependencies
102+
run: |
103+
python -m pip install --upgrade pip 'tox<3.0'
104+
- name: Create PSQL database
105+
run: |
106+
PGPASSWORD=postgres psql -h localhost -p 5432 -c 'create database spotify;' -U postgres
107+
- name: Build
108+
env:
109+
TOXENV: ${{ matrix.tox-env }}
110+
run: tox
111+
- name: Codecov
112+
env:
113+
COVERAGE_PROCESS_START: .coveragerc
114+
run: |
115+
pip install codecov
116+
codecov -e ${{ matrix.tox-env }}
117+
118+
base:
119+
runs-on: ubuntu-20.04
120+
env:
121+
AWS_DEFAULT_REGION: us-east-1
122+
AWS_ACCESS_KEY_ID: accesskey
123+
AWS_SECRET_ACCESS_KEY: secretkey
124+
125+
strategy:
126+
matrix:
127+
include:
128+
- python-version: 3.6
129+
tox-env: py36-aws
130+
- python-version: 3.7
131+
tox-env: py37-aws
132+
- python-version: 3.8
133+
tox-env: py38-aws
134+
- python-version: 3.9
135+
tox-env: py39-aws
136+
137+
- python-version: 3.6
138+
tox-env: py36-unixsocket
139+
OVERRIDE_SKIP_CI_TESTS: True
140+
- python-version: 3.7
141+
tox-env: py37-unixsocket
142+
OVERRIDE_SKIP_CI_TESTS: True
143+
- python-version: 3.8
144+
tox-env: py38-unixsocket
145+
OVERRIDE_SKIP_CI_TESTS: True
146+
- python-version: 3.9
147+
tox-env: py39-unixsocket
148+
OVERRIDE_SKIP_CI_TESTS: True
149+
150+
- python-version: 3.6
151+
tox-env: py36-apache
152+
- python-version: 3.7
153+
tox-env: py37-apache
154+
- python-version: 3.8
155+
tox-env: py38-apache
156+
- python-version: 3.9
157+
tox-env: py39-apache
158+
159+
- python-version: 3.6
160+
tox-env: py36-azureblob
161+
- python-version: 3.7
162+
tox-env: py37-azureblob
163+
- python-version: 3.8
164+
tox-env: py38-azureblob
165+
- python-version: 3.9
166+
tox-env: py39-azureblob
167+
168+
169+
- python-version: 3.9
170+
tox-env: flake8
171+
- python-version: 3.9
172+
tox-env: docs
173+
174+
steps:
175+
- uses: actions/checkout@v2
176+
- name: Set up Python ${{ matrix.python-version }}
177+
uses: actions/setup-python@v2
178+
with:
179+
python-version: ${{ matrix.python-version }}
180+
- name: Cache pip
181+
uses: actions/cache@v2
182+
with:
183+
# This path is specific to Ubuntu
184+
path: ~/.cache/pip
185+
# Look to see if there is a cache hit for the corresponding requirements files
186+
key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('requirements{0}.txt', matrix.spark-version-suffix))) }}
187+
- name: Install dependencies
188+
run: |
189+
python -m pip install --upgrade pip 'tox<4.0'
190+
- name: Build
191+
env:
192+
TOXENV: ${{ matrix.tox-env }}
193+
OVERRIDE_SKIP_CI_TESTS: ${{ matrix.OVERRIDE_SKIP_CI_TESTS }}
194+
run: tox
195+
- name: Codecov
196+
if: ${{ matrix.tox-env != 'flake8' && matrix.tox-env != 'docs' }}
197+
env:
198+
COVERAGE_PROCESS_START: .coveragerc
199+
run: |
200+
pip install codecov
201+
codecov -e ${{ matrix.tox-env }}

‎.travis.yml‎

Lines changed: 0 additions & 105 deletions
This file was deleted.

‎README.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
.. image:: https://img.shields.io/pypi/l/luigi.svg?style=flat
1515
:target: https://pypi.python.org/pypi/luigi
1616

17-
Luigi is a Python (3.6, 3.7 tested) package that helps you build complex
17+
Luigi is a Python (3.6, 3.7, 3.8, 3.9 tested) package that helps you build complex
1818
pipelines of batch jobs. It handles dependency resolution, workflow management,
1919
visualization, handling failures, command line integration, and much more.
2020

‎codecov.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
codecov:
2+
require_ci_to_pass: yes
3+
notify:
4+
after_n_builds: 24
5+
wait_for_ci: yes
6+
17
coverage:
28
precision: 2 # Just copied from default
39
round: down # Just copied from default

‎doc/central_scheduler.rst‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ To run the server as a daemon run:
3131
3232
Note that this requires ``python-daemon``.
3333
By default, the server starts on AF_INET and AF_INET6 port ``8082``
34-
(which can be changed with the ``--port`` flag) and listens on all IPs.
35-
(To use an AF_UNIX socket use the ``--unix-socket`` flag)
34+
(which can be changed with the ``--port`` flag) and listens on all IPs. To change the default behavior of listening on all IPs, pass the ``--address`` flag and the IP address to listen on.
35+
To use an AF_UNIX socket use the ``--unix-socket`` flag.
3636

3737
For a full list of configuration options and defaults,
3838
see the :ref:`scheduler configuration section <scheduler-config>`.

‎setup.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,8 @@ def get_static_files(path):
113113
'Programming Language :: Python :: 3.5',
114114
'Programming Language :: Python :: 3.6',
115115
'Programming Language :: Python :: 3.7',
116+
'Programming Language :: Python :: 3.8',
117+
'Programming Language :: Python :: 3.9',
116118
'Topic :: System :: Monitoring',
117119
],
118120
)

‎test/conftest.py‎

Whitespace-only changes.

‎test/contrib/_webhdfs_test.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,10 @@
2020

2121
from luigi.contrib import webhdfs
2222

23-
from nose.plugins.attrib import attr
23+
import pytest
2424

2525

26-
@attr('apache')
26+
@pytest.mark.apache
2727
class TestWebHdfsTarget(unittest.TestCase):
2828

2929
'''

0 commit comments

Comments
 (0)