Skip to content

Commit 232a395

Browse files
pducolinpducolin
authored
Remove python requirements (#1748)
Co-authored-by: paola.ducolin <[email protected]>
1 parent 9584820 commit 232a395

File tree

9 files changed

+100
-61
lines changed

9 files changed

+100
-61
lines changed

‎.dda/extend/commands/validate/__init__.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
from __future__ import annotations
2+
3+
from dda.cli.base import dynamic_group
4+
5+
6+
@dynamic_group(short_help="Static analysis commands")
7+
def cmd() -> None:
8+
"""
9+
Commands related to static analysis.
10+
"""
11+
pass

‎.dda/extend/commands/validate/python/__init__.py‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
from __future__ import annotations
2+
3+
from typing import TYPE_CHECKING
4+
5+
import click
6+
from dda.cli.base import dynamic_command, pass_app
7+
8+
if TYPE_CHECKING:
9+
from dda.cli.application import Application
10+
11+
12+
@dynamic_command(
13+
short_help="Static analysis for Python code",
14+
# features=["legacy-test-infra-definitions"],
15+
dependencies=[
16+
"ruff==0.12.2",
17+
"vulture==2.3",
18+
"invoke==2.2.0",
19+
"termcolor==3.1.0",
20+
"pyyaml==6.0.2",
21+
"pyperclip==1.11.0",
22+
"pydantic==2.11.4",
23+
"colorama>=0.4.6",
24+
"lxml~=5.2.2",
25+
"python-gitlab==6.4.0",
26+
"PyGithub==1.59.1",
27+
"boto3>=1.28.0",
28+
"pyright==1.1.405",
29+
],
30+
)
31+
@click.argument("paths", nargs=-1, type=click.Path(exists=True), default=["tasks", ".dda"])
32+
@click.option("--fix", is_flag=True, help="Fix linting errors")
33+
@pass_app
34+
def cmd(app: Application, paths: list[str], fix: bool) -> None:
35+
"""
36+
Lint python code.
37+
"""
38+
paths_list = list[str](paths)
39+
app.display(f"Running pyright on {paths_list}")
40+
app.subprocess.run(["pyright"] + paths_list)
41+
if fix:
42+
app.subprocess.run(["ruff", "check", "--fix"] + paths_list)
43+
app.subprocess.run(["ruff", "check"] + paths_list)
44+
app.subprocess.run(
45+
["vulture", "--ignore-decorators", "@task,@dynamic_command,@dynamic_group", "--ignore-names", "test_*,Test*"]
46+
+ paths_list
47+
)

‎.github/workflows/lint.yml‎

Lines changed: 9 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -39,26 +39,18 @@ jobs:
3939
install-mode: "goinstall"
4040
- name: Check tidy
4141
run: go mod tidy && git diff-files --exit-code go.mod go.sum
42-
lint-python-type:
43-
runs-on: ubuntu-latest
44-
steps:
45-
- name: Checkout
46-
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
47-
- name: Install Python requirements
48-
run: pip3 install -r requirements.txt
49-
- name: Install pyright
50-
run: pip3 install pyright
51-
- name: Run pyright
52-
run: pyright tasks
5342
lint-python:
5443
runs-on: ubuntu-latest
5544
steps:
5645
- name: Checkout
57-
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
58-
- name: Install lint python requirements
59-
run: pip3 install ruff vulture
46+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
47+
48+
- name: Install dda
49+
uses: DataDog/datadog-agent-dev@00e4a423088309efce1d5ba6b8c5366eef648710
50+
with:
51+
version: v0.30.2
52+
features: legacy-test-infra-definitions
53+
6054
- name: Run linters
6155
run: |
62-
ruff format --check tasks
63-
ruff check tasks
64-
vulture --ignore-decorators @task --ignore-names 'test_*,Test*' tasks
56+
dda validate python

‎.gitlab-ci.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -80,14 +80,14 @@ integration-testing:
8080
- eval $(ssh-agent -s)
8181
- chmod 400 $E2E_PRIVATE_KEY_PATH
8282
- ssh-add $E2E_PRIVATE_KEY_PATH
83-
- pip install -r requirements.txt
8483
- go install github.com/DataDog/[email protected]
8584
- orchestrion pin
8685
- export DD_CIVISIBILITY_ENABLED=true
8786
- export DD_CIVISIBILITY_AGENTLESS_ENABLED=true
8887
- export DD_ENV=ci
8988
- export DD_API_KEY=$(vault kv get -field=api_key kv/k8s/gitlab-runner/test-infra-definitions/dd-token)
9089
- export GOFLAGS="${GOFLAGS} '-toolexec=orchestrion toolexec'"
90+
- dda self dep sync -f legacy-test-infra-definitions
9191
script:
9292
- |
9393
if [ ! -f ./dist/main ]; then
@@ -148,10 +148,10 @@ bump-version-on-datadog-agent:
148148
- git checkout -b $PR_BRANCH
149149
- git push origin $PR_BRANCH # Create the reference to push the commit through API later
150150
- export PREVIOUS_SHA=$(cat .gitlab/common/test_infra_version.yml | grep 'TEST_INFRA_DEFINITIONS_BUILDIMAGES:' | awk -F " " '{print $NF}')
151+
- dda self dep sync -f legacy-test-infra-definitions
151152
- dda inv -e buildimages.update-test-infra-definitions --commit-sha $CI_COMMIT_SHA $EXTRA_UPDATE_ARGS
152153
- dda inv -e tidy
153154
- git add -u
154155
- dda inv -e git.push-signed-commits --branch $PR_BRANCH --commit-message "[test-infra-definitions][automated] Bump test-infra-definitions to $CI_COMMIT_SHORT_SHA"
155156
- popd
156-
- pip install -r requirements.txt
157-
- inv ci.create-bump-pr-and-close-stale-ones-on-datadog-agent --branch auto-bump/bump-test-infra-$CI_COMMIT_SHORT_SHA --new-commit-sha $CI_COMMIT_SHA --old-commit-sha $PREVIOUS_SHA
157+
- dda inv ci.create-bump-pr-and-close-stale-ones-on-datadog-agent --branch auto-bump/bump-test-infra-$CI_COMMIT_SHORT_SHA --new-commit-sha $CI_COMMIT_SHA --old-commit-sha $PREVIOUS_SHA

‎.pre-commit-config.yaml‎

Lines changed: 15 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -6,26 +6,25 @@ repos:
66
- id: golangci-lint
77
- repo: local
88
hooks:
9-
- id: pyright
10-
name: pyright
11-
entry: pyright
12-
language: system # Use local install for pyright, else we cannot import the requirements and we get errors on external libraries
13-
"types_or": [python, pyi]
9+
- id: validate-python-fix
10+
name: validate-python-fix
11+
language: system
12+
entry: dda validate python
13+
args: [--fix]
1414
require_serial: true
1515
additional_dependencies: []
16+
types_or: [python, pyi]
1617
minimum_pre_commit_version: "2.9.2"
17-
- repo: https://github.com/astral-sh/ruff-pre-commit
18-
rev: v0.12.2
19-
hooks:
20-
- id: ruff
21-
args: [--fix]
22-
- id: ruff-format
23-
- repo: https://github.com/jendrikseipp/vulture
24-
rev: "v2.3"
25-
hooks:
26-
- id: vulture
2718
- repo: local
2819
hooks:
20+
- id: validate-python
21+
name: validate-python
22+
entry: dda validate python
23+
language: system # Use local install for pyright, else we cannot import the requirements and we get errors on external libraries
24+
require_serial: true
25+
additional_dependencies: []
26+
types_or: [python, pyi]
27+
minimum_pre_commit_version: "2.9.2"
2928
- id: go-tidy
3029
name: go-tidy
3130
entry: go mod tidy
@@ -38,4 +37,4 @@ repos:
3837
entry: "inv ci.check-protected-branch"
3938
language: system
4039
pass_filenames: false
41-
stages: [pre-commit, pre-push]
40+
stages: [pre-commit, pre-push]

‎.python-version‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
3.12

‎integration-tests/invoke_test.go‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -89,13 +89,13 @@ func testAzureInvokeVM(t *testing.T, tmpConfigFile string, workingDirectory stri
8989
stackName = sanitizeStackName(stackName)
9090

9191
t.Log("creating vm")
92-
createCmd := exec.Command("invoke", "az.create-vm", "--no-interactive", "--stack-name", stackName, "--config-path", tmpConfigFile, "--no-add-known-host")
92+
createCmd := exec.Command("dda", "inv", "az.create-vm", "--no-interactive", "--stack-name", stackName, "--config-path", tmpConfigFile, "--no-add-known-host")
9393
createCmd.Dir = workingDirectory
9494
createOutput, err := createCmd.Output()
9595
assert.NoError(t, err, "Error found creating vm: %s", string(createOutput))
9696

9797
t.Log("destroying vm")
98-
destroyCmd := exec.Command("invoke", "az.destroy-vm", "--no-clean-known-hosts", "--stack-name", stackName, "--config-path", tmpConfigFile)
98+
destroyCmd := exec.Command("dda", "inv", "az.destroy-vm", "--no-clean-known-hosts", "--stack-name", stackName, "--config-path", tmpConfigFile)
9999
destroyCmd.Dir = workingDirectory
100100
destroyOutput, err := destroyCmd.Output()
101101
require.NoError(t, err, "Error found destroying stack: %s", string(destroyOutput))
@@ -108,13 +108,13 @@ func testAwsInvokeVM(t *testing.T, tmpConfigFile string, workingDirectory string
108108
stackName = sanitizeStackName(stackName)
109109

110110
t.Log("creating vm")
111-
createCmd := exec.Command("invoke", "aws.create-vm", "--no-interactive", "--stack-name", stackName, "--config-path", tmpConfigFile, "--use-fakeintake", "--no-add-known-host")
111+
createCmd := exec.Command("dda", "inv", "aws.create-vm", "--no-interactive", "--stack-name", stackName, "--config-path", tmpConfigFile, "--use-fakeintake", "--no-add-known-host")
112112
createCmd.Dir = workingDirectory
113113
createOutput, err := createCmd.Output()
114114
assert.NoError(t, err, "Error found creating vm: %s", string(createOutput))
115115

116116
t.Log("destroying vm")
117-
destroyCmd := exec.Command("invoke", "aws.destroy-vm", "--no-clean-known-hosts", "--stack-name", stackName, "--config-path", tmpConfigFile)
117+
destroyCmd := exec.Command("dda", "inv", "aws.destroy-vm", "--no-clean-known-hosts", "--stack-name", stackName, "--config-path", tmpConfigFile)
118118
destroyCmd.Dir = workingDirectory
119119
destroyOutput, err := destroyCmd.Output()
120120
require.NoError(t, err, "Error found destroying stack: %s", string(destroyOutput))
@@ -127,13 +127,13 @@ func testGcpInvokeVM(t *testing.T, tmpConfigFile string, workingDirectory string
127127
stackName = sanitizeStackName(stackName)
128128

129129
t.Log("creating vm")
130-
createCmd := exec.Command("invoke", "gcp.create-vm", "--no-interactive", "--stack-name", stackName, "--config-path", tmpConfigFile, "--use-fakeintake", "--no-add-known-host")
130+
createCmd := exec.Command("dda", "inv", "gcp.create-vm", "--no-interactive", "--stack-name", stackName, "--config-path", tmpConfigFile, "--use-fakeintake", "--no-add-known-host")
131131
createCmd.Dir = workingDirectory
132132
createOutput, err := createCmd.Output()
133133
assert.NoError(t, err, "Error found creating vm: %s", string(createOutput))
134134

135135
t.Log("destroying vm")
136-
destroyCmd := exec.Command("invoke", "gcp.destroy-vm", "--no-clean-known-hosts", "--stack-name", stackName, "--config-path", tmpConfigFile)
136+
destroyCmd := exec.Command("dda", "inv", "gcp.destroy-vm", "--no-clean-known-hosts", "--stack-name", stackName, "--config-path", tmpConfigFile)
137137
destroyCmd.Dir = workingDirectory
138138
destroyOutput, err := destroyCmd.Output()
139139
require.NoError(t, err, "Error found destroying stack: %s", string(destroyOutput))
@@ -146,7 +146,7 @@ func testInvokeDockerVM(t *testing.T, tmpConfigFile string, workingDirectory str
146146
t.Log("creating vm with docker")
147147
var stdOut, stdErr bytes.Buffer
148148

149-
createCmd := exec.Command("invoke", "aws.create-docker", "--no-interactive", "--stack-name", stackName, "--config-path", tmpConfigFile, "--use-fakeintake", "--use-loadBalancer")
149+
createCmd := exec.Command("dda", "inv", "aws.create-docker", "--no-interactive", "--stack-name", stackName, "--config-path", tmpConfigFile, "--use-fakeintake", "--use-loadBalancer")
150150
createCmd.Dir = workingDirectory
151151
createCmd.Stdout = &stdOut
152152
createCmd.Stderr = &stdErr
@@ -157,7 +157,7 @@ func testInvokeDockerVM(t *testing.T, tmpConfigFile string, workingDirectory str
157157
stdErr.Reset()
158158

159159
t.Log("destroying vm with docker")
160-
destroyCmd := exec.Command("invoke", "aws.destroy-docker", "--stack-name", stackName, "--config-path", tmpConfigFile)
160+
destroyCmd := exec.Command("dda", "inv", "aws.destroy-docker", "--stack-name", stackName, "--config-path", tmpConfigFile)
161161
destroyCmd.Dir = workingDirectory
162162
destroyCmd.Stdout = &stdOut
163163
destroyCmd.Stderr = &stdErr
@@ -174,13 +174,13 @@ func testInvokeKind(t *testing.T, tmpConfigFile string, workingDirectory string)
174174
stackName := strings.Join(stackParts, "-")
175175
stackName = sanitizeStackName(stackName)
176176
t.Log("creating kind cluster")
177-
createCmd := exec.Command("invoke", "aws.create-kind", "--no-interactive", "--stack-name", stackName, "--config-path", tmpConfigFile, "--use-fakeintake", "--use-loadBalancer", "--install-argorollout")
177+
createCmd := exec.Command("dda", "inv", "aws.create-kind", "--no-interactive", "--stack-name", stackName, "--config-path", tmpConfigFile, "--use-fakeintake", "--use-loadBalancer", "--install-argorollout")
178178
createCmd.Dir = workingDirectory
179179
createOutput, err := createCmd.Output()
180180
assert.NoError(t, err, "Error found creating kind cluster: %s", string(createOutput))
181181

182182
t.Log("destroying kind cluster")
183-
destroyCmd := exec.Command("invoke", "aws.destroy-kind", "--stack-name", stackName, "--config-path", tmpConfigFile)
183+
destroyCmd := exec.Command("dda", "inv", "aws.destroy-kind", "--stack-name", stackName, "--config-path", tmpConfigFile)
184184
destroyCmd.Dir = workingDirectory
185185
destroyOutput, err := destroyCmd.Output()
186186
require.NoError(t, err, "Error found destroying kind cluster: %s", string(destroyOutput))
@@ -194,13 +194,13 @@ func testInvokeKindOperator(t *testing.T, tmpConfigFile string, workingDirectory
194194
}
195195
stackName = sanitizeStackName(stackName)
196196
t.Log("creating kind cluster with operator")
197-
createCmd := exec.Command("invoke", "aws.create-kind", "--install-agent-with-operator", "true", "--no-interactive", "--stack-name", stackName, "--config-path", tmpConfigFile, "--use-fakeintake", "--use-loadBalancer")
197+
createCmd := exec.Command("dda", "inv", "aws.create-kind", "--install-agent-with-operator", "true", "--no-interactive", "--stack-name", stackName, "--config-path", tmpConfigFile, "--use-fakeintake", "--use-loadBalancer")
198198
createCmd.Dir = workingDirectory
199199
createOutput, err := createCmd.Output()
200200
assert.NoError(t, err, "Error found creating kind cluster: %s; %s", string(createOutput), err)
201201

202202
t.Log("destroying kind cluster with operator")
203-
destroyCmd := exec.Command("invoke", "aws.destroy-kind", "--stack-name", stackName, "--config-path", tmpConfigFile)
203+
destroyCmd := exec.Command("dda", "inv", "aws.destroy-kind", "--stack-name", stackName, "--config-path", tmpConfigFile)
204204
destroyCmd.Dir = workingDirectory
205205
destroyOutput, err := destroyCmd.Output()
206206
require.NoError(t, err, "Error found destroying kind cluster: %s", string(destroyOutput))
@@ -240,7 +240,7 @@ func createTemporaryConfigurationFile() (string, error) {
240240
func setupTestInfra(tmpConfigFile string) error {
241241
var setupStdout, setupStderr bytes.Buffer
242242

243-
setupCmd := exec.Command("invoke", "setup", "--no-interactive", "--config-path", tmpConfigFile)
243+
setupCmd := exec.Command("dda", "inv", "setup", "--no-interactive", "--config-path", tmpConfigFile)
244244
setupCmd.Stdout = &setupStdout
245245
setupCmd.Stderr = &setupStderr
246246

‎requirements.txt‎

Lines changed: 0 additions & 11 deletions
This file was deleted.

‎tasks/tool.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ def is_windows():
2525
just_fix_windows_console()
2626
except ImportError:
2727
print(
28-
"colorama is not up to date, terminal colors may not work properly. Please run 'pip install -r requirements.txt' to fix this."
28+
"colorama is not up to date, terminal colors may not work properly. Please run 'dda self dep sync' to fix this."
2929
)
3030

3131

0 commit comments

Comments
 (0)