From 533f1659921d8c67ad4385cbea8689681c51b6ee Mon Sep 17 00:00:00 2001 From: Michael-J-Ward Date: Tue, 1 Oct 2024 13:48:47 -0500 Subject: [PATCH 1/5] make script shebangs more flexible --- dev/python_lint.sh | 2 +- dev/release/verify-release-candidate.sh | 2 +- dev/rust_lint.sh | 2 +- docs/build.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/python_lint.sh b/dev/python_lint.sh index 3bc67fb12..29f0d4833 100755 --- a/dev/python_lint.sh +++ b/dev/python_lint.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 14c0baee8..3879a267f 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/dev/rust_lint.sh b/dev/rust_lint.sh index b1285cbc3..eeb9e2302 100755 --- a/dev/rust_lint.sh +++ b/dev/rust_lint.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/docs/build.sh b/docs/build.sh index 7e8bb0b54..5afe85812 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file From 2c04ae2d4d949d47ed42b75d83034315c406b1aa Mon Sep 17 00:00:00 2001 From: Michael-J-Ward Date: Tue, 1 Oct 2024 14:04:27 -0500 Subject: [PATCH 2/5] move location of python tests This change allows us to run the test suite against a release candidate hosted on test.pypi.org. ``` pip install --extra-index-url https://test.pypi.org/simple/ datafusion==40.0.0 pytest --import-mode=importlib python/tests ``` --- python/{datafusion => }/tests/__init__.py | 0 python/{datafusion => }/tests/conftest.py | 0 python/{datafusion => }/tests/data_test_context/data.json | 0 python/{datafusion => }/tests/generic.py | 0 python/{datafusion => }/tests/test_aggregation.py | 0 python/{datafusion => }/tests/test_catalog.py | 0 python/{datafusion => }/tests/test_config.py | 0 python/{datafusion => }/tests/test_context.py | 0 python/{datafusion => }/tests/test_dataframe.py | 0 python/{datafusion => }/tests/test_expr.py | 0 python/{datafusion => }/tests/test_functions.py | 0 python/{datafusion => }/tests/test_imports.py | 0 python/{datafusion => }/tests/test_indexing.py | 0 python/{datafusion => }/tests/test_input.py | 0 python/{datafusion => }/tests/test_sql.py | 0 python/{datafusion => }/tests/test_store.py | 0 python/{datafusion => }/tests/test_substrait.py | 0 python/{datafusion => }/tests/test_udaf.py | 0 python/{datafusion => }/tests/test_udwf.py | 0 python/{datafusion => }/tests/test_wrapper_coverage.py | 0 20 files changed, 0 insertions(+), 0 deletions(-) rename python/{datafusion => }/tests/__init__.py (100%) rename python/{datafusion => }/tests/conftest.py (100%) rename python/{datafusion => }/tests/data_test_context/data.json (100%) rename python/{datafusion => }/tests/generic.py (100%) rename python/{datafusion => }/tests/test_aggregation.py (100%) rename python/{datafusion => }/tests/test_catalog.py (100%) rename python/{datafusion => }/tests/test_config.py (100%) rename python/{datafusion => }/tests/test_context.py (100%) rename python/{datafusion => }/tests/test_dataframe.py (100%) rename python/{datafusion => }/tests/test_expr.py (100%) rename python/{datafusion => }/tests/test_functions.py (100%) rename python/{datafusion => }/tests/test_imports.py (100%) rename python/{datafusion => }/tests/test_indexing.py (100%) rename python/{datafusion => }/tests/test_input.py (100%) rename python/{datafusion => }/tests/test_sql.py (100%) rename python/{datafusion => }/tests/test_store.py (100%) rename python/{datafusion => }/tests/test_substrait.py (100%) rename python/{datafusion => }/tests/test_udaf.py (100%) rename python/{datafusion => }/tests/test_udwf.py (100%) rename python/{datafusion => }/tests/test_wrapper_coverage.py (100%) diff --git a/python/datafusion/tests/__init__.py b/python/tests/__init__.py similarity index 100% rename from python/datafusion/tests/__init__.py rename to python/tests/__init__.py diff --git a/python/datafusion/tests/conftest.py b/python/tests/conftest.py similarity index 100% rename from python/datafusion/tests/conftest.py rename to python/tests/conftest.py diff --git a/python/datafusion/tests/data_test_context/data.json b/python/tests/data_test_context/data.json similarity index 100% rename from python/datafusion/tests/data_test_context/data.json rename to python/tests/data_test_context/data.json diff --git a/python/datafusion/tests/generic.py b/python/tests/generic.py similarity index 100% rename from python/datafusion/tests/generic.py rename to python/tests/generic.py diff --git a/python/datafusion/tests/test_aggregation.py b/python/tests/test_aggregation.py similarity index 100% rename from python/datafusion/tests/test_aggregation.py rename to python/tests/test_aggregation.py diff --git a/python/datafusion/tests/test_catalog.py b/python/tests/test_catalog.py similarity index 100% rename from python/datafusion/tests/test_catalog.py rename to python/tests/test_catalog.py diff --git a/python/datafusion/tests/test_config.py b/python/tests/test_config.py similarity index 100% rename from python/datafusion/tests/test_config.py rename to python/tests/test_config.py diff --git a/python/datafusion/tests/test_context.py b/python/tests/test_context.py similarity index 100% rename from python/datafusion/tests/test_context.py rename to python/tests/test_context.py diff --git a/python/datafusion/tests/test_dataframe.py b/python/tests/test_dataframe.py similarity index 100% rename from python/datafusion/tests/test_dataframe.py rename to python/tests/test_dataframe.py diff --git a/python/datafusion/tests/test_expr.py b/python/tests/test_expr.py similarity index 100% rename from python/datafusion/tests/test_expr.py rename to python/tests/test_expr.py diff --git a/python/datafusion/tests/test_functions.py b/python/tests/test_functions.py similarity index 100% rename from python/datafusion/tests/test_functions.py rename to python/tests/test_functions.py diff --git a/python/datafusion/tests/test_imports.py b/python/tests/test_imports.py similarity index 100% rename from python/datafusion/tests/test_imports.py rename to python/tests/test_imports.py diff --git a/python/datafusion/tests/test_indexing.py b/python/tests/test_indexing.py similarity index 100% rename from python/datafusion/tests/test_indexing.py rename to python/tests/test_indexing.py diff --git a/python/datafusion/tests/test_input.py b/python/tests/test_input.py similarity index 100% rename from python/datafusion/tests/test_input.py rename to python/tests/test_input.py diff --git a/python/datafusion/tests/test_sql.py b/python/tests/test_sql.py similarity index 100% rename from python/datafusion/tests/test_sql.py rename to python/tests/test_sql.py diff --git a/python/datafusion/tests/test_store.py b/python/tests/test_store.py similarity index 100% rename from python/datafusion/tests/test_store.py rename to python/tests/test_store.py diff --git a/python/datafusion/tests/test_substrait.py b/python/tests/test_substrait.py similarity index 100% rename from python/datafusion/tests/test_substrait.py rename to python/tests/test_substrait.py diff --git a/python/datafusion/tests/test_udaf.py b/python/tests/test_udaf.py similarity index 100% rename from python/datafusion/tests/test_udaf.py rename to python/tests/test_udaf.py diff --git a/python/datafusion/tests/test_udwf.py b/python/tests/test_udwf.py similarity index 100% rename from python/datafusion/tests/test_udwf.py rename to python/tests/test_udwf.py diff --git a/python/datafusion/tests/test_wrapper_coverage.py b/python/tests/test_wrapper_coverage.py similarity index 100% rename from python/datafusion/tests/test_wrapper_coverage.py rename to python/tests/test_wrapper_coverage.py From 5e048b0fca43b934f1541d3f76323f4624c05dd5 Mon Sep 17 00:00:00 2001 From: Michael-J-Ward Date: Tue, 1 Oct 2024 15:03:36 -0500 Subject: [PATCH 3/5] add documentation for testing against release candidate --- dev/release/README.md | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/dev/release/README.md b/dev/release/README.md index 93c2f97b9..49fd9de2d 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -159,10 +159,29 @@ Send the email to start the vote. ## Verifying a Release -Install the release from testpypi: +Running the unit tests against a testpypi release candidate: ```bash -pip install --extra-index-url https://test.pypi.org/simple/ datafusion==0.7.0 +# clone a fresh repo +git clone https://github.com/apache/datafusion-python.git +cd datafusion-python + +# checkout the release commit +git fetch --tags +git checkout 40.0.0-rc1 + +# create the env +python3 -m venv venv +source venv/bin/activate + +# install release candidate +pip install --extra-index-url https://test.pypi.org/simple/ datafusion==40.0.0 + +# only dep needed to run tests is pytest +pip install pytest + +# run the tests +pytest --import-mode=importlib python/tests ``` Try running one of the examples from the top-level README, or write some custom Python code to query some available From 51cd80889603709d60d75204a6f62048d7ff4a2c Mon Sep 17 00:00:00 2001 From: Michael-J-Ward Date: Tue, 1 Oct 2024 15:04:11 -0500 Subject: [PATCH 4/5] add dev script for cleaning maturin build artifacts --- dev/clean.sh | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100755 dev/clean.sh diff --git a/dev/clean.sh b/dev/clean.sh new file mode 100755 index 000000000..0d86680e8 --- /dev/null +++ b/dev/clean.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# This cleans up the project by removing build artifacts and other generated files. + +# Function to remove a directory and print the action +remove_dir() { + if [ -d "$1" ]; then + echo "Removing directory: $1" + rm -rf "$1" + fi +} + +# Function to remove a file and print the action +remove_file() { + if [ -f "$1" ]; then + echo "Removing file: $1" + rm -f "$1" + fi +} + +# Remove .pytest_cache directory +remove_dir .pytest_cache/ + +# Remove target directory +remove_dir target/ + +# Remove any __pycache__ directories +find python/ -type d -name "__pycache__" -print | while read -r dir; do + remove_dir "$dir" +done + +# Remove pytest-coverage.lcov file +# remove_file .coverage +# remove_file pytest-coverage.lcov + +# Remove rust-coverage.lcov file +# remove_file rust-coverage.lcov + +# Remove pyo3 files +find python/ -type f -name '_internal.*.so' -print | while read -r file; do + remove_file "$file" +done + +echo "Cleanup complete." \ No newline at end of file From ca8834d04af1d2068c1aba9d299a33448fb26630 Mon Sep 17 00:00:00 2001 From: Michael-J-Ward Date: Tue, 1 Oct 2024 15:08:32 -0500 Subject: [PATCH 5/5] update ruff lint config for new test location --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4e03ce8db..6e10333a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,7 +78,7 @@ max-doc-length = 88 # Disable docstring checking for these directories [tool.ruff.lint.per-file-ignores] -"python/datafusion/tests/*" = ["D"] +"python/tests/*" = ["D"] "examples/*" = ["D", "W505"] "dev/*" = ["D"] "benchmarks/*" = ["D", "F"]