Added docs and tasks

Author: AsafMah

.env.example

@@ -0,0 +1,22 @@
+# For e2e tests:
+ENGINE_CONNECTION_STRING= # Format: https://<cluster_name>.<region>.kusto.windows.net
+DM_CONNECTION_STRING= # Optional - if not set, will infer from ENGINE_CONNECTION_STRING. Format: https://ingest-<cluster_name>.<region>.kusto.windows.net
+TEST_DATABASE=
+
+# For credential tests:
+
+# app key provider:
+AZURE_CLIENT_ID=
+AZURE_CLIENT_SECRET=
+AZURE_TENANT_ID=
+# certificate provider:
+CERT_THUMBPRINT=
+CERT_PUBLIC_CERT_PATH=
+CERT_PEM_KEY_PATH=
+# managed identity provider:
+MSI_OBJECT_ID=
+MSI_CLIENT_ID=
+# user password provider:
+USER_NAME=
+USER_PASS=
+USER_AUTH_ID=

.github/workflows/build.yml

@@ -35,27 +35,25 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           enable-cache: true
-      - name: Install the project
-        run: uv sync --locked --all-extras --dev
       - name: EtoE Test with pytest
         env:
           TEST_DATABASE: ${{ secrets.TEST_DATABASE }}
           TEST_BLOB: ${{ secrets.TEST_BLOB }}
           ENGINE_CONNECTION_STRING: ${{ secrets.ENGINE_CONNECTION_STRING }}
           APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING: https://ade.applicationinsights.io/subscriptions/12534eb3-8109-4d84-83ad-576c0d5e1d06/resourcegroups/clients_e2e_test/providers/microsoft.insights/components/kusto-e2e-app-insights
           APPLICATION_INSIGHTS_TEST_DATABASE: kusto-e2e-app-insights
-        run: uv run pytest --junit-xml pytest.xml
-
-#      todo: re-enable on a different PR
-#      - name: "Run Ruff Formatter"
-#        id: ruff-formatter
-#        uses: astral-sh/ruff-action@v3
-#        with:
-#          args: "format --diff"
-#          
-#      - name: "Run Ruff Linter"
-#        run: ruff check --diff        
+        run: uvx poe test --ci
 
+      #      todo: re-enable on a different PR
+      #      - name: "Run Ruff Formatter"
+      #        id: ruff-formatter
+      #        uses: astral-sh/ruff-action@v3
+      #        with:
+      #          args: "format --diff"
+      #          
+      #      - name: "Run Ruff Linter"
+      #        run: ruff check --diff        
+
       - name: Upload Unit Test Results
         if: always()
         uses: actions/upload-artifact@v4

.gitignore

@@ -308,4 +308,6 @@ build/
 .mypy*
 
 
-*/.tox/
+*/.tox/
+
+.env

CONTRIBUTING.md

@@ -4,25 +4,60 @@ If you would like to become an active contributor to this project please
 follow the instructions provided in [Microsoft Azure Projects Contribution Guidelines](https://azure.github.io/azure-sdk/python_documentation.html).
 
 ## Prerequisites
+
+### Cluster
 In order to run E2E tests, you need a Kusto database you have admin rights on.
 A Kusto free cluster is the easiest way to acquire one.
 You can cerate a free Kusto cluster here: https://dataexplorer.azure.com/home
 
 Make sure you set streaming ingestion to enabled in the cluster's configuration:  
 https://learn.microsoft.com/en-us/azure/data-explorer/ingest-data-streaming?tabs=azure-portal%2Ccsharp
 
-You should set then following environment vars where you run E2Es (in IDE run config, shell window, computer, etc).
+
+### Tools
+
+The project uses [uv](https://docs.astral.sh/uv/) as the project manager.
+You will need to install it in order to develop on this project.  
+
+See [Installing uv](https://docs.astral.sh/uv/getting-started/installation/).
+
+Additionally, the project uses [poethepoet](https://poethepoet.natn.io/index.html) as the task runner.
+
+You can install it with uv:
+
+```bash
+uv tool install poethepoet
+
+poe <command>
+```
+
+Or you can use `uvx` to use it directly without installing:
+
+```bash
+uvx poe <command>
+```
+
+
+### Environment Variables
+
+Some tests require you to set environment variables to run.
+
+The project supports a .env file, which you can create in the root of the project.  
+See [.env.example](./.env.example) for an example of a .env file.
+
+#### E2E Tests Environment Variables
+
 ```shell
-ENGINE_CONNECTION_STRING=<Your cluster URI>
-DM_CONNECTION_STRING=<Your ingest cluster URI> # Optional - if not set, will infer from ENGINE_CONNECTION_STRING
+ENGINE_CONNECTION_STRING=<Your cluster URI> # Format: https://<cluster_name>.<region>.kusto.windows.net
+DM_CONNECTION_STRING=<Your ingest cluster URI> # Optional - if not set, will infer from ENGINE_CONNECTION_STRING. Format: https://ingest-<cluster_name>.<region>.kusto.windows.net
 TEST_DATABASE=<The name of the database>
 ```
 
-The E2E tests authenticate with DefaultAzureCredential, and will fall back to interactive login if needed.
+The E2E tests authenticate with [DefaultAzureCredential](https://learn.microsoft.com/en-us/python/api/azure-identity/azure.identity.defaultazurecredential?view=azure-python).
 
+It is recommended to use the [azure cli](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest) command `az login` to authenticate with your Azure account to run the tests.
 
-For more information on DefaultAzureCredential, see:  
-https://learn.microsoft.com/en-us/python/api/azure-identity/azure.identity.defaultazurecredential?view=azure-python
+#### Token Provider Tests
 
 To run the optional `token_provider` tests, you will need to set the booleans at the top of the file `test_token_providers.py` and the following environment variables in addition to the previous ones:
 ```shell
@@ -43,41 +78,61 @@ USER_PASS=<Your user password>
 USER_AUTH_ID=<Your user auth ID> # optional
 ```
 
-## Requirements
 
-In order to work on this project, we recommend using the dev requirements:
+## Development
+
+As mentioned before, this project uses `uv` and `poe` to manage the development environment and tasks.
+
+To see the tasks available, you can run:
+
+```bash
+poe
+```
+
+### Setup
+
+Setup the project by syncing `uv`:
 
 ```bash
-pip install -r dev_requirements.txt
+poe sync
 ```
 
-These including testing related packages as well as styling ([black](https://black.readthedocs.io/en/stable/))
+This will install the dependencies, set up the virtual environment, and install the tools needed for development.
 
-## Building and Testing
+### Testing
 
 This project uses [pytest](https://docs.pytest.org/en/latest/).
 
+To test the project, simply run:
+
+```bash
+poe test
+```
 
-Since the tests use the package as a third-party, the easiest way to set it up is installing it in edit mode:
+To test without E2E tests, you can run:
 
 ```bash
-pip install -e ./azure-kusto-data ./azure-kusto-ingest
+poe test --no-e2e
 ```
 
-After which, running tests is simple.
+### Style
 
-Just run:
+This project uses [ruff](https://docs.astral.sh/ruff/) for linting and formatting.
+
+Before commiting your changes, make sure the code is properly formatted, or the commit will be rejected.
 
 ```bash
-pytest ./azure-kusto-data ./azure-kusto-ingest 
+    poe format # formats the code directly
+    poe check-format # returns a diff of what needs to be formatted    
 ```
 
-## Style
-
-We use black, and allow for line-length of 160, so please run:
+Also make sure to lint the code:
 
 ```bash
-black --line-length=160 ./azure-kusto-data ./azure-kusto-ingest
+    poe lint
+    
+    # You can auto-fix some issues with:
+    poe lint --fix
 ```
 
 ## PRs
@@ -90,7 +145,7 @@ We welcome contributions. In order to make the PR process efficient, please foll
 * **Please provide any related information needed to understand the change** - docs, guidelines, use-case, best practices and so on. Opinions are accepted, but have to be backed up.
 * **Checks should pass** - these including linting with black and running tests.
 
-## Code of Conduct
+# Code of Conduct
 This project's code of conduct can be found in the
 [CODE_OF_CONDUCT.md file](https://github.com/Azure/azure-sdk-for-python/blob/master/CODE_OF_CONDUCT.md)
 (v1.4.0 of the http://contributor-covenant.org/ CoC).

pyproject.toml

@@ -17,17 +17,13 @@ email = "[email protected]"
 [dependency-groups]
 dev = [
     "pytest>=8.4.1",
+    "pytest-xdist[psutil]>=3.8.0",
+    "pytest-asyncio>=1.1.0",
     "responses>=0.25.8",
     "aioresponses>=0.7.8",
-    "pytest-asyncio>=1.1.0",
     "asgiref>=3.9.1",
-    "setuptools>=80.9.0",
-    "ipython>=8.12.3",
-    "pyarrow>=21.0.0",
     "pandas>=2.3.1",
     "ruff>=0.12.8",
-    "basedpyright>=1.31.1",
-    "pytest-xdist[psutil]>=3.8.0",
 ]
 
 [tool.uv.workspace]
@@ -51,5 +47,28 @@ ignore = ["E731"] # E731 - dont assign lambdas.
 
 [tool.pytest.ini_options]
 testpaths = ["azure-kusto-data/tests", "azure-kusto-ingest/tests"]
-addopts = ["-s", "--import-mode=importlib", "-n", "8"]
+addopts = ["-s", "--import-mode=importlib", '-n', 'auto']
 pythonpath = ["."]
+
+[tool.poe]
+envfile = ".env"
+
+[tool.poe.tasks]
+sync = "uv sync --all-packages --all-extras --dev --locked"
+build = "uv build --all-packages"
+format = "ruff format"
+check-format = "ruff format --diff"
+lint = "ruff check"
+lint-fix = "ruff check --fix"
+lint-fix-unsafe = "ruff check --fix --unsafe-fixes"
+show-version = "uv version"
+[tool.poe.tasks.test]
+sequence = ["sync", { cmd = 'pytest ${NO_E2E:+ --ignore=azure-kusto-ingest/tests/test_e2e_ingest.py --ignore=azure-kusto-data/tests/test_e2e_data.py } ${CI:+ -n8 --junit-xml pytest.xml}'}]
+
+[tool.poe.tasks.test.args]
+NO_E2E = { help = "Skip E2E tests", type = "boolean", options = ["--no-e2e"] }
+CI = { help = "Run tests in CI mode", type = "boolean", options = ["--ci"] }
+
+[tool.poe.tasks.bump-version]
+sequence = [{ cmd = "uv version --bump $bump" }, { cmd = "uv version --bump $bump --package azure-kusto-data" }, { cmd = "uv version --bump $bump --package azure-kusto-ingest" }, { cmd = "uv version --bump $bump --package quick-start" }]
+args = { bump = { help = "Bump version [possible values: major, minor, patch, stable, alpha, beta, rc, post, dev]", positional = true, default = "patch" } }