Release/0.9.4

.gitattributes

@@ -1,3 +1 @@
-test/test_data/*.pkl filter=lfs diff=lfs merge=lfs -text
-test/test_results/*.txt filter=lfs diff=lfs merge=lfs -text
 *.sh text eol=lf

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Install dependencies
         run: |
           pip install flake8 black sqlfluff yamllint pylint check-jsonschema
-          npm install -g markdownlint-cli
+          npm install -g markdownlint-cli2
 
       - name: Lint Python
         run: flake8 --config=.flake8 src/ test/ || exit 1
@@ -44,7 +44,7 @@ jobs:
         run: yamllint src || exit 1
 
       - name: Lint Markdown
-        run: markdownlint '**/*.md' || exit 1
+        run: markdownlint-cli2 '[^.]*/**/*.md' '*.md' --config .markdownlint.json || exit 1
 
       - name: Lint BOM files
         run: find src -name "bom.yml" -exec sh -c 'printf "%-50s " "$$1"; check-jsonschema --schemafile test/src-bom.schema.json "$$1" || check-jsonschema --schemafile test/src-bom.schema.json "$$1"' _ {} \;
@@ -81,7 +81,7 @@ jobs:
       - name: Install dependencies
         run: |
           pip install -r requirements.txt
-          pip install pytest-mock
+          pip install pytest-mock flake8 black pylint
 
       - name: Install system dependencies
         run: |
@@ -91,6 +91,32 @@ jobs:
       - name: Run bash tests
         run: pytest test/core/test_bash_scripts.py -v
 
+  test-bash-slow:
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/devel' || startsWith(github.ref, 'refs/heads/release/')
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11.12"
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+          pip install pytest-mock flake8 black pylint
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y bats jq gawk pandoc zip
+          npm install -g markdownlint-cli2
+
+      - name: Run slow bash tests (build/package/compile)
+        run: pytest test/core/test_bash_scripts.py -v --run-slow
+
   test-core:
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -1,29 +1,47 @@
-connection.json
+# === macOS ===
 .DS_Store
+
+# === Python ===
+*.pyc
+__pycache__/
 .venv*
-*.token
+
+# === Configuration Files ===
+connection.json
 config.json
 config-*.json
 config.yml
 config-*.yml
 conf/*.sql
 snowflake.local.yml
-output/**
+test/conf/*config-download.yml
+# Safety net: prevent accidental re-introduction of binary pickle fixture files
+test/test_data/*.pkl
+!config-basic.json
+
+# === Credentials & Secrets ===
+*.token
+*credentials*.*
+
+# === Logs ===
 log.*
 *.log
-*credentials*.*
-*.pyc
+.logs/*
+
+# === Build Artifacts ===
 build/*
 package/*
-.logs/*
-instruments-def.*
+output/**
 metrics/*
-*.zip
-!config-basic.json
-LICENSE.md
+instruments-def.*
 *-deploy-script*
-*.pdf
-test/conf/*config-download.yml
 dynatrace-snowflake-observability-agent
+
+# === Distribution & Archives ===
+*.zip
+*.whl
 *.pkg
+*.pdf
+
+# === AI & Development Context ===
 .github/context/*

.vscode/settings.json

@@ -1,6 +1,4 @@
 {
-    "python.envFile": "${workspaceFolder}/.venv/bin/python",
-    "python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python",
     "python.testing.pytestEnabled": true,
     "python.testing.unittestEnabled": false,
     "python.testing.pytestArgs": [
@@ -216,5 +214,8 @@
     },
     "[yaml]": {
         "editor.defaultFormatter": "esbenp.prettier-vscode"
+    },
+    "chat.promptFilesLocations": {
+        ".github/context/prompts": true
     }
 }

.windsurf/workflows/project-context.md

@@ -0,0 +1,16 @@
+---
+description: Load DSOA project instructions and coding standards for all work
+---
+
+Before starting any task, read and follow the project instructions:
+
+1. Read `.github/copilot-instructions.md` for complete project context including:
+   - Architecture (plugin-based, triad pattern)
+   - Code style requirements (black, flake8, pylint 10.00/10)
+   - Testing requirements (pytest, dual-mode mock/live)
+   - Documentation standards
+   - Delivery process (Proposal → Plan → Implementation)
+
+2. Always use the Python virtual environment at `.venv/`
+
+3. Run `make lint` before considering any change complete

Makefile

@@ -16,10 +16,13 @@ lint-yaml:
 	yamllint src
 
 lint-markdown:
-	markdownlint '**/*.md' --config .markdownlint.json
+	markdownlint-cli2 '[^.]*/**/*.md' '*.md' --config .markdownlint.json
 
 lint-bom:
 	find src -name "bom.yml" -exec sh -c 'printf "%-50s " "$$1"; .venv/bin/check-jsonschema --schemafile test/src-bom.schema.json "$$1" || check-jsonschema --schemafile test/src-bom.schema.json "$$1"' _ {} \;
 
 # Run all linting checks (stops on first failure, like CI)
-lint: lint-python lint-format lint-pylint lint-sql lint-yaml lint-markdown lint-bom
+lint: lint-python lint-format lint-pylint lint-sql lint-yaml lint-markdown lint-bom
+
+docs:
+	./scripts/dev/build_docs.sh

README.md

@@ -22,4 +22,5 @@ analyzing, and detecting anomalies in data processing. It delivers observability
 - [Version changelog](docs/CHANGELOG.md)
 - [Contribution guidelines](docs/CONTRIBUTING.md)
 - [Plugin development guide](docs/PLUGIN_DEVELOPMENT.md)
+- [Development log](docs/DEVLOG.md)
 - [Appendix and reference](docs/APPENDIX.md)

docs/CHANGELOG.md

@@ -2,6 +2,43 @@
 
 All notable changes to this project will be documented in this file.
 
+## Dynatrace Snowflake Observability Agent 0.9.4
+
+Released on TBD
+
+> **Note**: Detailed technical changes and implementation notes are available in [DEVLOG.md](DEVLOG.md).
+
+### New in 0.9.4
+
+- **New Plugins**: Added Pipes, Streams, Stage, and Data Lineage monitoring plugins
+- **Configurable Lookback Time**: Per-plugin configuration for historical data catchup window
+- **SNOWFLAKE.TELEMETRY.EVENTS Support**: Agent now correctly reads from the Snowflake-managed shared event table when it is configured as the account-level event table
+
+### Fixed in 0.9.4
+
+- **Dynamic Tables — Grant Granularity**: `P_GRANT_MONITOR_DYNAMIC_TABLES()` now derives grant scope from the `include` pattern. `DB.%.%` grants at database level, `DB.SCHEMA.%` at schema level, and `DB.SCHEMA.TABLE` on a specific named table only — eliminating previous over-granting when a schema or table was explicitly specified.
+- **Span Timestamp Handling**: Fixed spans being re-processed after agent restart due to incorrect timestamp being recorded as last-processed marker
+- **OTLP Compliance**: Fixed log `observed_timestamp` field to use nanoseconds per OTLP specification
+
+### Changed in 0.9.4
+
+- **Event Log Plugin — Cross-Tenant Monitoring** *(behavior change)*: DSOA instances now report `WARN`/`ERROR` log entries, metrics, and spans from all other `DTAGENT_*_DB` instances by default. Use `plugins.event_log.cross_tenant_monitoring: false` to opt out. It is recommended to keep this enabled in only one primary DSOA tenant to avoid duplicate reporting across deployments.
+- **Shares Plugin**: Fixed inbound shares with deleted databases not being properly reported. The `snowflake.share.has_details_reported` attribute now correctly shows `TRUE` for deleted-DB shares, and the `_MESSAGE` field provides clear context about database deletion status
+- **Self-Monitoring**: Fixed database name filtering for self-monitoring logs
+
+### Improved in 0.9.4
+
+- **Budgets Plugin**: Enhanced budget data collection using `SYSTEM$SHOW_BUDGETS_IN_ACCOUNT()`.
+- **Query Hierarchy Validation**: Confirmed and validated span hierarchy for nested stored procedure call chains (`IS_ROOT`/`IS_PARENT` flags) with dedicated test coverage for OTel parent-child propagation.
+- **Error Handling — Two-Phase Commit**: Query telemetry is now marked as processed only after the OTLP flush succeeds, preventing silent data loss when trace export fails.
+- **Event Log Lookback — Configurable**: The event log lookback window (previously hardcoded to 24 h) is now driven by `plugins.event_log.lookback_hours` config key.
+- **Test Infrastructure**: Refactored tests to use synthetic JSON fixtures for input/output validation instead of live Dynatrace API calls.
+- **Test Fixtures**: Migrated all plugin test input data from binary Python pickle files (`.pkl`) to human-readable NDJSON format (`.ndjson`), improving transparency and enabling direct manual inspection and version control of test data.
+- **Event Tables Cost Optimization**: Added guidance for fine-tuning Event Table usage to manage Snowflake costs.
+- **Timestamp Handling**: Unified timestamp handling with smart unit detection, eliminating wasteful conversions
+- **Build System**: Development scripts now auto-activate virtual environment
+- **Test Infrastructure**: Refactored tests to use synthetic JSON fixtures instead of live API calls
+
 ## Dynatrace Snowflake Observability Agent 0.9.3
 
 Released on February 12, 2026
@@ -166,7 +203,7 @@ Released on May 20, 2025.
 
 - **Teardown Process**: Correctly tears down tagged instances.
 - **Span Event Reporting**: Removed the hard limit of 128 span events. The limit is now configurable via `OTEL.SPANS.MAX_EVENT_COUNT`.
-- **Spans for Queries**: Fixed the problem with a hierarchy of query calls not being represented by a hierarchy of spans (_0.8.2 Hotfix 1_).
+- **Spans for Queries**: Fixed the problem with a hierarchy of query calls not being represented by a hierarchy of spans (*0.8.2 Hotfix 1*).
 - **Self-Monitoring Configuration**: Plugin default configurations no longer overwrite self-monitoring settings.
 - **Self-Monitoring BizEvents**: BizEvents are now sent by default when Dynatrace Snowflake Observability Agent is deployed and executed.
 
@@ -293,7 +330,7 @@ Released on Oct 8, 2024.
 
 ### Added in 0.7.2
 
-- Pickle for testing of the Users plugin.
+- Fixtures for testing of the Users plugin.
 - Copyright statements to the code.
 
 ### Fixed in 0.7.2

docs/CONTRIBUTING.md

@@ -203,20 +203,20 @@ pytest test/plugins/
 ./scripts/dev/test.sh test_budgets
 ```
 
-**Regenerate test data (Pickles):**
-If you modify a plugin's SQL logic, you may need to update the test data.
+**Regenerate NDJSON fixtures:**
+If you modify a plugin's SQL logic, you may need to regenerate its fixture data from a live Snowflake environment.
 
 ```bash
 ./scripts/dev/test.sh test_budgets -p
 ```
 
 ### Test Data
 
-Tests use example test data from the `test/test_data` folder:
+Tests use NDJSON fixture files from the `test/test_data/` folder. Each fixture file contains one JSON object per line, named `{plugin_name}[_{view_suffix}].ndjson`.
 
-- Pickle (`*.pkl`) files are used for test execution
-- ndJSON files are provided for reference only
-- Test results are validated against expected data in `test_results`
+Fixtures are version-controlled. To regenerate them from a live Snowflake environment, run the relevant plugin test with the `-p` flag (requires `test/credentials.yml`).
+
+Expected telemetry output is stored in `test/test_results/test_<plugin>/` as JSON files and used for regression comparison.
 
 ### Setting Up Test Environment
 
@@ -246,7 +246,7 @@ To run tests in live mode:
 3. **Generate `test/conf/config-download.yml`** by running:
 
    ```bash
-   PYTHONPATH="./src" pytest -s -v "test/core/test_config.py::TestConfig::test_init" --pickle_conf y
+   PYTHONPATH="./src" pytest -s -v "test/core/test_config.py::TestConfig::test_init" --save_conf y
    ```
 
 ### Running Tests in Local Mode
@@ -373,7 +373,10 @@ Before submitting a PR, please ensure:
 - [ ] You have added tests for any new functionality
 - [ ] All tests pass locally (`pytest` and `./test/bash/run_tests.sh`)
 - [ ] Documentation (`README.md`, `PLUGIN_DEVELOPMENT.md`, etc.) is updated if needed
+- [ ] User-facing changes are documented in `docs/CHANGELOG.md` (highlights only)
+- [ ] Technical implementation details are documented in `docs/DEVLOG.md`
 - [ ] If adding a plugin, `instruments-def.yml` is defined and valid
+- [ ] New use cases are documented in `docs/USECASES.md` under the appropriate Data Platform Observability theme(s)
 - [ ] Code follows the [Semantic Conventions](#semantic-conventions)
 - [ ] If changing SQL objects, all names are UPPERCASE
 - [ ] If adding new semantic fields, they follow naming rules