migrate Data subpackage (#19523)

* update

* update

* update

* update

* Update checkgroup.yml

* More

* Add note

* Labeller should be kept as long as we have the stubs

* update

* update

* update

* Apply suggestions from code review

* init

* ci fix

* pin version range

* https://www.neptune.ai/

---------

Co-authored-by: Carlos Mocholí <[email protected]>
Co-authored-by: Jirka Borovec <[email protected]>
Co-authored-by: Jirka <[email protected]>

Author: 3 people

.actions/assistant.py

@@ -44,11 +44,7 @@
         "requirements/fabric/base.txt",
         "requirements/fabric/strategies.txt",
     ),
-    "data": (
-        "requirements/data/data.txt",
-        "requirements/data/cloud.txt",
-        "requirements/data/examples.txt",
-    ),
+    "data": ("requirements/data/data.txt",),
 }
 REQUIREMENT_FILES_ALL = list(chain(*REQUIREMENT_FILES.values()))
 

.github/CODEOWNERS

@@ -41,10 +41,8 @@
 /src/lightning/pytorch/core/hooks.py         @williamfalcon @tchaton @awaelchli @carmocca
 /src/lightning/pytorch/core/module.py        @williamfalcon @tchaton @awaelchli @carmocca
 
-# Data Utilities
-/examples/data/           @tchaton @nohalon @justusschock @lantiga
+# Lightning Data
 /src/lightning/data/      @tchaton
-/tests/tests_data         @tchaton
 
 # Lightning Fabric
 /src/lightning/fabric                       @awaelchli @carmocca @justusschock

.github/checkgroup.yml

@@ -150,29 +150,6 @@ subprojects:
       - "build-pl (3.10, 2.0, 11.8.0)"
       - "build-pl (3.10, 2.1, 12.1.0)"
 
-  # SECTIONS: lightning_data
-
-  - id: "lightning_data: CPU workflow"
-    paths:
-      - ".actions/*"
-      - "requirements/data/**"
-      - "src/lightning/data/**"
-      - "src/lightning_data/*"
-      - "src/lightning/__init__.py"
-      - "src/lightning/__setup__.py"
-      - "src/lightning/__version__.py"
-      - "tests/tests_data/**"
-      - "examples/data/**"
-      - "pyproject.toml" # includes pytest config
-      - ".github/workflows/ci-tests-data.yml"
-      - "!requirements/*/docs.txt"
-      - "!*.md"
-      - "!**/*.md"
-    checks:
-      - "data-cpu (macOS-11, lightning, 3.10, 2.1)"
-      - "data-cpu (ubuntu-20.04, lightning, 3.10, 2.1)"
-      - "data-cpu (windows-2022, lightning, 3.10, 2.1)"
-
   # SECTION: lightning_fabric
 
   - id: "lightning_fabric: CPU workflow"

.github/label-change.yml

@@ -14,7 +14,6 @@ data:
   - changed-files:
       - any-glob-to-any-file:
           - "src/lightning/data/**"
-          - "tests/tests_data/**"
           - "requirements/data/**"
 
 store:

.github/workflows/ci-pkg-install.yml

@@ -71,6 +71,7 @@ jobs:
           pkg-folder: dist/${{ env.PKG_DIR }}
           pkg-name: ${{ matrix.pkg-name }}
           pkg-extra: ${{ matrix.pkg-extra }}
+          pip-flags: "--pre" # todo: drop this when `lidata` is released (not as RC only)
 
       - name: Run CLI (via python)
         if: ${{ (matrix.pkg-name == 'lightning' || matrix.pkg-name == 'notset') && matrix.pkg-extra == 'app' }}
@@ -100,13 +101,19 @@ jobs:
           rm -f ./*/__*.py
           rm -f ./**/__*.py
           mv lightning lit  # rename lightning folder to prevent accidental local imports
-      - name: drop App doctest
-        if: ${{ (matrix.pkg-name == 'lightning' || matrix.pkg-name == 'notset') && matrix.pkg-extra != 'app' }}
+      - name: drop Secondary doctest
+        if: ${{ matrix.pkg-name == 'lightning' || matrix.pkg-name == 'notset' }}
         working-directory: src/lit
         run: |
-          rm -rf app
-          rm -rf data
-          rm -rf store
+          items=("data" "store" "app")
+          for item in "${items[@]}"; do
+            if [[ "$item" == "${{ matrix.pkg-extra }}" ]]; then
+              echo "Skipping $item"
+              continue # Skip this iteration
+            fi
+            echo "Removing $item"
+            rm -rf $item
+          done
       - name: Install pytest doctest extension
         run: |
           pip install -q "pytest-doctestplus>=0.9.0"

.github/workflows/ci-tests-data.yml

@@ -193,7 +193,6 @@ our_model.tar
 test.png
 saved_models
 data/
-!src/lightning/data/
 !examples/data/
 !tests/tests_pytorch/utilities/data/
 !requirements/data/

.gitignore

@@ -102,7 +102,6 @@ repos:
         exclude: |
           (?x)^(
               src/lightning/app/CHANGELOG.md|
-              src/lightning/data/CHANGELOG.md|
               src/lightning/fabric/CHANGELOG.md|
               src/lightning/pytorch/CHANGELOG.md|
               README.md

.pre-commit-config.yaml

@@ -62,7 +62,7 @@ Here's the full documentation for the :class:`~lightning.pytorch.loggers.MLFlowL
 
 Neptune.ai
 ==========
-To use `Neptune.ai <https://neptune.ai/>`_ first install the neptune package:
+To use `Neptune.ai <https://www.neptune.ai/>`_ first install the neptune package:
 
 .. code-block:: bash
 

docs/source-pytorch/visualize/supported_exp_managers.rst

@@ -102,9 +102,6 @@ lint.ignore-init-module-imports = true
     "RET504",  # todo:Unnecessary variable assignment before `return` statement
     "RET503",
 ]
-"src/lightning/data/**" = [
-    "S310",  # todo: Audit URL open for permitted schemes. Allowing use of `file:` or custom schemes is often unexpected.
-]
 "tests/**" = [
     "S101",  # Use of `assert` detected
     "S105", "S106",  # todo: Possible hardcoded password: ...