Merge branch 'tensorflow:master' into wake_vision

Author: Ekhao

.github/actions/setup/action.yml

@@ -14,6 +14,10 @@ inputs:
     description: 'Extra dependencies'
     required: false
     default: 'tests-all'
+  use-cache:
+    description: 'Use cache'
+    required: false
+    default: 'true'
 
 runs:
   using: 'composite'
@@ -51,13 +55,18 @@ runs:
     - name: Store inputs for identifying Python cache
       run: echo "${{ env.today }}-${{ inputs.extras }}" >> python-cache-keys.txt
       shell: bash
-    - uses: actions/setup-python@v5
+    - if: ${{ inputs.use-cache == 'true' }}
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ inputs.python-version }}
         cache: 'pip'
         cache-dependency-path: |
           setup.py
           python-cache-keys.txt
+    - if: ${{ inputs.use-cache == 'false' }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python-version }}
 
     # Install Python deps.
     # If Python cache is restored then we don't need to install dependencies again.
@@ -70,12 +79,12 @@ runs:
     # Get TFDS library from the repository instead of cache.
     - run: echo "PYTHONPATH=$(pwd)" >> $GITHUB_ENV
       shell: bash
-    # Get mlcroissant from the repository to get the latest features.
+    # Get mlcroissant from PyPI to fix the version.
     - if: ${{ runner.os == 'Linux' }}
       run: sudo apt-get install -y libgraphviz-dev
       shell: bash
     - if: ${{ runner.os == 'macOS' }}
       run: brew install graphviz
       shell: bash
-    - run: pip install mlcroissant==1.0.1
+    - run: pip install mlcroissant==1.0.5
       shell: bash

.github/workflows/pytest.yml

@@ -79,13 +79,13 @@ jobs:
       # Do not cancel in-progress jobs if any matrix job fails.
       fail-fast: false
       matrix:
-        tf-version: ['tensorflow', 'tf-nightly']
+        tf-version: ['tensorflow']
         # Can't reference env variables in matrix
         num-shards: ${{ fromJson(needs.shards-job.outputs.num-shards) }}
         shard-id: ${{ fromJson(needs.shards-job.outputs.shard-ids) }}
         # TF suppported versions: https://www.tensorflow.org/install/pip#software_requirements
-        python-version: ['3.10', '3.11']
-        os-version: [ubuntu-latest, macos-latest]
+        python-version: ['3.10', '3.11', '3.12']
+        os-version: [ubuntu-latest]
 
     steps:
     - uses: actions/checkout@v3
@@ -185,8 +185,7 @@ jobs:
     - uses: ./.github/actions/setup
       with:
         tf-version: tensorflow
-
-    - run: pip install ipython
+        use-cache: false
 
     # Test each notebook sequentially.
     - name: Run notebook
@@ -202,6 +201,7 @@ jobs:
                 --ExecutePreprocessor.timeout=600 \
                 --ExecutePreprocessor.kernel_name=tfds-notebook \
                 --to notebook \
-                --execute $notebook
+                --execute $notebook && \
+              pip install tensorflow  # reinstall tensorflow if it was uninstalled
             fi
           done

CHANGELOG.md

@@ -10,6 +10,26 @@ and this project adheres to
 
 ### Added
 
+### Changed
+
+### Deprecated
+
+### Removed
+
+### Fixed
+
+### Security
+
+## [4.9.6] - 2024-06-04
+
+### Added
+
+-   Full support for Python 3.12.
+
+## [4.9.5] - 2024-05-30
+
+### Added
+
 -   Support to download and prepare datasets using the
     [Parquet](https://parquet.apache.org) data format.
     ```python
@@ -19,6 +39,16 @@ and this project adheres to
     print(next(iter(ds)))
     ```
 
+-   [`tfds.data_source`](https://www.tensorflow.org/datasets/api_docs/python/tfds/data_source)
+    is pickable, thus working smoothly with
+    [PyGrain](https://github.com/google/grain). Learn more by following the
+    [tutorial](https://www.tensorflow.org/datasets/data_source).
+
+-   TFDS plays nicely with
+    [Croissant](https://mlcommons.org/working-groups/croissant). Learn more by
+    following the
+    [recipe](https://colab.research.google.com/github/mlcommons/croissant/blob/main/python/mlcroissant/recipes/tfds_croissant_builder.ipynb).
+
 ### Changed
 
 ### Deprecated

docs/catalog/_toc.yaml

@@ -33,6 +33,10 @@ toc:
   - path: /datasets/catalog/scientific_papers
     title: scientific_papers
   title: Abstractive text summarization
+- section:
+  - path: /datasets/catalog/dices
+    title: dices
+  title: Age
 - section:
   - path: /datasets/catalog/ag_news_subset
     title: ag_news_subset
@@ -96,6 +100,8 @@ toc:
     title: ogbg_molpcba
   title: Biology
 - section:
+  - path: /datasets/catalog/dices
+    title: dices
   - path: /datasets/catalog/sift1m
     title: sift1m
   title: Categorical
@@ -170,6 +176,8 @@ toc:
     title: bot_adversarial_dialogue
   - path: /datasets/catalog/databricks_dolly
     title: databricks_dolly
+  - path: /datasets/catalog/dices
+    title: dices
   title: Dialogue
 - section:
   - path: /datasets/catalog/aeslc
@@ -195,6 +203,10 @@ toc:
   - path: /datasets/catalog/sun397
     title: sun397
   title: Fine grained image classification
+- section:
+  - path: /datasets/catalog/dices
+    title: dices
+  title: Gender
 - section:
   - path: /datasets/catalog/ogbg_molpcba
     title: ogbg_molpcba
@@ -570,6 +582,8 @@ toc:
     title: blimp
   - path: /datasets/catalog/databricks_dolly
     title: databricks_dolly
+  - path: /datasets/catalog/dices
+    title: dices
   - path: /datasets/catalog/e2e_cleaned
     title: e2e_cleaned
   - path: /datasets/catalog/irc_disentanglement
@@ -626,6 +640,8 @@ toc:
     title: bool_q
   - path: /datasets/catalog/bot_adversarial_dialogue
     title: bot_adversarial_dialogue
+  - path: /datasets/catalog/dices
+    title: dices
   - path: /datasets/catalog/e2e_cleaned
     title: e2e_cleaned
   - path: /datasets/catalog/imdb_reviews
@@ -706,6 +722,8 @@ toc:
 - section:
   - path: /datasets/catalog/anli
     title: anli
+  - path: /datasets/catalog/dices
+    title: dices
   - path: /datasets/catalog/paws_wiki
     title: paws_wiki
   - path: /datasets/catalog/sci_tail
@@ -726,6 +744,8 @@ toc:
     title: clevr
   - path: /datasets/catalog/databricks_dolly
     title: databricks_dolly
+  - path: /datasets/catalog/dices
+    title: dices
   - path: /datasets/catalog/imdb_reviews
     title: imdb_reviews
   - path: /datasets/catalog/math_dataset
@@ -869,6 +889,10 @@ toc:
   - path: /datasets/catalog/trivia_qa
     title: trivia_qa
   title: Question generation
+- section:
+  - path: /datasets/catalog/dices
+    title: dices
+  title: Race national ethnic origin
 - section:
   - path: /datasets/catalog/istella
     title: istella
@@ -942,6 +966,8 @@ toc:
     title: robosuite_panda_pick_place_can
   title: Rlds
 - section:
+  - path: /datasets/catalog/aloha_mobile
+    title: aloha_mobile
   - path: /datasets/catalog/asu_table_top_converted_externally_to_rlds
     title: asu_table_top_converted_externally_to_rlds
   - path: /datasets/catalog/austin_buds_dataset_converted_externally_to_rlds
@@ -970,6 +996,8 @@ toc:
     title: berkeley_rpt_converted_externally_to_rlds
   - path: /datasets/catalog/bridge
     title: bridge
+  - path: /datasets/catalog/bridge_data_msr
+    title: bridge_data_msr
   - path: /datasets/catalog/cmu_franka_exploration_dataset_converted_externally_to_rlds
     title: cmu_franka_exploration_dataset_converted_externally_to_rlds
   - path: /datasets/catalog/cmu_play_fusion
@@ -978,20 +1006,28 @@ toc:
     title: cmu_stretch
   - path: /datasets/catalog/columbia_cairlab_pusht_real
     title: columbia_cairlab_pusht_real
+  - path: /datasets/catalog/conq_hose_manipulation
+    title: conq_hose_manipulation
   - path: /datasets/catalog/dlr_edan_shared_control_converted_externally_to_rlds
     title: dlr_edan_shared_control_converted_externally_to_rlds
   - path: /datasets/catalog/dlr_sara_grid_clamp_converted_externally_to_rlds
     title: dlr_sara_grid_clamp_converted_externally_to_rlds
   - path: /datasets/catalog/dlr_sara_pour_converted_externally_to_rlds
     title: dlr_sara_pour_converted_externally_to_rlds
+  - path: /datasets/catalog/dobbe
+    title: dobbe
   - path: /datasets/catalog/eth_agent_affordances
     title: eth_agent_affordances
+  - path: /datasets/catalog/fmb
+    title: fmb
   - path: /datasets/catalog/fractal20220817_data
     title: fractal20220817_data
   - path: /datasets/catalog/iamlab_cmu_pickup_insert_converted_externally_to_rlds
     title: iamlab_cmu_pickup_insert_converted_externally_to_rlds
   - path: /datasets/catalog/imperialcollege_sawyer_wrist_cam
     title: imperialcollege_sawyer_wrist_cam
+  - path: /datasets/catalog/io_ai_tech
+    title: io_ai_tech
   - path: /datasets/catalog/jaco_play
     title: jaco_play
   - path: /datasets/catalog/kaist_nonprehensile_converted_externally_to_rlds
@@ -1000,6 +1036,8 @@ toc:
     title: kuka
   - path: /datasets/catalog/maniskill_dataset_converted_externally_to_rlds
     title: maniskill_dataset_converted_externally_to_rlds
+  - path: /datasets/catalog/mimic_play
+    title: mimic_play
   - path: /datasets/catalog/mt_opt
     title: mt_opt
   - path: /datasets/catalog/nyu_door_opening_surprising_effectiveness
@@ -1008,8 +1046,14 @@ toc:
     title: nyu_franka_play_dataset_converted_externally_to_rlds
   - path: /datasets/catalog/nyu_rot_dataset_converted_externally_to_rlds
     title: nyu_rot_dataset_converted_externally_to_rlds
+  - path: /datasets/catalog/plex_robosuite
+    title: plex_robosuite
+  - path: /datasets/catalog/robo_set
+    title: robo_set
   - path: /datasets/catalog/roboturk
     title: roboturk
+  - path: /datasets/catalog/spoc_robot
+    title: spoc_robot
   - path: /datasets/catalog/stanford_hydra_dataset_converted_externally_to_rlds
     title: stanford_hydra_dataset_converted_externally_to_rlds
   - path: /datasets/catalog/stanford_kuka_multimodal_dataset_converted_externally_to_rlds
@@ -1020,6 +1064,8 @@ toc:
     title: stanford_robocook_converted_externally_to_rlds
   - path: /datasets/catalog/taco_play
     title: taco_play
+  - path: /datasets/catalog/tidybot
+    title: tidybot
   - path: /datasets/catalog/tokyo_u_lsmo_converted_externally_to_rlds
     title: tokyo_u_lsmo_converted_externally_to_rlds
   - path: /datasets/catalog/toto
@@ -1044,6 +1090,8 @@ toc:
     title: utokyo_xarm_bimanual_converted_externally_to_rlds
   - path: /datasets/catalog/utokyo_xarm_pick_and_place_converted_externally_to_rlds
     title: utokyo_xarm_pick_and_place_converted_externally_to_rlds
+  - path: /datasets/catalog/vima_converted_externally_to_rlds
+    title: vima_converted_externally_to_rlds
   - path: /datasets/catalog/viola
     title: viola
   title: Robotics
@@ -1274,6 +1322,8 @@ toc:
     title: databricks_dolly
   - path: /datasets/catalog/definite_pronoun_resolution
     title: definite_pronoun_resolution
+  - path: /datasets/catalog/dices
+    title: dices
   - path: /datasets/catalog/doc_nli
     title: doc_nli
   - path: /datasets/catalog/dolphin_number_word
@@ -1520,6 +1570,8 @@ toc:
     title: bool_q
   - path: /datasets/catalog/bot_adversarial_dialogue
     title: bot_adversarial_dialogue
+  - path: /datasets/catalog/dices
+    title: dices
   - path: /datasets/catalog/imdb_reviews
     title: imdb_reviews
   - path: /datasets/catalog/natural_instructions
@@ -1536,6 +1588,8 @@ toc:
 - section:
   - path: /datasets/catalog/bot_adversarial_dialogue
     title: bot_adversarial_dialogue
+  - path: /datasets/catalog/dices
+    title: dices
   - path: /datasets/catalog/real_toxicity_prompts
     title: real_toxicity_prompts
   title: Text classification toxicity prediction
@@ -1673,7 +1727,6 @@ toc:
   - path: /datasets/catalog/duke_ultrasound
     title: duke_ultrasound
   - path: /datasets/catalog/qm9
-    status: nightly
     title: qm9
   title: Uncategorized
 - section: