Store example data directly inside the datafusion-examples (#19141)

datafusion-examples/data/README.md

@@ -0,0 +1,27 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+## Example datasets
+
+| Filename                 | Path                                                                      | Description                                                                                                                                                                                                     |
+| ------------------------ | ------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `cars.csv`               | [`data/csv/cars.csv`](./csv/cars.csv)                                     | Time-series–like dataset containing car identifiers, speed values, and timestamps. Used in window function and time-based query examples (e.g. ordering, window frames).                                        |
+| `regex.csv`              | [`data/csv/regex.csv`](./csv/regex.csv)                                   | Dataset for regular expression examples. Contains input values, regex patterns, replacement strings, and optional flags. Covers ASCII, Unicode, and locale-specific text processing.                            |
+| `window_1.csv`           | [`data/csv/window_1.csv`](./csv/window_1.csv)                             | Numeric dataset designed for window function demonstrations. Includes ordering keys and incremental values suitable for running totals, ranking, and frame-based calculations.                                  |
+| `alltypes_plain.parquet` | [`data/parquet/alltypes_plain.parquet`](./parquet/alltypes_plain.parquet) | Parquet file containing columns of many Arrow/DataFusion-supported types (boolean, integers, floating-point, strings, timestamps). Used to demonstrate Parquet scanning, schema inference, and typed execution. |

datafusion-examples/data/csv/cars.csv

@@ -0,0 +1,26 @@
+car,speed,time
+red,20.0,1996-04-12T12:05:03.000000000
+red,20.3,1996-04-12T12:05:04.000000000
+red,21.4,1996-04-12T12:05:05.000000000
+red,21.5,1996-04-12T12:05:06.000000000
+red,19.0,1996-04-12T12:05:07.000000000
+red,18.0,1996-04-12T12:05:08.000000000
+red,17.0,1996-04-12T12:05:09.000000000
+red,7.0,1996-04-12T12:05:10.000000000
+red,7.1,1996-04-12T12:05:11.000000000
+red,7.2,1996-04-12T12:05:12.000000000
+red,3.0,1996-04-12T12:05:13.000000000
+red,1.0,1996-04-12T12:05:14.000000000
+red,0.0,1996-04-12T12:05:15.000000000
+green,10.0,1996-04-12T12:05:03.000000000
+green,10.3,1996-04-12T12:05:04.000000000
+green,10.4,1996-04-12T12:05:05.000000000
+green,10.5,1996-04-12T12:05:06.000000000
+green,11.0,1996-04-12T12:05:07.000000000
+green,12.0,1996-04-12T12:05:08.000000000
+green,14.0,1996-04-12T12:05:09.000000000
+green,15.0,1996-04-12T12:05:10.000000000
+green,15.1,1996-04-12T12:05:11.000000000
+green,15.2,1996-04-12T12:05:12.000000000
+green,8.0,1996-04-12T12:05:13.000000000
+green,2.0,1996-04-12T12:05:14.000000000

datafusion-examples/data/csv/regex.csv

@@ -0,0 +1,12 @@
+values,patterns,replacement,flags
+abc,^(a),bb\1bb,i
+ABC,^(A).*,B,i
+aBc,(b|d),e,i
+AbC,(B|D),e,
+aBC,^(b|c),d,
+4000,\b4([1-9]\d\d|\d[1-9]\d|\d\d[1-9])\b,xyz,
+4010,\b4([1-9]\d\d|\d[1-9]\d|\d\d[1-9])\b,xyz,
+Düsseldorf,[\p{Letter}-]+,München,
+Москва,[\p{L}-]+,Moscow,
+Köln,[a-zA-Z]ö[a-zA-Z]{2},Koln,
+اليوم,^\p{Arabic}+$,Today,

datafusion-examples/data/csv/window_1.csv

@@ -0,0 +1,101 @@
+ts,inc_col,desc_col
+1,1,100
+1,5,98
+5,10,93
+9,15,91
+10,20,86
+11,21,84
+16,26,81
+21,29,77
+22,30,75
+26,33,71
+26,37,70
+28,40,69
+31,43,64
+33,44,62
+38,45,59
+42,49,55
+47,51,50
+51,53,45
+53,58,41
+53,61,40
+58,65,39
+63,70,36
+67,75,31
+68,78,28
+70,83,23
+72,88,22
+72,90,17
+76,91,13
+81,95,10
+85,97,6
+86,100,5
+88,105,2
+91,109,1
+96,111,-1
+97,115,-4
+98,119,-5
+100,120,-6
+101,124,-8
+102,126,-12
+104,129,-16
+104,131,-17
+108,135,-19
+112,140,-24
+113,143,-25
+113,144,-29
+114,147,-34
+114,148,-37
+117,149,-42
+122,151,-47
+126,155,-48
+131,156,-49
+131,159,-53
+136,160,-57
+136,163,-58
+136,165,-61
+139,170,-65
+141,172,-67
+146,177,-68
+147,181,-71
+147,182,-73
+152,186,-75
+154,187,-76
+159,192,-78
+161,196,-83
+163,197,-87
+164,199,-91
+167,203,-95
+172,207,-98
+173,209,-101
+177,213,-105
+180,214,-106
+185,216,-111
+186,219,-114
+191,221,-116
+195,222,-120
+195,225,-125
+199,226,-128
+203,231,-129
+207,236,-134
+210,237,-139
+213,242,-142
+218,245,-143
+221,247,-146
+224,248,-150
+226,253,-154
+230,254,-158
+232,259,-163
+235,261,-168
+238,266,-172
+238,269,-176
+239,272,-181
+244,275,-184
+245,278,-189
+247,283,-193
+250,286,-196
+254,289,-201
+258,291,-203
+262,296,-208
+264,301,-210
+264,305,-213

datafusion-examples/examples/builtin_functions/regexp.rs

@@ -1,5 +1,4 @@
 // Licensed to the Apache Software Foundation (ASF) under one
-// Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
 // distributed with this work for additional information
 // regarding copyright ownership.  The ASF licenses this file
@@ -18,12 +17,11 @@
 
 //! See `main.rs` for how to run it.
 
-use std::{fs::File, io::Write};
+use std::path::PathBuf;
 
 use datafusion::common::{assert_batches_eq, assert_contains};
 use datafusion::error::Result;
 use datafusion::prelude::*;
-use tempfile::tempdir;
 
 /// This example demonstrates how to use the regexp_* functions
 ///
@@ -35,29 +33,12 @@ use tempfile::tempdir;
 /// https://docs.rs/regex/latest/regex/#grouping-and-flags
 pub async fn regexp() -> Result<()> {
     let ctx = SessionContext::new();
-    // content from file 'datafusion/physical-expr/tests/data/regex.csv'
-    let csv_data = r#"values,patterns,replacement,flags
-abc,^(a),bb\1bb,i
-ABC,^(A).*,B,i
-aBc,(b|d),e,i
-AbC,(B|D),e,
-aBC,^(b|c),d,
-4000,\b4([1-9]\d\d|\d[1-9]\d|\d\d[1-9])\b,xyz,
-4010,\b4([1-9]\d\d|\d[1-9]\d|\d\d[1-9])\b,xyz,
-Düsseldorf,[\p{Letter}-]+,München,
-Москва,[\p{L}-]+,Moscow,
-Köln,[a-zA-Z]ö[a-zA-Z]{2},Koln,
-اليوم,^\p{Arabic}+$,Today,"#;
-    let dir = tempdir()?;
-    let file_path = dir.path().join("regex.csv");
-    {
-        let mut file = File::create(&file_path)?;
-        // write CSV data
-        file.write_all(csv_data.as_bytes())?;
-    } // scope closes the file
-    let file_path = file_path.to_str().unwrap();
-
-    ctx.register_csv("examples", file_path, CsvReadOptions::new())
+    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("data")
+        .join("csv")
+        .join("regex.csv");
+
+    ctx.register_csv("examples", path.to_str().unwrap(), CsvReadOptions::new())
         .await?;
 
     //

datafusion-examples/examples/custom_data_source/csv_json_opener.rs

@@ -17,9 +17,10 @@
 
 //! See `main.rs` for how to run it.
 
+use std::path::PathBuf;
 use std::sync::Arc;
 
-use arrow::datatypes::{DataType, Field, Schema};
+use arrow::datatypes::{DataType, Field, Schema, TimeUnit};
 use datafusion::common::config::CsvOptions;
 use datafusion::{
     assert_batches_eq,
@@ -31,7 +32,6 @@ use datafusion::{
     },
     error::Result,
     physical_plan::metrics::ExecutionPlanMetricsSet,
-    test_util::aggr_test_schema,
 };
 
 use datafusion::datasource::physical_plan::FileScanConfigBuilder;
@@ -50,12 +50,20 @@ pub async fn csv_json_opener() -> Result<()> {
 
 async fn csv_opener() -> Result<()> {
     let object_store = Arc::new(LocalFileSystem::new());
-    let schema = aggr_test_schema();
-
-    let testdata = datafusion::test_util::arrow_test_data();
-    let path = format!("{testdata}/csv/aggregate_test_100.csv");
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("car", DataType::Utf8, false),
+        Field::new("speed", DataType::Float64, false),
+        Field::new(
+            "time",
+            DataType::Timestamp(TimeUnit::Nanosecond, None),
+            false,
+        ),
+    ]));
 
-    let path = std::path::Path::new(&path).canonicalize()?;
+    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("data")
+        .join("csv")
+        .join("cars.csv");
 
     let options = CsvOptions {
         has_header: Some(true),
@@ -71,7 +79,7 @@ async fn csv_opener() -> Result<()> {
 
     let scan_config =
         FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), source)
-            .with_projection_indices(Some(vec![12, 0]))?
+            .with_projection_indices(Some(vec![0, 1]))?
             .with_limit(Some(5))
             .with_file(PartitionedFile::new(path.display().to_string(), 10))
             .build();
@@ -89,15 +97,15 @@ async fn csv_opener() -> Result<()> {
     }
     assert_batches_eq!(
         &[
-            "+--------------------------------+----+",
-            "| c13                            | c1 |",
-            "+--------------------------------+----+",
-            "| 6WfVFBVGJSQb7FhA7E0lBwdvjfZnSW | c  |",
-            "| C2GT5KVyOPZpgKVl110TyZO0NcJ434 | d  |",
-            "| AyYVExXK6AR2qUTxNZ7qRHQOVGMLcz | b  |",
-            "| 0keZ5G8BffGwgF2RwQD59TFzMStxCB | a  |",
-            "| Ig1QcuKsjHXkproePdERo2w0mYzIqd | b  |",
-            "+--------------------------------+----+",
+            "+-----+-------+",
+            "| car | speed |",
+            "+-----+-------+",
+            "| red | 20.0  |",
+            "| red | 20.3  |",
+            "| red | 21.4  |",
+            "| red | 21.5  |",
+            "| red | 19.0  |",
+            "+-----+-------+",
         ],
         &result
     );

datafusion-examples/examples/custom_data_source/csv_sql_streaming.rs

@@ -17,7 +17,8 @@
 
 //! See `main.rs` for how to run it.
 
-use datafusion::common::test_util::datafusion_test_data;
+use std::path::PathBuf;
+
 use datafusion::error::Result;
 use datafusion::prelude::*;
 
@@ -27,7 +28,10 @@ pub async fn csv_sql_streaming() -> Result<()> {
     // create local execution context
     let ctx = SessionContext::new();
 
-    let testdata = datafusion_test_data();
+    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("data")
+        .join("csv")
+        .join("window_1.csv");
 
     // Register a table source and tell DataFusion the file is ordered by `ts ASC`.
     // Note it is the responsibility of the user to make sure
@@ -38,7 +42,7 @@ pub async fn csv_sql_streaming() -> Result<()> {
     // register csv file with the execution context
     ctx.register_csv(
         "ordered_table",
-        &format!("{testdata}/window_1.csv"),
+        path.to_str().unwrap(),
         CsvReadOptions::new().file_sort_order(vec![sort_expr]),
     )
     .await?;

datafusion-examples/examples/data_io/parquet_encrypted.rs

@@ -24,6 +24,7 @@ use datafusion::logical_expr::{col, lit};
 use datafusion::parquet::encryption::decrypt::FileDecryptionProperties;
 use datafusion::parquet::encryption::encrypt::FileEncryptionProperties;
 use datafusion::prelude::{ParquetReadOptions, SessionContext};
+use std::path::PathBuf;
 use std::sync::Arc;
 use tempfile::TempDir;
 
@@ -32,13 +33,14 @@ pub async fn parquet_encrypted() -> datafusion::common::Result<()> {
     // The SessionContext is the main high level API for interacting with DataFusion
     let ctx = SessionContext::new();
 
-    // Find the local path of "alltypes_plain.parquet"
-    let testdata = datafusion::test_util::parquet_test_data();
-    let filename = &format!("{testdata}/alltypes_plain.parquet");
+    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("data")
+        .join("parquet")
+        .join("alltypes_plain.parquet");
 
     // Read the sample parquet file
     let parquet_df = ctx
-        .read_parquet(filename, ParquetReadOptions::default())
+        .read_parquet(path.to_str().unwrap(), ParquetReadOptions::default())
         .await?;
 
     // Show information from the dataframe

datafusion-examples/examples/data_io/parquet_exec_visitor.rs

@@ -17,6 +17,7 @@
 
 //! See `main.rs` for how to run it.
 
+use std::path::PathBuf;
 use std::sync::Arc;
 
 use datafusion::datasource::file_format::parquet::ParquetFormat;
@@ -35,17 +36,22 @@ use futures::StreamExt;
 pub async fn parquet_exec_visitor() -> datafusion::common::Result<()> {
     let ctx = SessionContext::new();
 
-    let test_data = datafusion::test_util::parquet_test_data();
+    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("data")
+        .join("parquet")
+        .join("alltypes_plain.parquet");
 
     // Configure listing options
     let file_format = ParquetFormat::default().with_enable_pruning(true);
     let listing_options = ListingOptions::new(Arc::new(file_format));
 
+    let table_path = format!("file://{}", path.to_str().unwrap());
+
     // First example were we use an absolute path, which requires no additional setup.
     let _ = ctx
         .register_listing_table(
             "my_table",
-            &format!("file://{test_data}/alltypes_plain.parquet"),
+            &table_path,
             listing_options.clone(),
             None,
             None,