Merge branch 'develop'

Author: dolfinus

.github/workflows/get-matrix.yml

@@ -86,7 +86,7 @@ jobs:
 
       - name: Check if base files are changed
         id: changed-base
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         with:
           files_from_source_file: .github/workflows/data/base/tracked.txt
           files_ignore_from_source_file: .github/workflows/data/base/ignored.txt
@@ -97,7 +97,7 @@ jobs:
 
       - name: Check if db-related files are changed
         id: changed-db
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         with:
           files_from_source_file: .github/workflows/data/db/tracked.txt
           files_ignore_from_source_file: .github/workflows/data/db/ignored.txt
@@ -108,7 +108,7 @@ jobs:
 
       - name: Check if file-related files are changed
         id: changed-file
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         with:
           files_from_source_file: .github/workflows/data/file/tracked.txt
           files_ignore_from_source_file: .github/workflows/data/file/ignored.txt
@@ -119,7 +119,7 @@ jobs:
 
       - name: Check if file-df-related files are changed
         id: changed-file-df
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         with:
           files_from_source_file: .github/workflows/data/file-df/tracked.txt
           files_ignore_from_source_file: .github/workflows/data/file-df/ignored.txt
@@ -130,7 +130,7 @@ jobs:
 
       - name: Check if core files are changed
         id: changed-core
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         with:
           files_from_source_file: .github/workflows/data/core/tracked.txt
           files_ignore_from_source_file: .github/workflows/data/core/ignored.txt
@@ -160,7 +160,7 @@ jobs:
 
       - name: Check if Clickhouse files are changed
         id: changed-clickhouse
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         with:
           files_from_source_file: .github/workflows/data/clickhouse/tracked.txt
           files_ignore_from_source_file: .github/workflows/data/clickhouse/ignored.txt
@@ -190,7 +190,7 @@ jobs:
 
       - name: Check if Greenplum files are changed
         id: changed-greenplum
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         with:
           files_from_source_file: .github/workflows/data/greenplum/tracked.txt
           files_ignore_from_source_file: .github/workflows/data/greenplum/ignored.txt
@@ -220,7 +220,7 @@ jobs:
 
       - name: Check if Hive files are changed
         id: changed-hive
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         with:
           files_from_source_file: .github/workflows/data/hive/tracked.txt
           files_ignore_from_source_file: .github/workflows/data/hive/ignored.txt
@@ -250,7 +250,7 @@ jobs:
 
       - name: Check if Kafka files are changed
         id: changed-kafka
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         with:
           files_from_source_file: .github/workflows/data/kafka/tracked.txt
           files_ignore_from_source_file: .github/workflows/data/kafka/ignored.txt
@@ -280,7 +280,7 @@ jobs:
 
       - name: Check if LocalFS files are changed
         id: changed-local-fs
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         with:
           files_from_source_file: .github/workflows/data/local-fs/tracked.txt
           files_ignore_from_source_file: .github/workflows/data/local-fs/ignored.txt
@@ -310,7 +310,7 @@ jobs:
 
       - name: Check if MongoDB files are changed
         id: changed-mongodb
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         with:
           files_from_source_file: .github/workflows/data/mongodb/tracked.txt
           files_ignore_from_source_file: .github/workflows/data/mongodb/ignored.txt
@@ -340,7 +340,7 @@ jobs:
 
       - name: Check if MSSQL files are changed
         id: changed-mssql
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         with:
           files_from_source_file: .github/workflows/data/mssql/tracked.txt
           files_ignore_from_source_file: .github/workflows/data/mssql/ignored.txt
@@ -370,7 +370,7 @@ jobs:
 
       - name: Check if MySQL files are changed
         id: changed-mysql
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         with:
           files_from_source_file: .github/workflows/data/mysql/tracked.txt
           files_ignore_from_source_file: .github/workflows/data/mysql/ignored.txt
@@ -400,7 +400,7 @@ jobs:
 
       - name: Check if Oracle files are changed
         id: changed-oracle
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         with:
           files_from_source_file: .github/workflows/data/oracle/tracked.txt
           files_ignore_from_source_file: .github/workflows/data/oracle/ignored.txt
@@ -430,7 +430,7 @@ jobs:
 
       - name: Check if Postgres files are changed
         id: changed-postgres
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         with:
           files_from_source_file: .github/workflows/data/postgres/tracked.txt
           files_ignore_from_source_file: .github/workflows/data/postgres/ignored.txt
@@ -460,7 +460,7 @@ jobs:
 
       - name: Check if Teradata files are changed
         id: changed-teradata
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         with:
           files_from_source_file: .github/workflows/data/teradata/tracked.txt
           files_ignore_from_source_file: .github/workflows/data/teradata/ignored.txt
@@ -490,7 +490,7 @@ jobs:
 
       - name: Check if FTP files are changed
         id: changed-ftp
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         with:
           files_from_source_file: .github/workflows/data/ftp/tracked.txt
           files_ignore_from_source_file: .github/workflows/data/ftp/ignored.txt
@@ -520,7 +520,7 @@ jobs:
 
       - name: Check if FTPS files are changed
         id: changed-ftps
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         with:
           files_from_source_file: .github/workflows/data/ftps/tracked.txt
           files_ignore_from_source_file: .github/workflows/data/ftps/ignored.txt
@@ -550,7 +550,7 @@ jobs:
 
       - name: Check if HDFS files are changed
         id: changed-hdfs
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         with:
           files_from_source_file: .github/workflows/data/hdfs/tracked.txt
           files_ignore_from_source_file: .github/workflows/data/hdfs/ignored.txt
@@ -580,7 +580,7 @@ jobs:
 
       - name: Check if S3 files are changed
         id: changed-s3
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         with:
           files_from_source_file: .github/workflows/data/s3/tracked.txt
           files_ignore_from_source_file: .github/workflows/data/s3/ignored.txt
@@ -610,7 +610,7 @@ jobs:
 
       - name: Check if SFTP files are changed
         id: changed-sftp
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         with:
           files_from_source_file: .github/workflows/data/sftp/tracked.txt
           files_ignore_from_source_file: .github/workflows/data/sftp/ignored.txt
@@ -640,7 +640,7 @@ jobs:
 
       - name: Check if Samba files are changed
         id: changed-samba
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         with:
           files_from_source_file: .github/workflows/data/samba/tracked.txt
           files_ignore_from_source_file: .github/workflows/data/samba/ignored.txt
@@ -670,7 +670,7 @@ jobs:
 
       - name: Check if WebDAV files are changed
         id: changed-webdav
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         with:
           files_from_source_file: .github/workflows/data/webdav/tracked.txt
           files_ignore_from_source_file: .github/workflows/data/webdav/ignored.txt

docs/changelog/0.13.4.rst

@@ -0,0 +1,14 @@
+0.13.4 (2025-03-20)
+===================
+
+Doc only Changes
+----------------
+
+- Prefer ``ReadOptions(partitionColumn=..., numPartitions=..., queryTimeout=...)``
+  instead of ``ReadOptions(partition_column=..., num_partitions=..., query_timeout=...)``,
+  to match Spark documentation. (:github:pull:`352`)
+
+- Prefer ``WriteOptions(if_exists=...)`` instead of ``WriteOptions(mode=...)`` for IDE suggestions. (:github:pull:`354`)
+
+- Document all options of supported file formats.
+  (:github:pull:`355`, :github:pull:`356`, :github:pull:`357`, :github:pull:`358`, :github:pull:`359`, :github:pull:`360`, :github:pull:`361`, :github:pull:`362`)

docs/changelog/index.rst

@@ -3,6 +3,7 @@
     :caption: Changelog
 
     DRAFT
+    0.13.4
     0.13.3
     0.13.1
     0.13.0

docs/connection/db_connection/clickhouse/execute.rst

@@ -50,7 +50,7 @@ Examples
 
     df = clickhouse.fetch(
         "SELECT value FROM some.reference_table WHERE key = 'some_constant'",
-        options=Clickhouse.FetchOptions(query_timeout=10),
+        options=Clickhouse.FetchOptions(queryTimeout=10),
     )
     clickhouse.close()
     value = df.collect()[0][0]  # get value from first row and first column
@@ -96,7 +96,7 @@ Examples
         ENGINE = MergeTree()
         ORDER BY id
         """,
-        options=Clickhouse.ExecuteOptions(query_timeout=10),
+        options=Clickhouse.ExecuteOptions(queryTimeout=10),
     )
 
 Notes

docs/connection/db_connection/clickhouse/read.rst

@@ -41,7 +41,7 @@ Snapshot strategy:
         source="schema.table",
         columns=["id", "key", "CAST(value AS String) value", "updated_dt"],
         where="key = 'something'",
-        options=Clickhouse.ReadOptions(partition_column="id", num_partitions=10),
+        options=Clickhouse.ReadOptions(partitionColumn="id", numPartitions=10),
     )
     df = reader.run()
 
@@ -61,7 +61,7 @@ Incremental strategy:
         columns=["id", "key", "CAST(value AS String) value", "updated_dt"],
         where="key = 'something'",
         hwm=DBReader.AutoDetectHWM(name="clickhouse_hwm", expression="updated_dt"),
-        options=Clickhouse.ReadOptions(partition_column="id", num_partitions=10),
+        options=Clickhouse.ReadOptions(partitionColumn="id", numPartitions=10),
     )
 
     with IncrementalStrategy():

docs/connection/db_connection/clickhouse/sql.rst

@@ -44,10 +44,10 @@ Examples
             key = 'something'
         """,
         options=Clickhouse.SQLOptions(
-            partition_column="id",
-            num_partitions=10,
-            lower_bound=0,
-            upper_bound=1000,
+            partitionColumn="id",
+            numPartitions=10,
+            lowerBound=0,
+            upperBound=1000,
         ),
     )
 

docs/connection/db_connection/clickhouse/types.rst

@@ -77,7 +77,7 @@ So instead of relying on Spark to create tables:
 
         writer = DBWriter(
             connection=clickhouse,
-            table="default.target_tbl",
+            target="default.target_tbl",
             options=Clickhouse.WriteOptions(
                 if_exists="append",
                 # ENGINE is required by Clickhouse
@@ -105,7 +105,7 @@ Always prefer creating tables with specific types **BEFORE WRITING DATA**:
 
         writer = DBWriter(
             connection=clickhouse,
-            table="default.target_tbl",
+            target="default.target_tbl",
             options=Clickhouse.WriteOptions(if_exists="append"),
         )
         writer.run(df)
@@ -375,7 +375,7 @@ For parsing JSON columns in ClickHouse, :obj:`JSON.parse_column <onetl.file.form
 
     reader = DBReader(
         connection=clickhouse,
-        table="default.source_tbl",
+        target="default.source_tbl",
         columns=[
             "id",
             "toJSONString(array_column) array_column",

docs/connection/db_connection/greenplum/execute.rst

@@ -50,7 +50,7 @@ Examples
 
     df = greenplum.fetch(
         "SELECT value FROM some.reference_table WHERE key = 'some_constant'",
-        options=Greenplum.FetchOptions(query_timeout=10),
+        options=Greenplum.FetchOptions(queryTimeout=10),
     )
     greenplum.close()
     value = df.collect()[0][0]  # get value from first row and first column
@@ -97,7 +97,7 @@ Examples
         )
         DISTRIBUTED BY id
         """,
-        options=Greenplum.ExecuteOptions(query_timeout=10),
+        options=Greenplum.ExecuteOptions(queryTimeout=10),
     )
 
 Interaction schema

docs/connection/db_connection/greenplum/read.rst

@@ -275,11 +275,11 @@ In this case, custom column can be used instead:
         reader = DBReader(
             connection=greenplum,
             source="schema.view_with_partition_column",
-            options=Greenplum.Options(
+            options=Greenplum.ReadOptions(
                 # parallelize data using specified column
-                partition_column="part_column",
+                partitionColumn="part_column",
                 # create 10 Spark tasks, each will read only part of table data
-                num_partitions=10,
+                partitions=10,
             ),
         )
         df = reader.run()
@@ -348,8 +348,8 @@ and then read this table using ``DBReader``:
 
     .. code-block:: python
 
-        df1 = DBReader(connection=greenplum, table="public.table1", ...).run()
-        df2 = DBReader(connection=greenplum, table="public.table2", ...).run()
+        df1 = DBReader(connection=greenplum, target="public.table1", ...).run()
+        df2 = DBReader(connection=greenplum, target="public.table2", ...).run()
 
         joined_df = df1.join(df2, on="col")
 

docs/connection/db_connection/greenplum/types.rst

@@ -66,7 +66,7 @@ So instead of relying on Spark to create tables:
 
         writer = DBWriter(
             connection=greenplum,
-            table="public.table",
+            target="public.table",
             options=Greenplum.WriteOptions(
                 if_exists="append",
                 # by default distribution is random
@@ -96,7 +96,7 @@ Always prefer creating table with desired DDL **BEFORE WRITING DATA**:
 
         writer = DBWriter(
             connection=greenplum,
-            table="public.table",
+            target="public.table",
             options=Greenplum.WriteOptions(if_exists="append"),
         )
         writer.run(df)