feat: FIR-43722 add streaming to python sdk (#424)

Author: stepansergeevitch

.github/workflows/code-check.yml

@@ -30,4 +30,4 @@ jobs:
         pip install ".[dev]"
 
     - name: Run pre-commit checks
-      uses: pre-commit/action@v2.0.3
+      uses: pre-commit/action@v3.0.1

docsrc/Connecting_and_queries.rst

@@ -672,6 +672,37 @@ will send a cancel request to the server and the query will be stopped.
     print(successful) # False
 
 
+
+Streaming query results
+==============================
+
+By default, the driver will fetch all the results at once and store them in memory.
+This does not always fit the needs of the application, especially when the result set is large.
+In this case, you can use the `execute_stream` cursor method to fetch results in chunks.
+
+.. note::
+    The `execute_stream` method is not supported with :ref:`connecting_and_queries:Server-side asynchronous query execution`. It can only be used with regular queries.
+
+.. note::
+    If you enable result streaming, the query execution might finish successfully, but the actual error might be returned while iterating the rows.
+
+Synchronous example:
+::
+
+    with connection.cursor() as cursor:
+        cursor.execute_stream("SELECT * FROM my_huge_table")
+        for row in cursor:
+            # Process the row
+            print(row)
+
+Asynchronous example:
+::
+    async with async_connection.cursor() as cursor:
+        await cursor.execute_stream("SELECT * FROM my_huge_table")
+        async for row in cursor:
+            # Process the row
+            print(row)
+
 Thread safety
 ==============================
 

examples/dbapi.ipynb

@@ -194,7 +194,7 @@
    "id": "02e5db2f",
    "metadata": {},
    "source": [
-    "### Error handling\n",
+    "## Error handling\n",
     "If one query fails during the execution, all remaining queries are canceled.\n",
     "However, you still can fetch results for successful queries"
    ]
@@ -219,6 +219,34 @@
     "cursor.fetchall()"
    ]
   },
+  {
+   "cell_type": "raw",
+   "id": "9789285b0362e8a6",
+   "metadata": {
+    "collapsed": false
+   },
+   "source": [
+    "## Query result streaming\n",
+    "\n",
+    "Streaming is useful for large result sets, when you want to process rows one by one without loading all of them into memory."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "id": "e96d2bda533b250d",
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "cursor.execute_stream(\"select * from generate_series(1, 1000000)\")\n",
+    "for row in cursor:\n",
+    "    print(row)\n",
+    "    if row[0] > 10:\n",
+    "        break\n",
+    "# Remaining rows will not be fetched"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "b1cd4ff2",
@@ -377,6 +405,32 @@
     "    pass\n",
     "async_conn.closed"
    ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "80a885228cbad698",
+   "metadata": {
+    "collapsed": false
+   },
+   "source": [
+    "## Query result streaming"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "id": "5eaaf1c35bac6fc6",
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "await cursor.execute_stream(\"select * from generate_series(1, 1000000)\")\n",
+    "async for row in cursor:\n",
+    "    print(row)\n",
+    "    if row[0] > 10:\n",
+    "        break\n",
+    "# Remaining rows will not be fetched"
+   ]
   }
  ],
  "metadata": {

setup.cfg

@@ -55,6 +55,7 @@ dev =
     devtools==0.7.0
     mypy==1.*,<1.10.0
     pre-commit==3.5.0
+    psutil==7.0.0
     pyfakefs>=4.5.3,<=5.6.0
     pytest==7.2.0
     pytest-cov==3.0.0