Merge pull request #87 from CartoDB/copy-endpoints-support

Copy endpoints support

Author: Rafa de la Torre

.gitignore

@@ -44,6 +44,7 @@ htmlcov/
 nosetests.xml
 coverage.xml
 *,cover
+.pytest_cache/
 
 # Translations
 *.mo

CONTRIBUTING.md

@@ -4,9 +4,10 @@ Contributions are totally welcome. However, contributors must sign a Contributor
 
 ## Release process
 
-1. Update version number and information at `setup.py`, `conf.py` and `NEWS.md`.
-2. You must be maintainer at [carto pypi repo](https://pypi.python.org/pypi/carto/).
-3. Prepare a `~/.pypirc` file:
+1. Make sure documentation is generated properly. See the [doc](https://github.com/CartoDB/carto-python/tree/master/doc) directory for more information.
+2. Update version number and information at `setup.py`, `conf.py` and `NEWS.md`.
+3. You must be maintainer at [carto pypi repo](https://pypi.python.org/pypi/carto/).
+4. Prepare a `~/.pypirc` file:
 
 ```
 [distutils]
@@ -23,7 +24,7 @@ username=your_username
 password=your_password
 ```
 
-4. Upload the package to the test repository: `python setup.py sdist upload -r pypitest`.
-5. Install it in a new environment: `pip install --index-url=https://test.pypi.org/simple --extra-index-url=https://pypi.org/simple carto`.
-6. Test it.
-7. Release it: `python setup.py sdist upload -r pypi`.
+5. Upload the package to the test repository: `python setup.py sdist upload -r pypitest`.
+6. Install it in a new environment: `pip install --index-url=https://test.pypi.org/simple --extra-index-url=https://pypi.org/simple carto`.
+7. Test it.
+8. Release it: `python setup.py sdist upload -r pypi`.

NEWS

@@ -1,3 +1,12 @@
+Mmm-dd-yyyy: version 1.3.0
+ - Added a `CopySQLClient` for efficient streaming of data to and from CARTO (#87)
+ - Fixed CI tests
+ - Added a request header `User-Agent: carto-python-sdk/x.y.z` where `x.y.z` is the current version of the library.
+ - Added a `client=cps-x.y.z` parameter to requests, where `x.y.z` is the current version of the library.
+
+Feb-22-2028: version 1.2.2
+ - Updated release instructions in CONTRIBUTING.md
+
 Feb-21-2018: version 1.2.1
  - Added AuthAPIClient class for future Auth API usage and API key validation (#69)
 

README.md

@@ -139,6 +139,62 @@ updateJob = batchSQLClient.update(job_id, NEW_QUERY)
 cancelJob = batchSQLClient.cancel(job_id)
 ```
 
+**COPY queries**
+
+COPY queries allow you to use the [PostgreSQL COPY command](https://www.postgresql.org/docs/10/static/sql-copy.html) for efficient streaming of data to and from CARTO.
+
+Here is a basic example of its usage:
+
+```python
+from carto.sql import SQLClient
+from carto.sql import CopySQLClient
+
+sql_client = SQLClient(auth_client)
+copy_client = CopySQLClient(auth_client)
+
+# Create a destination table for the copy with the right schema
+sql_client.send("""
+    CREATE TABLE IF NOT EXISTS copy_example (
+      the_geom geometry(Geometry,4326),
+      name text,
+      age integer
+    )
+    """)
+sql_client.send("SELECT CDB_CartodbfyTable(current_schema, 'copy_example')")
+
+# COPY FROM a csv file in the filesytem
+from_query = 'COPY copy_example (the_geom, name, age) FROM stdin WITH (FORMAT csv, HEADER true)'
+result = copy_client.copyfrom_file_path(from_query, 'copy_from.csv')
+
+# COPY TO a file in the filesystem
+to_query = 'COPY copy_example TO stdout WITH (FORMAT csv, HEADER true)'
+copy_client.copyto_file_path(to_query, 'export.csv')
+```
+
+Here's an equivalent, more pythonic example of the COPY FROM, using a `file` object:
+
+```python
+with open('copy_from.csv', 'rb') as f:
+    copy_client.copyfrom_file_object(from_query, f)
+```
+
+And here is a demonstration of how to generate and stream data directly (no need for a file at all):
+
+```python
+def rows():
+    # note the \n to delimit rows
+    yield bytearray(u'the_geom,name,age\n', 'utf-8')
+    for i in range(1,80):
+        row = u'SRID=4326;POINT({lon} {lat}),{name},{age}\n'.format(
+            lon = i,
+            lat = i,
+            name = 'fulano',
+            age = 100 - i
+        )
+        yield bytearray(row, 'utf-8')
+copy_client.copyfrom(from_query, rows())
+```
+
 For more examples on how to use the SQL API, please refer to the **examples** folder or the API docs.
 
 Import API

carto/auth.py

@@ -17,6 +17,7 @@
 import re
 import sys
 import warnings
+import pkg_resources
 
 from pyrestcli.auth import BaseAuthClient, BasicAuthClient
 
@@ -64,7 +65,23 @@ def check_base_url(self, base_url):
         return base_url
 
 
-class APIKeyAuthClient(_UsernameGetter, _BaseUrlChecker, BaseAuthClient):
+class _ClientIdentifier:
+
+    CARTO_VERSION = pkg_resources.require('carto')[0].version
+
+    def get_user_agent(self, name='carto-python-sdk'):
+        return "{name}/{version}".format(
+            name=name,
+            version=self.CARTO_VERSION)
+
+    def get_client_identifier(self, prefix='cps'):
+        return "{prefix}-{version}".format(
+            prefix=prefix,
+            version=self.CARTO_VERSION)
+
+
+class APIKeyAuthClient(_UsernameGetter, _BaseUrlChecker, _ClientIdentifier,
+                       BaseAuthClient):
     """
     This class provides you with authenticated access to CARTO's APIs using
     your API key.
@@ -90,6 +107,8 @@ def __init__(self, base_url, api_key, organization=None, session=None):
         self.api_key = api_key
         base_url = self.check_base_url(base_url)
         self.username = self.get_user_name(base_url)
+        self.user_agent = self.get_user_agent()
+        self.client_id = self.get_client_identifier()
 
         super(APIKeyAuthClient, self).__init__(base_url, session=session)
 
@@ -120,12 +139,20 @@ def send(self, relative_path, http_method, **requests_args):
 
     def prepare_send(self, http_method, **requests_args):
         http_method = http_method.lower()
+        params = {
+            "api_key": self.api_key,
+            "client": self.client_id
+        }
         if (http_method in ['post', 'put']) and "json" in requests_args:
-            requests_args["json"].update({"api_key": self.api_key})
+            requests_args["json"].update(params)
         else:
             if "params" not in requests_args:
                 requests_args["params"] = {}
-            requests_args["params"].update({"api_key": self.api_key})
+            requests_args["params"].update(params)
+
+        if not requests_args.get('headers', None):
+            requests_args['headers'] = {}
+        requests_args['headers'].update({'User-Agent': self.user_agent})
 
         return http_method, requests_args