Skip to content

Commit 04950cd

Browse files
dhermesdhermes
committed
Using GAPIC datastore object (and an HTTP equivalent) for lookup.
This is the last method to be ported over so the Connection() base class as well as the _DatastoreAPIOverGRPC and _DatastoreAPIOverHttp helper classes have been totally removed. This commit represents a "working" implementation (i.e. system tests pass) but the unit tests have yet to be updated.
1 parent d9f2dda commit 04950cd

File tree

6 files changed

+65
-192
lines changed

6 files changed

+65
-192
lines changed

‎datastore/google/cloud/datastore/_gax.py‎

Lines changed: 0 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,13 @@
1919
import sys
2020

2121
from google.cloud.gapic.datastore.v1 import datastore_client
22-
from google.cloud.proto.datastore.v1 import datastore_pb2_grpc
2322
from google.gax.errors import GaxError
2423
from google.gax.grpc import exc_to_code
2524
from google.gax.utils import metrics
2625
from grpc import StatusCode
2726
import six
2827

29-
from google.cloud._helpers import make_insecure_stub
3028
from google.cloud._helpers import make_secure_channel
31-
from google.cloud._helpers import make_secure_stub
3229
from google.cloud._http import DEFAULT_USER_AGENT
3330
from google.cloud import exceptions
3431

@@ -92,50 +89,6 @@ def _grpc_catch_rendezvous():
9289
six.reraise(error_class, new_exc, sys.exc_info()[2])
9390

9491

95-
class _DatastoreAPIOverGRPC(object):
96-
"""Helper mapping datastore API methods.
97-
98-
Makes requests to send / receive protobuf content over gRPC.
99-
100-
Methods make bare API requests without any helpers for constructing
101-
the requests or parsing the responses.
102-
103-
:type connection: :class:`Connection`
104-
:param connection: A connection object that contains helpful
105-
information for making requests.
106-
"""
107-
108-
def __init__(self, connection):
109-
parse_result = six.moves.urllib_parse.urlparse(
110-
connection.api_base_url)
111-
host = parse_result.hostname
112-
if parse_result.scheme == 'https':
113-
self._stub = make_secure_stub(
114-
connection.credentials, DEFAULT_USER_AGENT,
115-
datastore_pb2_grpc.DatastoreStub, host,
116-
extra_options=_GRPC_EXTRA_OPTIONS)
117-
else:
118-
self._stub = make_insecure_stub(
119-
datastore_pb2_grpc.DatastoreStub, host)
120-
121-
def lookup(self, project, request_pb):
122-
"""Perform a ``lookup`` request.
123-
124-
:type project: str
125-
:param project: The project to connect to. This is
126-
usually your project name in the cloud console.
127-
128-
:type request_pb: :class:`.datastore_pb2.LookupRequest`
129-
:param request_pb: The request protobuf object.
130-
131-
:rtype: :class:`.datastore_pb2.LookupResponse`
132-
:returns: The returned protobuf response object.
133-
"""
134-
request_pb.project_id = project
135-
with _grpc_catch_rendezvous():
136-
return self._stub.Lookup(request_pb)
137-
138-
13992
class GAPICDatastoreAPI(datastore_client.DatastoreClient):
14093
"""An API object that sends proto-over-gRPC requests.
14194

‎datastore/google/cloud/datastore/_http.py‎

Lines changed: 21 additions & 126 deletions
Original file line numberDiff line numberDiff line change
@@ -14,22 +14,13 @@
1414

1515
"""Connections to Google Cloud Datastore API servers."""
1616

17-
import os
18-
1917
from google.rpc import status_pb2
2018

2119
from google.cloud import _http as connection_module
22-
from google.cloud.environment_vars import DISABLE_GRPC
2320
from google.cloud import exceptions
2421
from google.cloud.proto.datastore.v1 import datastore_pb2 as _datastore_pb2
2522

2623
from google.cloud.datastore import __version__
27-
try:
28-
from google.cloud.datastore._gax import _DatastoreAPIOverGRPC
29-
_HAVE_GRPC = True
30-
except ImportError: # pragma: NO COVER
31-
_DatastoreAPIOverGRPC = None
32-
_HAVE_GRPC = False
3324

3425

3526
DATASTORE_API_HOST = 'datastore.googleapis.com'
@@ -42,8 +33,6 @@
4233
'/{project}:{method}')
4334
"""A template for the URL of a particular API call."""
4435

45-
_DISABLE_GRPC = os.getenv(DISABLE_GRPC, False)
46-
_USE_GRPC = _HAVE_GRPC and not _DISABLE_GRPC
4736
_CLIENT_INFO = connection_module.CLIENT_INFO_TEMPLATE.format(__version__)
4837

4938

@@ -148,121 +137,45 @@ def build_api_url(project, method, base_url):
148137
project=project, method=method)
149138

150139

151-
class _DatastoreAPIOverHttp(object):
152-
"""Helper mapping datastore API methods.
153-
154-
Makes requests to send / receive protobuf content over HTTP/1.1.
140+
class HTTPDatastoreAPI(object):
141+
"""An API object that sends proto-over-HTTP requests.
155142
156-
Methods make bare API requests without any helpers for constructing
157-
the requests or parsing the responses.
143+
Intended to provide the same methods as the GAPIC ``DatastoreClient``.
158144
159-
:type connection: :class:`Connection`
160-
:param connection: A connection object that contains helpful
161-
information for making requests.
145+
:type client: :class:`~google.cloud.datastore.client.Client`
146+
:param client: The client that provides configuration.
162147
"""
163148

164-
def __init__(self, connection):
165-
self.connection = connection
149+
def __init__(self, client):
150+
self.client = client
166151

167-
def lookup(self, project, request_pb):
152+
def lookup(self, project, read_options, key_pbs):
168153
"""Perform a ``lookup`` request.
169154
170155
:type project: str
171156
:param project: The project to connect to. This is
172157
usually your project name in the cloud console.
173158
174-
:type request_pb: :class:`.datastore_pb2.LookupRequest`
175-
:param request_pb: The request protobuf object.
176-
177-
:rtype: :class:`.datastore_pb2.LookupResponse`
178-
:returns: The returned protobuf response object.
179-
"""
180-
return _rpc(self.connection.http, project, 'lookup',
181-
self.connection.api_base_url,
182-
request_pb, _datastore_pb2.LookupResponse)
183-
184-
185-
class Connection(connection_module.Connection):
186-
"""A connection to the Google Cloud Datastore via the Protobuf API.
187-
188-
This class should understand only the basic types (and protobufs)
189-
in method arguments, however it should be capable of returning advanced
190-
types.
191-
192-
:type client: :class:`~google.cloud.datastore.client.Client`
193-
:param client: The client that owns the current connection.
194-
"""
195-
196-
def __init__(self, client):
197-
super(Connection, self).__init__(client)
198-
self.api_base_url = client._base_url
199-
if _USE_GRPC:
200-
self._datastore_api = _DatastoreAPIOverGRPC(self)
201-
else:
202-
self._datastore_api = _DatastoreAPIOverHttp(self)
203-
204-
def lookup(self, project, key_pbs,
205-
eventual=False, transaction_id=None):
206-
"""Lookup keys from a project in the Cloud Datastore.
207-
208-
Maps the ``DatastoreService.Lookup`` protobuf RPC.
209-
210-
This uses mostly protobufs
211-
(:class:`.entity_pb2.Key` as input and :class:`.entity_pb2.Entity`
212-
as output). It is used under the hood in
213-
:meth:`Client.get() <.datastore.client.Client.get>`:
214-
215-
.. code-block:: python
216-
217-
>>> from google.cloud import datastore
218-
>>> client = datastore.Client(project='project')
219-
>>> key = client.key('MyKind', 1234)
220-
>>> client.get(key)
221-
[<Entity object>]
222-
223-
Using a :class:`Connection` directly:
224-
225-
.. code-block:: python
226-
227-
>>> connection.lookup('project', [key.to_protobuf()])
228-
[<Entity protobuf>]
229-
230-
:type project: str
231-
:param project: The project to look up the keys in.
159+
:type read_options: :class:`.datastore_pb2.ReadOptions`
160+
:param read_options: The options for this lookup. Contains a
161+
either the transaction for the read or
162+
``STRONG`` or ``EVENTUAL`` read consistency.
232163
233164
:type key_pbs: list of
234165
:class:`.entity_pb2.Key`
235166
:param key_pbs: The keys to retrieve from the datastore.
236167
237-
:type eventual: bool
238-
:param eventual: If False (the default), request ``STRONG`` read
239-
consistency. If True, request ``EVENTUAL`` read
240-
consistency.
241-
242-
:type transaction_id: str
243-
:param transaction_id: If passed, make the request in the scope of
244-
the given transaction. Incompatible with
245-
``eventual==True``.
246-
247168
:rtype: :class:`.datastore_pb2.LookupResponse`
248-
:returns: The returned protobuf for the lookup request.
169+
:returns: The returned protobuf response object.
249170
"""
250-
lookup_request = _datastore_pb2.LookupRequest(keys=key_pbs)
251-
_set_read_options(lookup_request, eventual, transaction_id)
252-
return self._datastore_api.lookup(project, lookup_request)
253-
254-
255-
class HTTPDatastoreAPI(object):
256-
"""An API object that sends proto-over-HTTP requests.
257-
258-
Intended to provide the same methods as the GAPIC ``DatastoreClient``.
259-
260-
:type client: :class:`~google.cloud.datastore.client.Client`
261-
:param client: The client that provides configuration.
262-
"""
263-
264-
def __init__(self, client):
265-
self.client = client
171+
request_pb = _datastore_pb2.LookupRequest(
172+
project_id=project,
173+
read_options=read_options,
174+
keys=key_pbs,
175+
)
176+
return _rpc(self.client._http, project, 'lookup',
177+
self.client._base_url,
178+
request_pb, _datastore_pb2.LookupResponse)
266179

267180
def run_query(self, project, partition_id, read_options,
268181
query=None, gql_query=None):
@@ -390,21 +303,3 @@ def allocate_ids(self, project, key_pbs):
390303
return _rpc(self.client._http, project, 'allocateIds',
391304
self.client._base_url,
392305
request_pb, _datastore_pb2.AllocateIdsResponse)
393-
394-
395-
def _set_read_options(request, eventual, transaction_id):
396-
"""Validate rules for read options, and assign to the request.
397-
398-
Helper method for ``lookup()`` and ``run_query``.
399-
400-
:raises: :class:`ValueError` if ``eventual`` is ``True`` and the
401-
``transaction_id`` is not ``None``.
402-
"""
403-
if eventual and (transaction_id is not None):
404-
raise ValueError('eventual must be False when in a transaction')
405-
406-
opts = request.read_options
407-
if eventual:
408-
opts.read_consistency = _datastore_pb2.ReadOptions.EVENTUAL
409-
elif transaction_id:
410-
opts.transaction = transaction_id

‎datastore/google/cloud/datastore/batch.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,7 @@ def _commit(self):
249249
self.project, mode, self._mutations, transaction=self._id)
250250
_, updated_keys = _parse_commit_response(commit_response_pb)
251251
# If the back-end returns without error, we are guaranteed that
252-
# :meth:`Connection.commit` will return keys that match (length and
252+
# ``commit`` will return keys that match (length and
253253
# order) directly ``_partial_key_entities``.
254254
for new_key_pb, entity in zip(updated_keys,
255255
self._partial_key_entities):

‎datastore/google/cloud/datastore/client.py‎

Lines changed: 43 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515

1616
import os
1717

18+
from google.cloud.proto.datastore.v1 import datastore_pb2 as _datastore_pb2
19+
1820
from google.cloud._helpers import _LocalStack
1921
from google.cloud._helpers import (
2022
_determine_default_project as _base_default_project)
@@ -23,7 +25,6 @@
2325
from google.cloud.environment_vars import GCD_DATASET
2426
from google.cloud.environment_vars import GCD_HOST
2527

26-
from google.cloud.datastore._http import Connection
2728
from google.cloud.datastore._http import HTTPDatastoreAPI
2829
from google.cloud.datastore import helpers
2930
from google.cloud.datastore.batch import Batch
@@ -78,15 +79,18 @@ def _determine_default_project(project=None):
7879
return project
7980

8081

81-
def _extended_lookup(connection, project, key_pbs,
82+
def _extended_lookup(datastore_api, project, key_pbs,
8283
missing=None, deferred=None,
8384
eventual=False, transaction_id=None):
8485
"""Repeat lookup until all keys found (unless stop requested).
8586
8687
Helper function for :meth:`Client.get_multi`.
8788
88-
:type connection: :class:`google.cloud.datastore._http.Connection`
89-
:param connection: The connection used to connect to datastore.
89+
:type datastore_api:
90+
:class:`google.cloud.datastore._http.HTTPDatastoreAPI`
91+
or :class:`google.cloud.datastore._gax.GAPICDatastoreAPI`
92+
:param datastore_api: The datastore API object used to connect
93+
to datastore.
9094
9195
:type project: str
9296
:param project: The project to make the request for.
@@ -127,15 +131,11 @@ def _extended_lookup(connection, project, key_pbs,
127131
results = []
128132

129133
loop_num = 0
134+
read_options = _get_read_options(eventual, transaction_id)
130135
while loop_num < _MAX_LOOPS: # loop against possible deferred.
131136
loop_num += 1
132-
133-
lookup_response = connection.lookup(
134-
project=project,
135-
key_pbs=key_pbs,
136-
eventual=eventual,
137-
transaction_id=transaction_id,
138-
)
137+
lookup_response = datastore_api.lookup(
138+
project, read_options, key_pbs)
139139

140140
# Accumulate the new results.
141141
results.extend(result.entity for result in lookup_response.found)
@@ -210,9 +210,6 @@ def __init__(self, project=None, namespace=None,
210210
self._base_url = 'http://' + host
211211
except KeyError:
212212
self._base_url = _DATASTORE_BASE_URL
213-
# NOTE: Make sure all properties are set before passing to
214-
# ``Connection`` (e.g. ``_base_url``).
215-
self._connection = Connection(self)
216213

217214
@staticmethod
218215
def _determine_default(project):
@@ -347,7 +344,7 @@ def get_multi(self, keys, missing=None, deferred=None, transaction=None):
347344
transaction = self.current_transaction
348345

349346
entity_pbs = _extended_lookup(
350-
connection=self._connection,
347+
datastore_api=self._datastore_api,
351348
project=self.project,
352349
key_pbs=[k.to_protobuf() for k in keys],
353350
missing=missing,
@@ -569,3 +566,34 @@ def do_something(entity):
569566
if 'namespace' not in kwargs:
570567
kwargs['namespace'] = self.namespace
571568
return Query(self, **kwargs)
569+
570+
571+
def _get_read_options(eventual, transaction_id):
572+
"""Validate rules for read options, and assign to the request.
573+
574+
Helper method for ``lookup()`` and ``run_query``.
575+
576+
:type eventual: bool
577+
:param eventual: Flag indicating if ``EVENTUAL`` or ``STRONG``
578+
consistency should be used.
579+
580+
:type transaction_id: bytes
581+
:param transaction_id: A transaction identifier (may be null).
582+
583+
:rtype: :class:`.datastore_pb2.ReadOptions`
584+
:returns: The read options corresponding to the inputs.
585+
:raises: :class:`ValueError` if ``eventual`` is ``True`` and the
586+
``transaction_id`` is not ``None``.
587+
"""
588+
if transaction_id is None:
589+
if eventual:
590+
return _datastore_pb2.ReadOptions(
591+
read_consistency=_datastore_pb2.ReadOptions.EVENTUAL)
592+
else:
593+
return _datastore_pb2.ReadOptions()
594+
else:
595+
if eventual:
596+
raise ValueError('eventual must be False when in a transaction')
597+
else:
598+
return _datastore_pb2.ReadOptions(
599+
transaction=transaction_id)

‎datastore/google/cloud/datastore/query.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -362,8 +362,6 @@ def fetch(self, limit=None, offset=0, start_cursor=None, end_cursor=None,
362362
363363
:rtype: :class:`Iterator`
364364
:returns: The iterator for the query.
365-
:raises: ValueError if ``connection`` is not passed and no implicit
366-
default has been set.
367365
"""
368366
if client is None:
369367
client = self._client

0 commit comments

Comments
 (0)