From 5c99737b3e64802b0ac39a1e72839dd36674e48a Mon Sep 17 00:00:00 2001 From: Taras Pashkevych Date: Sat, 20 Apr 2019 18:18:49 +0300 Subject: [PATCH 1/5] issue #158 - fixing tests and documentation to support elasticsearch 7 while keeping lib compatible with previous version; specifying version explicitly during setup is now required --- README.rst | 12 ++- setup.py | 5 +- tests/conftest.py | 32 +++++++- tests/test_scan.py | 116 ++++++++++++++++++++++++-- tests/test_transport.py | 175 +++++++++++++++++++++++++++++++++++++--- 5 files changed, 317 insertions(+), 23 deletions(-) diff --git a/README.rst b/README.rst index c1564528..5025bf0e 100644 --- a/README.rst +++ b/README.rst @@ -15,9 +15,17 @@ aioelasticsearch Installation ------------ +For elasticsearch < 7: + +.. code-block:: shell + + pip install aioelasticsearch[6] + +For elasticsearch >= 7: + .. code-block:: shell - pip install aioelasticsearch + pip install aioelasticsearch[7] Usage ----- @@ -56,7 +64,7 @@ Asynchronous `scroll =6.0.0,<7.0.0', 'aiohttp>=3.5.0,<4.0.0', ], + extras_require={ + '6': ['elasticsearch>=6.0.0,<7.0.0'], + '7': ['elasticsearch>=7.0.0,<8.0.0'], + }, python_requires='>=3.5.3', packages=['aioelasticsearch'], include_package_data=True, diff --git a/tests/conftest.py b/tests/conftest.py index 8927c103..e651c04a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,6 +7,7 @@ import pytest from aiohttp.test_utils import unused_port from docker import from_env as docker_from_env +from elasticsearch import __version__ as elasticsearch_lib_version import aioelasticsearch @@ -55,7 +56,10 @@ def pytest_generate_tests(metafunc): if 'es_tag' in metafunc.fixturenames: tags = set(metafunc.config.option.es_tag) if not tags: - tags = ['6.0.0'] + if elasticsearch_lib_version[0] == 6: + tags = ['6.0.0'] + else: + tags = ['7.0.0'] else: tags = list(tags) metafunc.parametrize("es_tag", tags, scope='session') @@ -213,22 +217,44 @@ def pytest_pyfunc_call(pyfuncitem): @pytest.fixture -def populate(es, loop): +def populate(es, loop, es_major_version): async def do(index, doc_type, n, body): coros = [] await es.indices.create(index) + kwargs = {} + + if es_major_version < 7: + kwargs['doc_type'] = doc_type + for i in range(n): coros.append( es.index( index=index, - doc_type=doc_type, id=str(i), body=body, + **kwargs, ), ) await asyncio.gather(*coros, loop=loop) await es.indices.refresh() return do + + +@pytest.fixture +def es_major_version(): + return elasticsearch_lib_version[0] + + +@pytest.fixture(autouse=True) +def skip_before_es7_marker(request, es_major_version): + if request.node.get_closest_marker('skip_before_es7') and es_major_version < 7: + pytest.skip('skipping this test for elasticsearch lib versions < 7') + + +@pytest.fixture(autouse=True) +def skip_after_es7_marker(request, es_major_version): + if request.node.get_closest_marker('skip_after_es7') and es_major_version >= 7: + pytest.skip('skipping this test for elasticsearch lib versions >= 7') diff --git a/tests/test_scan.py b/tests/test_scan.py index 4eb80eb6..ad23b0b5 100644 --- a/tests/test_scan.py +++ b/tests/test_scan.py @@ -32,8 +32,9 @@ def test_scan_scroll_id_without_context_manager(es): scan.scroll_id +@pytest.mark.skip_after_es7 @pytest.mark.run_loop -async def test_scan_simple(es, populate): +async def test_scan_simple_before_es7(es, populate): index = 'test_aioes' doc_type = 'type_2' scroll_size = 3 @@ -63,8 +64,74 @@ async def test_scan_simple(es, populate): assert ids == {str(i) for i in range(10)} +@pytest.mark.skip_before_es7 @pytest.mark.run_loop -async def test_scan_equal_chunks_for_loop(es, es_clean, populate): +async def test_scan_simple_after_es7(es, populate): + index = 'test_aioes' + scroll_size = 3 + n = 10 + + body = {'foo': 1} + await populate(index, None, n, body) + ids = set() + + async with Scan( + es, + index=index, + size=scroll_size, + ) as scan: + assert isinstance(scan.scroll_id, str) + assert scan.total == {'relation': 'eq', 'value': 10} + async for doc in scan: + ids.add(doc['_id']) + assert doc == {'_id': mock.ANY, + '_index': 'test_aioes', + '_score': None, + '_source': {'foo': 1}, + '_type': '_doc', + 'sort': mock.ANY} + + assert ids == {str(i) for i in range(10)} + + +@pytest.mark.skip_before_es7 +@pytest.mark.run_loop +async def test_scan_equal_chunks_for_loop_after_es7(es, es_clean, populate): + for n, scroll_size in [ + (0, 1), # no results + (6, 6), # 1 scroll + (6, 8), # 1 scroll + (6, 3), # 2 scrolls + (6, 4), # 2 scrolls + (6, 2), # 3 scrolls + (6, 1), # 6 scrolls + ]: + es_clean() + + index = 'test_aioes' + doc_type = 'type_1' + body = {'foo': 1} + + await populate(index, doc_type, n, body) + + ids = set() + + async with Scan( + es, + index=index, + size=scroll_size, + ) as scan: + + async for doc in scan: + ids.add(doc['_id']) + + # check number of unique doc ids + assert len(ids) == n == scan.total['value'] + + +@pytest.mark.skip_after_es7 +@pytest.mark.run_loop +async def test_scan_equal_chunks_for_loop_before_es7(es, es_clean, populate): for n, scroll_size in [ (0, 1), # no results (6, 6), # 1 scroll @@ -98,6 +165,27 @@ async def test_scan_equal_chunks_for_loop(es, es_clean, populate): assert len(ids) == n == scan.total +@pytest.mark.skip_before_es7 +@pytest.mark.run_loop +async def test_scan_no_mask_index(es): + index = 'undefined-*' + doc_type = 'any' + scroll_size = 3 + + async with Scan( + es, + index=index, + size=scroll_size, + ) as scan: + assert scan.scroll_id is None + assert scan.total == {'relation': 'eq', 'value': 0} + cnt = 0 + async for doc in scan: # noqa + cnt += 1 + assert cnt == 0 + + +@pytest.mark.skip_after_es7 @pytest.mark.run_loop async def test_scan_no_mask_index(es): index = 'undefined-*' @@ -141,16 +229,20 @@ async def test_scan_no_scroll(es, loop, populate): @pytest.mark.run_loop -async def test_scan_no_index(es): +async def test_scan_no_index(es, es_major_version): index = 'undefined' doc_type = 'any' scroll_size = 3 + scan_kwargs = {} + if es_major_version < 7: + scan_kwargs['doc_type'] = doc_type + async with Scan( es, index=index, - doc_type=doc_type, size=scroll_size, + **scan_kwargs, ) as scan: assert scan.scroll_id is None assert scan.total == 0 @@ -161,12 +253,16 @@ async def test_scan_no_index(es): @pytest.mark.run_loop -async def test_scan_warning_on_failed_shards(es, populate, mocker): +async def test_scan_warning_on_failed_shards(es, populate, mocker, es_major_version): index = 'test_aioes' doc_type = 'type_2' scroll_size = 3 n = 10 + scan_kwargs = {} + if es_major_version < 7: + scan_kwargs['doc_type'] = doc_type + body = {'foo': 1} await populate(index, doc_type, n, body) @@ -175,9 +271,9 @@ async def test_scan_warning_on_failed_shards(es, populate, mocker): async with Scan( es, index=index, - doc_type=doc_type, size=scroll_size, raise_on_error=False, + **scan_kwargs, ) as scan: i = 0 async for doc in scan: # noqa @@ -192,12 +288,16 @@ async def test_scan_warning_on_failed_shards(es, populate, mocker): @pytest.mark.run_loop -async def test_scan_exception_on_failed_shards(es, populate, mocker): +async def test_scan_exception_on_failed_shards(es, populate, mocker, es_major_version): index = 'test_aioes' doc_type = 'type_2' scroll_size = 3 n = 10 + scan_kwargs = {} + if es_major_version < 7: + scan_kwargs['doc_type'] = doc_type + body = {'foo': 1} await populate(index, doc_type, n, body) @@ -207,8 +307,8 @@ async def test_scan_exception_on_failed_shards(es, populate, mocker): async with Scan( es, index=index, - doc_type=doc_type, size=scroll_size, + **scan_kwargs, ) as scan: with pytest.raises(ScanError) as cm: async for doc in scan: # noqa diff --git a/tests/test_transport.py b/tests/test_transport.py index d7216456..9d414701 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -161,20 +161,59 @@ async def test_mark_dead_with_sniff(auto_close, loop, es_server): assert len(t.connection_pool.connections) == 1 +@pytest.mark.skip_before_es7 @pytest.mark.run_loop -async def test_send_get_body_as_post(es_server, auto_close, loop): +async def test_send_get_body_as_post_after_es7(es_server, auto_close, loop): cl = auto_close(Elasticsearch([{'host': es_server['host'], 'port': es_server['port']}], send_get_body_as='POST', http_auth=es_server['auth'], loop=loop)) - await cl.create('test', 'type', '1', {'val': '1'}) - await cl.create('test', 'type', '2', {'val': '2'}) + await cl.create('test', '1', {'val': '1'}) + await cl.create('test', '2', {'val': '2'}) ret = await cl.mget( {"docs": [ {"_id": "1"}, {"_id": "2"} ]}, + index='test', ) + assert ret == {'docs': [{'_id': '1', + '_index': 'test', + '_source': {'val': '1'}, + + '_version': 1, + '_primary_term': 1, + 'found': True, + '_seq_no': 0, + '_type': '_doc', + }, + {'_id': '2', + '_index': 'test', + '_source': {'val': '2'}, + + '_version': 1, + '_primary_term': 1, + 'found': True, + '_seq_no': 1, + '_type': '_doc', + }]} + + +@pytest.mark.skip_after_es7 +@pytest.mark.run_loop +async def test_send_get_body_as_post_before_es7(es_server, auto_close, loop): + cl = auto_close(Elasticsearch([{'host': es_server['host'], + 'port': es_server['port']}], + send_get_body_as='POST', + http_auth=es_server['auth'], + loop=loop)) + await cl.create('test', 'type', '1', {'val': '1'}) + await cl.create('test', 'type', '2', {'val': '2'}) + ret = await cl.mget( + {"docs": [ + {"_id": "1"}, + {"_id": "2"} + ]}, index='test', doc_type='type') assert ret == {'docs': [{'_id': '1', '_index': 'test', @@ -190,8 +229,47 @@ async def test_send_get_body_as_post(es_server, auto_close, loop): 'found': True}]} +@pytest.mark.skip_before_es7 +@pytest.mark.run_loop +async def test_send_get_body_as_source_after_es7(es_server, auto_close, loop): + cl = auto_close(Elasticsearch([{'host': es_server['host'], + 'port': es_server['port']}], + send_get_body_as='source', + http_auth=es_server['auth'], + loop=loop)) + await cl.create('test', '1', {'val': '1'}) + await cl.create('test', '2', {'val': '2'}) + ret = await cl.mget( + {"docs": [ + {"_id": "1"}, + {"_id": "2"} + ]}, + index='test', ) + assert ret == {'docs': [{'_id': '1', + '_index': 'test', + '_source': {'val': '1'}, + + '_version': 1, + '_primary_term': 1, + 'found': True, + '_seq_no': 0, + '_type': '_doc', + }, + {'_id': '2', + '_index': 'test', + '_source': {'val': '2'}, + + '_version': 1, + '_primary_term': 1, + 'found': True, + '_seq_no': 1, + '_type': '_doc', + }]} + + +@pytest.mark.skip_after_es7 @pytest.mark.run_loop -async def test_send_get_body_as_source(es_server, auto_close, loop): +async def test_send_get_body_as_source_before_es7(es_server, auto_close, loop): cl = auto_close(Elasticsearch([{'host': es_server['host'], 'port': es_server['port']}], send_get_body_as='source', @@ -219,19 +297,58 @@ async def test_send_get_body_as_source(es_server, auto_close, loop): 'found': True}]} +@pytest.mark.skip_before_es7 @pytest.mark.run_loop -async def test_send_get_body_as_get(es_server, auto_close, loop): +async def test_send_get_body_as_get_after_es7(es_server, auto_close, loop): cl = auto_close(Elasticsearch([{'host': es_server['host'], 'port': es_server['port']}], http_auth=es_server['auth'], loop=loop)) - await cl.create('test', 'type', '1', {'val': '1'}) - await cl.create('test', 'type', '2', {'val': '2'}) + await cl.create('test', '1', {'val': '1'}) + await cl.create('test', '2', {'val': '2'}) ret = await cl.mget( {"docs": [ {"_id": "1"}, {"_id": "2"} ]}, + index='test', ) + assert ret == {'docs': [{'_id': '1', + '_index': 'test', + '_source': {'val': '1'}, + + '_version': 1, + '_primary_term': 1, + 'found': True, + '_seq_no': 0, + '_type': '_doc', + }, + {'_id': '2', + '_index': 'test', + '_source': {'val': '2'}, + + + '_version': 1, + '_primary_term': 1, + 'found': True, + '_seq_no': 1, + '_type': '_doc', + }]} + + +@pytest.mark.skip_after_es7 +@pytest.mark.run_loop +async def test_send_get_body_as_get_before_es7(es_server, auto_close, loop): + cl = auto_close(Elasticsearch([{'host': es_server['host'], + 'port': es_server['port']}], + http_auth=es_server['auth'], + loop=loop)) + await cl.create('test', 'type', '1', {'val': '1'}) + await cl.create('test', 'type', '2', {'val': '2'}) + ret = await cl.mget( + {"docs": [ + {"_id": "1"}, + {"_id": "2"} + ]}, index='test', doc_type='type') assert ret == {'docs': [{'_id': '1', '_index': 'test', @@ -247,14 +364,54 @@ async def test_send_get_body_as_get(es_server, auto_close, loop): 'found': True}]} +@pytest.mark.skip_before_es7 @pytest.mark.run_loop -async def test_send_get_body_as_source_none_params(es_server, - auto_close, loop): +async def test_send_get_body_as_source_none_params_after_es7(es_server, + auto_close, loop): cl = auto_close(Elasticsearch([{'host': es_server['host'], 'port': es_server['port']}], send_get_body_as='source', http_auth=es_server['auth'], loop=loop)) + await cl.create('test', '1', {'val': '1'}) + await cl.create('test', '2', {'val': '2'}) + ret = await cl.transport.perform_request( + 'GET', 'test/_mget', + body={"docs": [ + {"_id": "1"}, + {"_id": "2"} + ]}) + assert ret == {'docs': [{'_id': '1', + '_index': 'test', + '_source': {'val': '1'}, + + '_version': 1, + '_primary_term': 1, + 'found': True, + '_seq_no': 0, + '_type': '_doc', + }, + {'_id': '2', + '_index': 'test', + '_source': {'val': '2'}, + + '_version': 1, + '_primary_term': 1, + 'found': True, + '_seq_no': 1, + '_type': '_doc', + }]} + + +@pytest.mark.skip_after_es7 +@pytest.mark.run_loop +async def test_send_get_body_as_source_none_params_before_es7(es_server, + auto_close, loop): + cl = auto_close(Elasticsearch([{'host': es_server['host'], + 'port': es_server['port']}], + send_get_body_as='source', + http_auth=es_server['auth'], + loop=loop)) await cl.create('test', 'type', '1', {'val': '1'}) await cl.create('test', 'type', '2', {'val': '2'}) ret = await cl.transport.perform_request( From 9500ec06712f443da0840274c5c36c9a15a51603 Mon Sep 17 00:00:00 2001 From: Taras Pashkevych Date: Sat, 20 Apr 2019 18:35:25 +0300 Subject: [PATCH 2/5] comprehensible error for missing elasticsearch-py dependency during import --- aioelasticsearch/__init__.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/aioelasticsearch/__init__.py b/aioelasticsearch/__init__.py index 2d9edb6e..b44e4af5 100644 --- a/aioelasticsearch/__init__.py +++ b/aioelasticsearch/__init__.py @@ -1,9 +1,16 @@ import asyncio -from elasticsearch import Elasticsearch as _Elasticsearch # noqa # isort:skip -from elasticsearch.connection_pool import (ConnectionSelector, # noqa # isort:skip - RoundRobinSelector) -from elasticsearch.serializer import JSONSerializer # noqa # isort:skip +try: + from elasticsearch import Elasticsearch as _Elasticsearch # noqa # isort:skip + from elasticsearch.connection_pool import (ConnectionSelector, # noqa # isort:skip + RoundRobinSelector) + from elasticsearch.serializer import JSONSerializer # noqa # isort:skip +except ImportError: # pragma: no cover + raise RuntimeError( + 'Please reinstall the library specifying ES version -- ' + 'pip install aioelasticsearch[6] OR pip install aioelasticsearch[7]\n' + '\t\t or install elasticsearch-py manually -- https://github.com/elastic/elasticsearch-py.') + from .exceptions import * # noqa # isort:skip from .pool import AIOHttpConnectionPool # noqa # isort:skip From 50eed213c0272b6c4eb1691dababc7f6d8f187d6 Mon Sep 17 00:00:00 2001 From: Taras Pashkevych Date: Tue, 18 Jun 2019 20:54:28 +0300 Subject: [PATCH 3/5] add default es lib version as extras for tox --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index ca96238d..9a8da4e4 100644 --- a/tox.ini +++ b/tox.ini @@ -5,6 +5,8 @@ skip_missing_interpreters = True [testenv] deps = -r{toxinidir}/requirements.txt +extras = 6 + commands = pytest tests {posargs} [testenv:coverage] From 951230ca326990827dd51f17f5efe2fb4bd580d7 Mon Sep 17 00:00:00 2001 From: Taras Pashkevych Date: Tue, 18 Jun 2019 22:40:30 +0300 Subject: [PATCH 4/5] pip ipgrade --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index a18c4fa1..c4b40fc7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ python: - "3.6" - "3.7" install: + - pip install --upgrade pip - pip install -U tox script: - tox -- --es_tag=${ES_TAG} From 683564b9fff885b8930e81cf0d06c54500f5506f Mon Sep 17 00:00:00 2001 From: Taras Pashkevych Date: Tue, 18 Jun 2019 23:56:17 +0300 Subject: [PATCH 5/5] upgrade setuptools in travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index c4b40fc7..f07d6a82 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ python: - "3.7" install: - pip install --upgrade pip + - pip install --upgrade setuptools - pip install -U tox script: - tox -- --es_tag=${ES_TAG}