Add test support for python 3.6. (#90)

mbasset · web-flow · commit bacbc5818bdc · 2017-06-14T15:36:36.000-04:00
* Add test support for python 3.6.
Raise error when over 1000000 rows when fetching tables with paginate
Pandas fix for newer pandas version.
Bump package version 3.2.0

* The newest version of requests breaks the package.

* casing

* minor correction.

* Don’t need this step.
diff --git a/.travis.yml b/.travis.yml
@@ -5,9 +5,9 @@ python:
   - "3.3"
   - "3.4"
   - "3.5"
+  - "3.6"
 install:
   - pip install flake8
-  - pip install -r requirements.txt
 script:
   - flake8
   - python -W always setup.py -q test
diff --git a/quandl/get_table.py b/quandl/get_table.py
@@ -1,9 +1,9 @@
 from quandl.model.datatable import Datatable
+from quandl.errors.quandl_error import LimitExceededError
 from .api_config import ApiConfig
 from .message import Message
 import warnings
 import copy
-import os
 
 
 def get_table(datatable_code, **options):
@@ -24,17 +24,16 @@ def get_table(datatable_code, **options):
             data.extend(next_data)
 
         if page_count >= ApiConfig.page_limit:
-            if os.isatty(0):
-                warnings.warn(Message.WARN_DATA_LIMIT_EXCEEDED, UserWarning)
-            break
+            raise LimitExceededError(Message.WARN_DATA_LIMIT_EXCEEDED)
 
         next_cursor_id = next_data.meta['next_cursor_id']
+
         if next_cursor_id is None:
             break
         elif paginate is not True and next_cursor_id is not None:
-            if os.isatty(0):
-                warnings.warn(Message.WARN_PAGE_LIMIT_EXCEEDED, UserWarning)
+            warnings.warn(Message.WARN_PAGE_LIMIT_EXCEEDED, UserWarning)
             break
+
         page_count = page_count + 1
         options['qopts.cursor_id'] = next_cursor_id
     return data.to_pandas()
diff --git a/quandl/model/data_mixin.py b/quandl/model/data_mixin.py
@@ -6,6 +6,7 @@ class DataMixin(object):
     # DataFrame will respect order of input list of list
     def to_pandas(self, keep_column_indexes=[]):
         data = self.to_list()
+
         # ensure pandas gets a list of lists
         if data and isinstance(data, list) and not isinstance(data[0], list):
             data = [data]
@@ -22,14 +23,15 @@ def to_pandas(self, keep_column_indexes=[]):
 
         # unfortunately to_records() cannot handle unicode in 2.7
         df.index.name = str(df.index.name)
+
         # keep_column_indexes are 0 based, 0 is the first column
         if len(keep_column_indexes) > 0:
             self._validate_col_index(df, keep_column_indexes)
             # need to decrement all our indexes by 1 because
             # Date is considered a column by our API, but in pandas,
             # it is the index, so column 0 is the first column after Date index
             keep_column_indexes = list([x - 1 for x in keep_column_indexes])
-            df = df[keep_column_indexes]
+            df = df.iloc[:, keep_column_indexes]
         return df
 
     def to_numpy(self):
diff --git a/quandl/model/merged_dataset.py b/quandl/model/merged_dataset.py
@@ -162,7 +162,7 @@ def _initialize_raw_data(self):
     def _build_dataset_object(self, dataset_code, **options):
         options_copy = options.copy()
         # data_codes are tuples
-        # e.g., ('GOOG/NASDAQ_AAPL', {'column_index": [1,2]})
+        # e.g., ('WIKI/AAPL', {'column_index": [1,2]})
         # or strings
         # e.g., 'NSE/OIL'
         code = self._get_request_dataset_code(dataset_code)
diff --git a/quandl/version.py b/quandl/version.py
@@ -1 +1 @@
-VERSION = '3.1.0'
+VERSION = '3.2.0'
diff --git a/setup.cfg b/setup.cfg
@@ -1,2 +1,6 @@
 [bdist_wheel]
 universal = 1
+
+[flake8]
+max-line-length = 100
+exclude = .git,__init__.py,tmp,__pycache__,.eggs,Quandl.egg-info,build,dist,.tox
diff --git a/setup.py b/setup.py
@@ -18,7 +18,7 @@
 install_requires = [
     'pandas >= 0.14',
     'numpy >= 1.8',
-    'requests >= 2.7.0',
+    'requests >= 2.7.0, < 2.18',  # Version 2.18 appears to break pulling data.
     'inflection >= 0.3.1',
     'python-dateutil',
     'six',
diff --git a/test/helpers/merged_datasets_helper.py b/test/helpers/merged_datasets_helper.py
@@ -30,41 +30,41 @@ def setupDatasetsTest(unit_test, httpretty):
     unit_test.nse_oil = {'dataset': DatasetFactory.build(
         database_code='NSE', dataset_code='OIL')}
 
-    unit_test.goog_aapl = {'dataset': DatasetFactory.build(
-        database_code='GOOG', dataset_code='NASDAQ_AAPL')}
+    unit_test.wiki_aapl = {'dataset': DatasetFactory.build(
+        database_code='WIKI', dataset_code='AAPL')}
 
-    unit_test.goog_msft = {'dataset': DatasetFactory.build(
-        database_code='GOOG', dataset_code='NASDAQ_MSFT',
+    unit_test.wiki_msft = {'dataset': DatasetFactory.build(
+        database_code='WIKI', dataset_code='MSFT',
         newest_available_date='2015-07-30', oldest_available_date='2013-01-01')}
 
     unit_test.single_col = {'dataset': DatasetFactory.build(
         database_code='SINGLE', dataset_code='COLUMN',
         newest_available_date='2015-07-30', oldest_available_date='2013-01-01')}
 
     unit_test.oil_obj = Dataset('NSE/OIL', unit_test.nse_oil['dataset'])
-    unit_test.aapl_obj = Dataset('GOOG/AAPL', unit_test.goog_aapl['dataset'])
-    unit_test.goog_obj = Dataset('GOOG/MSFT', unit_test.goog_msft['dataset'])
+    unit_test.aapl_obj = Dataset('WIKI/AAPL', unit_test.wiki_aapl['dataset'])
+    unit_test.wiki_obj = Dataset('WIKI/MSFT', unit_test.wiki_msft['dataset'])
     unit_test.single_col_obj = Dataset('SINGLE/COLUMN', unit_test.single_col['dataset'])
 
     httpretty.register_uri(httpretty.GET,
                            re.compile(
                                'https://www.quandl.com/api/v3/datasets/.*/metadata'),
                            responses=[httpretty.Response(body=json.dumps(dataset))
                                       for dataset in
-                                      [unit_test.nse_oil, unit_test.goog_aapl,
-                                       unit_test.goog_msft]])
+                                      [unit_test.nse_oil, unit_test.wiki_aapl,
+                                       unit_test.wiki_msft]])
     # mock our query param column_index request
     httpretty.register_uri(httpretty.GET,
                            "https://www.quandl.com/api/v3/datasets/SINGLE/COLUMN/data",
                            body=json.dumps(unit_test.single_dataset_data))
     httpretty.register_uri(httpretty.GET,
-                           "https://www.quandl.com/api/v3/datasets/GOOG/NASDAQ_AAPL/data",
+                           "https://www.quandl.com/api/v3/datasets/WIKI/AAPL/data",
                            body=json.dumps(unit_test.dataset_data))
     httpretty.register_uri(httpretty.GET,
                            re.compile(
                                'https://www.quandl.com/api/v3/datasets/NSE/OIL/data'),
                            body=json.dumps(unit_test.dataset_data))
     httpretty.register_uri(httpretty.GET,
                            re.compile(
-                               'https://www.quandl.com/api/v3/datasets/GOOG/NASDAQ_MSFT/data'),
+                               'https://www.quandl.com/api/v3/datasets/WIKI/MSFT/data'),
                            body=json.dumps(unit_test.dataset_data))
diff --git a/test/test_connection.py b/test/test_connection.py
@@ -67,6 +67,6 @@ def test_build_request(self, mock):
                                  'accept': ('application/json, '
                                             'application/vnd.quandl+json;version=2015-04-09'),
                                  'request-source': 'python',
-                                 'request-source-version': '3.1.0'},
+                                 'request-source-version': '3.2.0'},
                         params={'per_page': 10, 'page': 2})
         self.assertEqual(mock.call_args, expected)
diff --git a/test/test_get.py b/test/test_get.py
@@ -121,15 +121,15 @@ def test_multiple_datasets_args_formed(self, mock):
         # requested_column_indexes is a dynamically added attribute
         self.oil_obj.requested_column_indexes = []
         mock.return_value = self.oil_obj
-        get(['GOOG/NASDAQ_AAPL.1', 'GOOG/NASDAQ_MSFT.2', 'NSE/OIL'])
-        expected = [call(('GOOG/NASDAQ_AAPL', {'column_index': [1]})),
-                    call(('GOOG/NASDAQ_MSFT', {'column_index': [2]})),
+        get(['WIKI/AAPL.1', 'WIKI/MSFT.2', 'NSE/OIL'])
+        expected = [call(('WIKI/AAPL', {'column_index': [1]})),
+                    call(('WIKI/MSFT', {'column_index': [2]})),
                     call('NSE/OIL')]
         self.assertEqual(mock.call_args_list, expected)
 
     @patch.object(MergedDataset, 'data')
     def test_query_params_are_formed_with_old_arg_names(self, mock_method):
-        get(['GOOG/NASDAQ_AAPL.1', 'GOOG/NASDAQ_MSFT.2', 'NSE/OIL'],
+        get(['WIKI/AAPL.1', 'WIKI/MSFT.2', 'NSE/OIL'],
             authtoken='authtoken', trim_start='2001-01-01',
             trim_end='2010-01-01', collapse='annual',
             transformation='rdiff', rows=4, sort_order='desc')
@@ -143,7 +143,7 @@ def test_query_params_are_formed_with_old_arg_names(self, mock_method):
 
     @patch.object(MergedDataset, 'data')
     def test_query_params_are_formed_with_new_arg_names(self, mock_method):
-        get(['GOOG/NASDAQ_AAPL.1', 'GOOG/NASDAQ_MSFT.2', 'NSE/OIL'],
+        get(['WIKI/AAPL.1', 'WIKI/MSFT.2', 'NSE/OIL'],
             api_key='authtoken', start_date='2001-01-01',
             end_date='2010-01-01', collapse='annual',
             transform='rdiff', rows=4, order='desc')
diff --git a/test/test_merged_dataset.py b/test/test_merged_dataset.py
@@ -28,14 +28,14 @@ def test_merged_dataset_calls_merged_dataset_get_dataset(self, mock):
         mock.return_value = self.oil_obj
         md = MergedDataset(
             [('NSE/OIL', {'column_index': [1, 2]}),
-             ('GOOG/NASDAQ_AAPL', {'column_index': [1]}),
-             ('GOOG/NASDAQ_MSFT')])
+             ('WIKI/AAPL', {'column_index': [1]}),
+             ('WIKI/MSFT')])
         md.data_fields()
 
         expected_calls = [
             call(('NSE/OIL', {'column_index': [1, 2]})),
-            call(('GOOG/NASDAQ_AAPL', {'column_index': [1]})),
-            call('GOOG/NASDAQ_MSFT')
+            call(('WIKI/AAPL', {'column_index': [1]})),
+            call('WIKI/MSFT')
         ]
         self.assertEqual(mock.call_count, 3)
         for index, expected in enumerate(expected_calls):
@@ -54,18 +54,18 @@ def test_removes_column_index_query_param(self, mock):
     def test_sets_dataset_codes_for_the_datasets(self):
         md = MergedDataset(
             [('NSE/OIL', {'column_index': [1, 2]}),
-             ('GOOG/NASDAQ_AAPL', {'column_index': [1]}),
-             ('GOOG/NASDAQ_MSFT')])
+             ('WIKI/AAPL', {'column_index': [1]}),
+             ('WIKI/MSFT')])
         self.assertEqual(md._datasets, None)
         self.assertItemsEqual([1, 2], md.dataset_codes[0][1]['column_index'])
         self.assertItemsEqual([1], md.dataset_codes[1][1]['column_index'])
-        self.assertEqual('O', md.dataset_codes[2][1])
+        self.assertEqual('I', md.dataset_codes[2][1])
 
     def test_sets_column_index_on_each_dataset(self):
         md = MergedDataset(
             [('NSE/OIL', {'column_index': [1, 2]}),
-             ('GOOG/NASDAQ_AAPL', {'column_index': [1]}),
-             ('GOOG/NASDAQ_MSFT')])
+             ('WIKI/AAPL', {'column_index': [1]}),
+             ('WIKI/MSFT')])
         md.data_fields()
         self.assertItemsEqual([1, 2], md._datasets[0].requested_column_indexes)
         self.assertItemsEqual([1], md._datasets[1].requested_column_indexes)
@@ -74,44 +74,44 @@ def test_sets_column_index_on_each_dataset(self):
     def test_merged_dataset_column_names(self):
         md = MergedDataset(
             [('NSE/OIL', {'column_index': [1, 2]}),
-             ('GOOG/NASDAQ_AAPL', {'column_index': [1]}),
-             ('GOOG/NASDAQ_MSFT')])
+             ('WIKI/AAPL', {'column_index': [1]}),
+             ('WIKI/MSFT')])
         expected = [six.u('Date'), six.u('NSE/OIL - column.1'),
                     six.u('NSE/OIL - column.2'),
-                    six.u('GOOG/NASDAQ_AAPL - column.1'),
-                    six.u('GOOG/NASDAQ_MSFT - column.1'),
-                    six.u('GOOG/NASDAQ_MSFT - column.2'),
-                    six.u('GOOG/NASDAQ_MSFT - column.3')]
+                    six.u('WIKI/AAPL - column.1'),
+                    six.u('WIKI/MSFT - column.1'),
+                    six.u('WIKI/MSFT - column.2'),
+                    six.u('WIKI/MSFT - column.3')]
         self.assertItemsEqual(md.column_names, expected)
 
     def test_merged_dataset_oldest_available_date(self):
         md = MergedDataset(
             [('NSE/OIL', {'column_index': [1, 2]}),
-             ('GOOG/NASDAQ_AAPL', {'column_index': [1]}),
-             ('GOOG/NASDAQ_MSFT')])
+             ('WIKI/AAPL', {'column_index': [1]}),
+             ('WIKI/MSFT')])
         self.assertEqual(md.oldest_available_date, datetime.date(2013, 1, 1))
 
     def test_merged_dataset_newest_available_date(self):
         md = MergedDataset(
             [('NSE/OIL', {'column_index': [1, 2]}),
-             ('GOOG/NASDAQ_AAPL', {'column_index': [1]}),
-             ('GOOG/NASDAQ_MSFT')])
+             ('WIKI/AAPL', {'column_index': [1]}),
+             ('WIKI/MSFT')])
         self.assertEqual(md.newest_available_date, datetime.date(2015, 7, 30))
 
     def test_merged_dataset_database_codes(self):
         md = MergedDataset(
             [('NSE/OIL', {'column_index': [1, 2]}),
-             ('GOOG/NASDAQ_AAPL', {'column_index': [1]}),
-             ('GOOG/NASDAQ_MSFT')])
-        self.assertItemsEqual(md.database_code, ['NSE', 'GOOG'])
+             ('WIKI/AAPL', {'column_index': [1]}),
+             ('WIKI/MSFT')])
+        self.assertItemsEqual(md.database_code, ['NSE', 'WIKI'])
 
     def test_merged_dataset_dataset_codes(self):
         md = MergedDataset(
             [('NSE/OIL', {'column_index': [1, 2]}),
-             ('GOOG/NASDAQ_AAPL', {'column_index': [1]}),
-             ('GOOG/NASDAQ_MSFT')])
+             ('WIKI/AAPL', {'column_index': [1]}),
+             ('WIKI/MSFT')])
         self.assertItemsEqual(
-            md.dataset_code, ['OIL', 'NASDAQ_AAPL', 'NASDAQ_MSFT'])
+            md.dataset_code, ['OIL', 'AAPL', 'MSFT'])
 
     def test_get_returns_merged_dataset_obj(self):
         md = MergedDataset(['NSE/OIL'])
@@ -141,8 +141,8 @@ def test_when_only_one_column_requested_adds_column_index_query_param(self, mock
         mock_method.return_value = self.data_list_obj
         MergedDataset(
             [('NSE/OIL', {'column_index': [1, 2]}),
-             ('GOOG/NASDAQ_AAPL', {'column_index': [1]}),
-             ('GOOG/NASDAQ_MSFT')]).data(params={'start_date': '2015-07-01'})
+             ('WIKI/AAPL', {'column_index': [1]}),
+             ('WIKI/MSFT')]).data(params={'start_date': '2015-07-01'})
         expected_calls = [call(params={'start_date': '2015-07-01'}),
                           call(params={'column_index': 1, 'start_date': '2015-07-01'}),
                           call(params={'start_date': '2015-07-01'})]
@@ -154,50 +154,50 @@ def test_when_only_one_column_requested_adds_column_index_query_param(self, mock
     def test_data_forwards_requests_to_datset_data(self, mock_method):
         mock_method.return_value = self.data_list_obj
         MergedDataset(
-            ['NSE/OIL', 'GOOG/NASDAQ_AAPL',
-             'GOOG/NASDAQ_MSFT']).data(params={'start_date': '2015-07-01'})
+            ['NSE/OIL', 'WIKI/AAPL',
+             'WIKI/MSFT']).data(params={'start_date': '2015-07-01'})
         self.assertEqual(mock_method.call_count, 3)
         for actual in mock_method.mock_calls:
             self.assertEqual(actual, call(params={'start_date': '2015-07-01'}))
 
     def test_get_merged_dataset_data_returns_correct_types(self):
         data = MergedDataset(
             [('NSE/OIL', {'column_index': [1, 2]}),
-             ('GOOG/NASDAQ_AAPL', {'column_index': [1]}),
-             ('GOOG/NASDAQ_MSFT')]).data()
+             ('WIKI/AAPL', {'column_index': [1]}),
+             ('WIKI/MSFT')]).data()
         self.assertIsInstance(data, MergedDataList)
         self.assertIsInstance(data[0], Data)
 
     def test_get_merged_dataset_creates_merged_pandas_dataframe(self):
         data = MergedDataset(
             [('NSE/OIL', {'column_index': [1, 2]}),
-             ('GOOG/NASDAQ_AAPL', {'column_index': [1]}),
-             ('GOOG/NASDAQ_MSFT')]).data()
+             ('WIKI/AAPL', {'column_index': [1]}),
+             ('WIKI/MSFT')]).data()
         self.assertIsInstance(data.to_pandas(), pandas.core.frame.DataFrame)
 
     def test_get_merged_dataset_data_returns_specified_columns(self):
         data = MergedDataset(
             [('NSE/OIL', {'column_index': [1, 2]}),
              ('SINGLE/COLUMN', {'column_index': [1]}),
-             ('GOOG/NASDAQ_MSFT')]).data()
+             ('WIKI/MSFT')]).data()
         actual = data.to_pandas().columns.tolist()
         expected = [six.u('NSE/OIL - column.1'),
                     six.u('NSE/OIL - column.2'),
                     six.u('SINGLE/COLUMN - column.1'),
-                    six.u('GOOG/NASDAQ_MSFT - column.1'),
-                    six.u('GOOG/NASDAQ_MSFT - column.2'),
-                    six.u('GOOG/NASDAQ_MSFT - column.3')]
+                    six.u('WIKI/MSFT - column.1'),
+                    six.u('WIKI/MSFT - column.2'),
+                    six.u('WIKI/MSFT - column.3')]
         self.assertItemsEqual(actual, expected)
 
     def test_get_merged_dataset_data_to_list(self):
         data = MergedDataset(
             [('NSE/OIL', {'column_index': [1, 2]}),
              ('SINGLE/COLUMN', {'column_index': [1]}),
-             'GOOG/NASDAQ_MSFT']).data()
+             'WIKI/MSFT']).data()
         results = data.to_list()
         # NSE/OIL two columns of data
         # SINGLE/COLUMN one column of data
-        # GOOG/NASDAQ_MSFT all 3 columns of data
+        # WIKI/MSFT all 3 columns of data
         expected = [[datetime.datetime(2015, 7, 11, 0, 0), 444.3, 10, 444.3, 444.3, 10, 3],
                     [datetime.datetime(2015, 7, 13, 0, 0), 433.3, 4, 433.3, 433.3, 4, 3],
                     [datetime.datetime(2015, 7, 14, 0, 0), 437.5, 3, 437.5, 437.5, 3, 3],
@@ -206,8 +206,8 @@ def test_get_merged_dataset_data_to_list(self):
             self.assertItemsEqual(results[index], expected_item)
 
     def test_get_merged_dataset_data_is_descending_when_specified_in_params(self):
-        data = MergedDataset(['NSE/OIL', 'GOOG/NASDAQ_AAPL',
-                              'GOOG/NASDAQ_MSFT']).data(params={'order': 'desc'})
+        data = MergedDataset(['NSE/OIL', 'WIKI/AAPL',
+                              'WIKI/MSFT']).data(params={'order': 'desc'})
         results = data.to_list()
         dates = list([x[0] for x in results])
         self.assertTrue(all(dates[i] >= dates[i + 1]
diff --git a/tox.ini b/tox.ini
@@ -2,12 +2,8 @@
 # in multiple virtualenvs. This configuration file will run the
 # test suite on all supported python versions. To use it, "pip install tox"
 # and then run "tox" from this directory.
-[flake8]
-max-line-length = 100
-exclude = __init__.py, tmp/*
-
 [tox]
-envlist = py2.7,py3.0,py3.1,py3.2,py3.3,py3.4,py3.5
+envlist = py2.7,py3.3,py3.4,py3.5,py3.6
 
 [testenv]
 commands =
