Merge pull request #685 from gliptak/jsdmx1

Fix quarter value handling for JSDMX / OECD (+ test)

Author: bashtage

pandas_datareader/io/jsdmx.py

@@ -4,6 +4,7 @@
 
 from collections import OrderedDict
 import itertools
+import re
 import sys
 
 import numpy as np
@@ -19,7 +20,7 @@ def read_jsdmx(path_or_buf):
     Parameters
     ----------
     path_or_buf : a valid SDMX-JSON string or file-like
-        http://sdmx.org/wp-content/uploads/2014/07/sdmx-json-data-message.pdf
+        https://github.com/sdmx-twg/sdmx-json
 
     Returns
     -------
@@ -63,6 +64,16 @@ def _get_indexer(index):
         return [':'.join(map(str, i)) for i in it]
 
 
+def _fix_quarter_values(value):
+    """Make raw quarter values Pandas-friendly (e.g. 'Q4-2018' -> '2018Q4')."""
+    m = re.match(r'Q([1-4])-(\d\d\d\d)', value)
+    if not m:
+        return value
+    quarter, year = m.groups()
+    value = '%sQ%s' % (quarter, year)
+    return value
+
+
 def _parse_values(dataset, index, columns):
     size = len(index)
     series = dataset['series']
@@ -94,6 +105,7 @@ def _parse_dimensions(dimensions):
 
         role = key.get('role', None)
         if role in ('time', 'TIME_PERIOD'):
+            values = [_fix_quarter_values(v) for v in values]
             values = pd.DatetimeIndex(values)
 
         arrays.append(values)

pandas_datareader/tests/io/data/jsdmx/oecd1.json

@@ -0,0 +1 @@
+{"header":{"id":"21d3962b-daf2-4592-935b-95f739817c5e","test":false,"prepared":"2019-09-17T01:11:32.7663652Z","sender":{"id":"OECD","name":"Organisation for Economic Co-operation and Development"},"links":[{"href":"https://stats.oecd.org:443/sdmx-json/data/QNA/AUS+AUT+BEL+CAN+CHL.GDP+B1_GE.CUR+VOBARSA.Q/all?startTime=2009-Q1&endTime=2011-Q4","rel":"request"}]},"dataSets":[{"action":"Information","series":{"0:0:0:0":{"attributes":[0,0,0,0],"observations":{"0":[1321158.283224,null],"1":[1329284.411391,null],"2":[1332420.036934,null],"3":[1341900.519795,null],"4":[1348826.866358,null],"5":[1357309.984757,null],"6":[1366135.372141,null],"7":[1379528.026307,null],"8":[1375564.330639,null],"9":[1391739.300428,null],"10":[1409471.042053,null],"11":[1425778.503062,null]}},"1:0:0:0":{"attributes":[0,1,0,0],"observations":{"0":[291301.971016,null],"1":[288549.835025,null],"2":[289832.36302,null],"3":[292539.931011,null],"4":[290100.39902,null],"5":[294924.315003,null],"6":[298297.998992,null],"7":[299788.246987,null],"8":[303845.586973,null],"9":[304862.62297,null],"10":[304928.114969,null],"11":[305691.786967,null]}},"2:0:0:0":{"attributes":[0,1,0,0],"observations":{"0":[352900.125286,null],"1":[352536.276624,null],"2":[356476.132846,null],"3":[359482.478359,null],"4":[361114.28448,null],"5":[364793.198733,null],"6":[366454.406767,null],"7":[368038.434781,null],"8":[370383.237272,null],"9":[371467.432781,null],"10":[372294.361559,null],"11":[372522.226377,null]}},"3:0:0:0":{"attributes":[0,2,0,0],"observations":{"0":[1621076.204491,null],"1":[1603270.878032,null],"2":[1610574.277634,null],"3":[1629547.229243,null],"4":[1648959.475887,null],"5":[1657505.152083,null],"6":[1669438.855352,null],"7":[1688284.516678,null],"8":[1700768.535079,null],"9":[1704049.370335,null],"10":[1727690.908401,null],"11":[1741393.276651,null]}},"4:0:0:0":{"attributes":[0,3,0,0],"observations":{"0":[104697282.668229,null],"1":[104198162.095999,null],"2":[105452700.611771,null],"3":[106755819.776836,null],"4":[106928265.22694,null],"5":[110749230.548409,null],"6":[113428382.094559,null],"7":[114793008.286418,null],"8":[116091875.972295,null],"9":[117789649.277033,null],"10":[118321699.337605,null],"11":[120601874.99935,null]}}}}],"structure":{"links":[{"href":"https://stats.oecd.org/sdmx-json/dataflow/QNA/all","rel":"dataflow"}],"name":"Quarterly National Accounts","description":"Quarterly National Accounts","dimensions":{"series":[{"keyPosition":0,"id":"LOCATION","name":"Country","values":[{"id":"AUS","name":"Australia"},{"id":"AUT","name":"Austria"},{"id":"BEL","name":"Belgium"},{"id":"CAN","name":"Canada"},{"id":"CHL","name":"Chile"}],"role":"REF_AREA"},{"keyPosition":1,"id":"SUBJECT","name":"Subject","values":[{"id":"B1_GE","name":"Gross domestic product - expenditure approach"}]},{"keyPosition":2,"id":"MEASURE","name":"Measure","values":[{"id":"VOBARSA","name":"National currency, volume estimates, OECD reference year, annual levels, seasonally adjusted"}]},{"keyPosition":3,"id":"FREQUENCY","name":"Frequency","values":[{"id":"Q","name":"Quarterly"}],"role":"FREQ"}],"observation":[{"id":"TIME_PERIOD","name":"Period","values":[{"id":"2009-Q1","name":"Q1-2009"},{"id":"2009-Q2","name":"Q2-2009"},{"id":"2009-Q3","name":"Q3-2009"},{"id":"2009-Q4","name":"Q4-2009"},{"id":"2010-Q1","name":"Q1-2010"},{"id":"2010-Q2","name":"Q2-2010"},{"id":"2010-Q3","name":"Q3-2010"},{"id":"2010-Q4","name":"Q4-2010"},{"id":"2011-Q1","name":"Q1-2011"},{"id":"2011-Q2","name":"Q2-2011"},{"id":"2011-Q3","name":"Q3-2011"},{"id":"2011-Q4","name":"Q4-2011"}],"role":"TIME_PERIOD"}]},"attributes":{"dataSet":[],"series":[{"id":"TIME_FORMAT","name":"Time Format","values":[{"id":"P3M","name":"Quarterly"}]},{"id":"UNIT","name":"Unit","values":[{"id":"AUD","name":"Australian Dollar"},{"id":"EUR","name":"Euro"},{"id":"CAD","name":"Canadian Dollar"},{"id":"CLP","name":"Chilean Peso"}],"role":"UNIT_MEASURE"},{"id":"POWERCODE","name":"Unit multiplier","default":"0","values":[{"id":"6","name":"Millions"}],"role":"UNIT_MULT"},{"id":"REFERENCEPERIOD","name":"Reference period","values":[{"id":"2010","name":"2010"}],"role":"BASE_PER"}],"observation":[{"id":"OBS_STATUS","name":"Observation Status","values":[]}]},"annotations":[{"title":"Copyright OECD - All rights reserved","uri":"","text":""},{"title":"Terms and Conditions","uri":"http://www.oecd.org/termsandconditions/","text":""},{"title":"Privacy Policy","uri":"http://www.oecd.org/privacy/","text":""},{"title":"MyOECD","uri":"https://www.oecd.org/login","text":""},{"title":"Contact Us","uri":"http://www.oecd.org/contact/","text":""}]}}

pandas_datareader/tests/io/test_jsdmx.py

@@ -86,3 +86,18 @@ def test_land_use(dirpath):
     values = np.array(values)
     expected = pd.DataFrame(values, index=exp_idx, columns=exp_col)
     tm.assert_frame_equal(result[exp_col], expected)
+
+
+@pytest.mark.skipif(not PANDAS_0210, reason='Broken on old pandas')
+def test_quartervalue(dirpath):
+    # https://stats.oecd.org/sdmx-json/data/QNA/AUS+AUT+BEL+CAN+CHL.GDP+B1_GE.CUR+VOBARSA.Q/all?startTime=2009-Q1&endTime=2011-Q4
+    result = read_jsdmx(os.path.join(dirpath, 'jsdmx',
+                                     'oecd1.json'))
+    assert isinstance(result, pd.DataFrame)
+    expected = pd.DatetimeIndex(['2009-01-01', '2009-04-01', '2009-07-01',
+                                 '2009-10-01', '2010-01-01', '2010-04-01',
+                                 '2010-07-01', '2010-10-01', '2011-01-01',
+                                 '2011-04-01', '2011-07-01', '2011-10-01'],
+                                dtype='datetime64[ns]', name=u'Period',
+                                freq=None)
+    tm.assert_index_equal(result.index, expected)