Merge pull request #7 from mhabedan/yodaH5

Add support for yoda-H5

Author: GraemeWatt

.github/workflows/ci.yml

@@ -15,26 +15,28 @@ jobs:
     strategy:
       matrix:
         runner: ['ubuntu-latest']
-        python-version: [ '3.7', '3.8', '3.9', '3.10', '3.11' ]
+        python-version: [ '3.8', '3.9', '3.10', '3.11', '3.12', '3.13' ]
         include:
-          - runner: 'ubuntu-20.04'
-            python-version: '3.6'
+          - runner: 'ubuntu-22.04'
+            python-version: '3.7'
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install pip dependencies
       run: |
         pip install --upgrade pip setuptools coveralls
         pip install -e '.[tests]'
+        hepdata-cli --help
     - name: Run tests
+      if: startsWith(matrix.python-version, '3.12')
       run: |
         pytest --cov=hepdata_cli
     - name: Run coveralls
-      if: startsWith(matrix.python-version, '3.11')
+      if: startsWith(matrix.python-version, '3.12')
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         COVERALLS_SERVICE_NAME: github
@@ -51,11 +53,11 @@ jobs:
     permissions:
       id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
     steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python 3.11
-      uses: actions/setup-python@v4
+    - uses: actions/checkout@v5
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.11'
+        python-version: '3.12'
     - name: Build PyPI package
       run: |
         pip install wheel

README.md

@@ -23,27 +23,18 @@ $ pip install --user hepdata-cli
 $ hepdata-cli --help
 ```
 
-With Python 3 (<3.7), if the `LANG` environment variable is not set, you might get an error like:
-
-```
-RuntimeError: Click will abort further execution because Python 3 was configured to use ASCII as encoding for the environment. Consult https://click.palletsprojects.com/python3/ for mitigation steps.
-```
-
-In this case, you will need to export a Unicode locale as described in the
-[Click documentation](https://click.palletsprojects.com/en/7.x/python3/#python-3-surrogate-handling).
-
 ## Installation (for developers)
 
 Install from GitHub in a [virtual environment](https://docs.python.org/3/tutorial/venv.html):
 
 ```code
 $ git clone https://github.com/HEPData/hepdata-cli.git
 $ cd hepdata-cli
-$ python3 -m venv ~/venv/hepdata-cli
-$ source ~/venv/hepdata-cli/bin/activate
-(hepdata-cli) $ pip install -e '.[tests]'
-(hepdata-cli) $ hepdata-cli --help
-(hepdata-cli) $ pytest --cov=hepdata_cli
+$ python3 -m venv venv
+$ source venv/bin/activate
+(venv) $ pip install -e '.[tests]'
+(venv) $ hepdata-cli --help
+(venv) $ pytest --cov=hepdata_cli
 ```
 
 ## Usage
@@ -74,11 +65,11 @@ An exact match of the keyword is first attempted, otherwise partial matches are
 
 The argument ```[-i/--ids IDTYPE]``` accepts ```IDTYPE``` equal to ```arxiv```, ```hepdata``` or```inspire```.
 
-The argument  ```[-f/--file-format FORMAT]``` accepts ```FORMAT``` equal to ```csv```, ```root```, ```yaml```, ```yoda```, ```yoda1```, or ```json```.
-In the first four cases a .tar.gz archive is downloaded and unpacked as a directory, whereas in the last case a .json file is downloaded.
+The argument  ```[-f/--file-format FORMAT]``` accepts ```FORMAT``` equal to ```csv```, ```root```, ```yaml```, ```yoda```, ```yoda1```, ```yoda.h5```, or ```json```.
+In the first six cases a .tar.gz archive is downloaded and unpacked as a directory, whereas in the last case a .json file is downloaded.
 
 The argument  ```[-t/--table-name TABLE-NAME]``` accepts a string giving the table name as input.
-In this case only the specified table is downloaded as a .csv, .root, .yaml, .yoda, .yoda1 or .json file.
+In this case only the specified table is downloaded as a .csv, .root, .yaml, .yoda, .yoda1, .yoda.h5, or .json file.
 
 The argument ```[-d/--download-dir DOWNLOAD-DIR]``` specifies the directory to download the files.
 If not specified, the default download directory is ```./hepdata-downloads```.

hepdata_cli/api.py

@@ -13,6 +13,7 @@
 # SITE_URL = "http://127.0.0.1:5000"
 
 UPLOAD_MAX_SIZE = 52000000  # Upload limit in bytes
+ALLOWED_FORMATS = ['csv', 'root', 'yaml', 'yoda', 'yoda1', 'yoda.h5', 'json']
 
 MAX_MATCHES, MATCHES_PER_PAGE = (10000, 10) if "pytest" not in sys.modules else (144, 12)
 
@@ -82,7 +83,7 @@ def download(self, id_list, file_format=None, ids=None, table_name='', download_
         Downloads from the hepdata database the specified records.
 
         :param id_list: list of ids to download. These can be obtained by the find function.
-        :param file_format: accepts one of ('csv', 'root', 'yaml', 'yoda', 'yoda1', 'json'). Specifies the download file format.
+        :param file_format: accepts one of ('csv', 'root', 'yaml', 'yoda', 'yoda1', 'yoda.h5', 'json'). Specifies the download file format.
         :param ids: accepts one of ('inspire', 'hepdata'). It specifies what type of ids have been passed.
         :param table_name: restricts download to specific tables.
         :param download_dir: defaults to ./hepdata-downloads. Specifies where to download the files.
@@ -139,13 +140,14 @@ def _build_urls(self, id_list, file_format, ids, table_name):
         if type(id_list) not in (tuple, list):
             id_list = id_list.split()
         assert len(id_list) > 0, 'Ids are required.'
-        assert file_format in ['csv', 'root', 'yaml', 'yoda', 'yoda1', 'json'], "allowed formats are: csv, root, yaml, yoda, yoda1 and json."
+        assert file_format in ALLOWED_FORMATS, f"allowed formats are: {ALLOWED_FORMATS}"
         assert ids in ['inspire', 'hepdata'], "allowed ids are: inspire and hepdata."
         if table_name == '':
             params = {'format': file_format}
         else:
             params = {'format': file_format, 'table': table_name}
-        urls = [resilient_requests('get', SITE_URL + '/record/' + ('ins' if ids == 'inspire' else '') + id_entry, params=params).url for id_entry in id_list]
+        urls = [resilient_requests('get', SITE_URL + '/record/' + ('ins' if ids == 'inspire' else '') + id_entry, params=params).url.replace('%2525', '%25') for id_entry in id_list]
+        # TODO: Investigate root cause of double URL encoding (https://github.com/HEPData/hepdata-cli/issues/8).
         return urls
 
     def _query(self, query, page, size):

hepdata_cli/cli.py

@@ -3,7 +3,7 @@
 import click
 
 from .version import __version__
-from .api import Client
+from .api import Client, ALLOWED_FORMATS
 
 
 @click.group()
@@ -27,7 +27,7 @@ def find(client, query, keyword, ids):
 
 @cli.command()
 @click.argument('id_list', nargs=-1, required=True, type=str)
-@click.option('-f', '--file-format', required=True, type=str, help='Specify file format (csv, root, yaml, yoda, yoda1, or json) to be downloaded.')
+@click.option('-f', '--file-format', required=True, type=str, help=f'Specify file format (from {ALLOWED_FORMATS}) to be downloaded.')
 @click.option('-i', '--ids', required=True, type=str, help='Specify which ids (hepdata or inspire) are given.')
 @click.option('-t', '--table-name', default='', type=str, help='Specify table to be downloaded.')
 @click.option('-d', '--download-dir', default='./hepdata-downloads', type=str, help='Specify where to download the files.')

hepdata_cli/version.py

@@ -1 +1 @@
-__version__ = "0.2.2"
+__version__ = "0.2.3"

setup.py

@@ -45,7 +45,7 @@
     install_requires=install_requirements,
     tests_require=test_requirements,
     extras_require=extras_require,
-    python_requires='>=3.6',
+    python_requires='>=3.7',
     entry_points={
         'console_scripts': [
             'hepdata-cli = hepdata_cli.cli:cli',

tests/test_download.py

@@ -33,17 +33,25 @@ def cleanup(directory):
 
 # arguments for testing
 
-test_download_arguments = [
+test_api_download_arguments = [
     (["73322"], "json", "hepdata", ''),
     (["1222326", "1694381", "1462258", "1309874"], "csv", "inspire", ''),
     (["61434"], "yaml", "hepdata", "Table1"),
     (["1762350"], "yoda", "inspire", "Number density and Sum p_T pT>0.15 GeV/c"),
+    (["2862529"], "yoda.h5", "inspire", "95% CL upper limit on XSEC times BF"),
+    (["2862529"], "yoda.h5", "inspire", '')
+]
+
+test_cli_download_arguments = [
+    (["2862529"], "json", "inspire", ''),
+    (["1222326", "1694381", "1462258", "1309874"], "root", "inspire", ''),
+    (["61434"], "yaml", "hepdata", "Table2"),
 ]
 
 
 # api testing
 
-@pytest.mark.parametrize("id_list, file_format, ids, table", test_download_arguments)
+@pytest.mark.parametrize("id_list, file_format, ids, table", test_api_download_arguments)
 def test_api_download(id_list, file_format, ids, table):
     test_download_dir = './.pytest_downloads/'
     mkdir(test_download_dir)
@@ -56,7 +64,7 @@ def test_api_download(id_list, file_format, ids, table):
 
 # cli testing
 
-@pytest.mark.parametrize("id_list, file_format, ids, table", test_download_arguments)
+@pytest.mark.parametrize("id_list, file_format, ids, table", test_cli_download_arguments)
 def test_cli_download(id_list, file_format, ids, table):
     test_download_dir = './.pytest_downloads/'
     mkdir(test_download_dir)

tests/test_search.py

@@ -10,20 +10,21 @@
 
 # arguments for testing
 
-test_find_arguments = [
+test_api_find_arguments = [
     ('reactions:"P P--> LQ LQ X"', None, None),
-    ('reactions:"P P --> gamma gamma"', None, None),
     ('reactions:"P P--> LQ LQ"', 'year', None),
-    ('reactions:"P P--> LQ LQ"', 'arxiv', None),
-    ('reactions:"P P--> LQ LQ"', None, 'arxiv'),
-    ('reactions:"P P--> LQ LQ"', None, 'hepdata'),
-    ('reactions:"P P"', None, 'hepdata'),
+    ('phrases:"(diffractive AND elastic)"', None, 'arxiv'),
 ]
 
+test_cli_find_arguments = [
+    ('reactions:"P P --> gamma gamma"', 'arxiv', None),
+    ('abstract:"baryon production"', 'arxiv', 'hepdata'),
+    ('abstract:"charmed baryon production"', 'arxiv', 'arxiv'),
+]
 
 # api test
 
-@pytest.mark.parametrize("query, keyword, ids", test_find_arguments)
+@pytest.mark.parametrize("query, keyword, ids", test_api_find_arguments)
 def test_api_find(query, keyword, ids):
     client = Client(verbose=True)
     search_result = client.find(query, keyword, ids)
@@ -37,7 +38,7 @@ def test_api_find(query, keyword, ids):
 
 # cli testing
 
-@pytest.mark.parametrize("query, keyword, ids", test_find_arguments)
+@pytest.mark.parametrize("query, keyword, ids", test_cli_find_arguments)
 def test_cli_find(query, keyword, ids):
     runner = CliRunner()
     result = runner.invoke(cli, ['find', query, '-kw', keyword, '-i', ids])