Support CKAN from 2.9 to 2.11 and remove support for old CKAN versions

.github/workflows/test.yml

@@ -4,10 +4,10 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
-          python-version: '3.6'
+          python-version: '3.9'
       - name: Install requirements
         run: pip install flake8 pycodestyle
       - name: Check syntax
@@ -17,16 +17,26 @@ jobs:
     needs: lint
     strategy:
       matrix:
-        ckan-version: [2.8, 2.9, 2.9-py2]
+        include:
+          - ckan-version: "2.11"
+            ckan-image: "ckan/ckan-dev:2.11-py3.10"
+            solr-version: "9"
+          - ckan-version: "2.10"
+            ckan-image: "ckan/ckan-dev:2.10-py3.10"
+            solr-version: "9"
+          - ckan-version: "2.9"
+            ckan-image: "ckan/ckan-dev:2.9-py3.9"
+            solr-version: "8"
       fail-fast: false
 
     name: CKAN ${{ matrix.ckan-version }}
     runs-on: ubuntu-latest
     container:
-      image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
+      image: ${{ matrix.ckan-image }}
+      options: --user root
     services:
       solr:
-        image: ckan/ckan-solr:${{ matrix.ckan-version }}
+        image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr${{ matrix.solr-version }}
       postgres:
         image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
         env:
@@ -44,34 +54,16 @@ jobs:
       CKAN_REDIS_URL: redis://redis:6379/1
 
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v3
-    - name: Install requirements (Python 3)
-      if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' && matrix.ckan-version != '2.9-py2'}}
+    - uses: actions/checkout@v4
+    - name: Install requirements (common)
       run: |
-        pip install cython
-        pip install cchardet
         pip install -r requirements.txt
         pip install -r dev-requirements.txt
-    - name: Install requirements (Python 2)
-      if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' || matrix.ckan-version == '2.9-py2'}}
-      run: |
-        pip install cython
-        pip install cchardet
-        pip install -r requirements-py2.txt
-        pip install -r dev-requirements-py2.txt
-    - name: Install requirements (common)
-      run: |
         pip install -e .
         # Replace default path to CKAN core config file with the one on the container
         sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
-    - name: Setup extension (CKAN >= 2.9)
-      if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' }}
+    - name: Setup extension
       run: |
         ckan -c test.ini db init
-    - name: Setup extension (CKAN < 2.9)
-      if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' }}
-      run: |
-        paster --plugin=ckan db init -c test.ini
     - name: Run tests
-      run: pytest --ckan-ini=test.ini --cov=ckanext.datapackager --cov-report=xml --cov-append --disable-warnings ckanext/datapackager/tests
+      run: pytest --ckan-ini=test.ini --cov=ckanext.datapackager --cov-report=term-missing --cov-append --disable-warnings ckanext/datapackager/tests

README.md

@@ -7,7 +7,7 @@ This extension adds importing and exporting of [Data Packages][data-packages] to
 
 ## Requirements
 
-* CKAN >= 2.8
+* CKAN >= 2.9
 
 ## Installing
 
@@ -128,14 +128,10 @@ For instance
 
 You'll need to install the dev requirements to run the tests:
 
-To run the tests on CKAN >= 2.9, do:
+To run the tests, do:
 
     pytest --ckan-ini=test.ini ckanext/dcat/tests
 
-To run the tests on CKAN <= 2.8, do:
-
-    nosetests --nologcapture --ckan --with-pylons=test-nose.ini ckanext/dcat/tests/nose
-
 Note that ckanext-datapackager's `test.ini` file assumes that the relative path from it
 to CKAN's `test-core.ini` file is `../ckan/test-core.ini`, i.e. that you have
 CKAN and ckanext-datapackager installed next to each other in the same directory. This

ckanext/datapackager/controllers/datapackage.py

@@ -22,7 +22,7 @@ def new(data=None, errors=None, error_summary=None):
     errors = errors or {}
     error_summary = error_summary or {}
     default_data = {
-        'owner_org': toolkit.request.params.get('group'),
+        'owner_org': toolkit.request.args.get('group'),
     }
     data = data or default_data
 
@@ -50,19 +50,16 @@ def import_datapackage():
         else:
             params = toolkit.request.params
 
-        if toolkit.check_ckan_version(min_version="2.9"):
-            if 'upload' in toolkit.request.files:
+        if 'upload' in toolkit.request.files:
+            if toolkit.request.files['upload'].filename:
                 params['upload'] = toolkit.request.files['upload']
 
         dataset = toolkit.get_action('package_create_from_datapackage')(
             context,
             params,
         )
 
-        if toolkit.check_ckan_version(min_version="2.9"):
-            return toolkit.redirect_to('dataset.read', id=dataset['name'])
-        else:
-            return toolkit.redirect_to('dataset_read', id=dataset['name'])
+        return toolkit.redirect_to('dataset.read', id=dataset['name'])
 
     except toolkit.ValidationError as e:
         errors = e.error_dict
@@ -81,7 +78,7 @@ def export_datapackage(package_id):
         'user': toolkit.c.user,
     }
 
-    r = make_response() if toolkit.check_ckan_version(min_version="2.9") else toolkit.response
+    r = make_response()
     r.content_disposition = 'attachment; filename=datapackage.json'.format(
         package_id)
     r.content_type = 'application/json'
@@ -94,20 +91,5 @@ def export_datapackage(package_id):
     except toolkit.ObjectNotFound:
         return toolkit.abort(404, 'Dataset not found')
 
-    if toolkit.check_ckan_version(min_version="2.9"):
-        r.data = json.dumps(datapackage_dict, indent=2)
-        return r
-    else:
-        return json.dumps(datapackage_dict, indent=2)
-
-
-if not toolkit.check_ckan_version(u'2.9'):
-    class DataPackageController(toolkit.BaseController):
-        def new(self, data=None, errors=None, error_summary=None):
-            return new(data, errors, error_summary)
-        def import_datapackage(self):
-            return import_datapackage()
-        def export_datapackage(self, package_id):
-            return export_datapackage(package_id)
-
-
+    r.data = json.dumps(datapackage_dict, indent=2)
+    return r

ckanext/datapackager/logic/action/create.py

@@ -1,10 +1,7 @@
 import random
-import cgi
 import json
 import tempfile
 
-import six
-
 import ckan.plugins.toolkit as toolkit
 from frictionless_ckan_mapper import frictionless_to_ckan as converter
 from werkzeug.datastructures import FileStorage
@@ -76,7 +73,7 @@ def package_create_from_datapackage(context, data_dict):
                 toolkit.get_action('package_delete')(
                     context, {'id': dataset_id})
             except Exception as e2:
-                six.raise_from(e, e2)
+                raise e from e2
             else:
                 raise e
 
@@ -88,10 +85,7 @@ def _load_and_validate_datapackage(url=None, upload=None):
     try:
 
         if _upload_attribute_is_valid(upload):
-            if toolkit.check_ckan_version(min_version="2.9"):
-                dp = datapackage.DataPackage(upload)
-            else:
-                dp = datapackage.DataPackage(upload.file)
+            dp = datapackage.DataPackage(upload)
         else:
 
             dp = datapackage.DataPackage(url)
@@ -151,14 +145,11 @@ def _create_and_upload_resource_with_inline_data(context, resource):
     data = resource['data']
 
     del resource['data']
-    if not isinstance(data, six.string_types):
+    if not isinstance(data, str):
         data = json.dumps(data, indent=2)
 
     with tempfile.NamedTemporaryFile(prefix=prefix) as f:
-        if six.PY3:
-            f.write(six.binary_type(data, 'utf-8'))
-        else:
-            f.write(six.binary_type(data))
+        f.write(bytes(data, 'utf-8'))
         f.seek(0)
 
         _create_and_upload_resource(context, resource, f)
@@ -183,21 +174,10 @@ def _create_and_upload_local_resource(context, resource):
 def _create_and_upload_resource(context, resource, the_file):
     resource['url'] = 'url'
     resource['url_type'] = 'upload'
-
-    if toolkit.check_ckan_version(min_version="2.9"):
-        resource['upload'] = FileStorage(the_file, the_file.name, the_file.name)
-    else:
-        resource['upload'] = _UploadLocalFileStorage(the_file)
+    resource['upload'] = FileStorage(the_file, the_file.name, the_file.name)
 
     toolkit.get_action('resource_create')(context, resource)
 
 
 def _upload_attribute_is_valid(upload):
     return hasattr(upload, 'read') or hasattr(upload, 'file') and hasattr(upload.file, 'read')
-
-# Used only in CKAN < 2.9
-class _UploadLocalFileStorage(cgi.FieldStorage):
-    def __init__(self, fp, *args, **kwargs):
-        self.name = fp.name
-        self.filename = fp.name
-        self.file = fp

ckanext/datapackager/plugin/__init__.py

@@ -1,21 +1,18 @@
+from flask import Blueprint
+
 import ckan.plugins as plugins
 import ckan.plugins.toolkit as toolkit
+from ckanext.datapackager.controllers import datapackage
 from ckanext.datapackager.logic.action.create import package_create_from_datapackage
 from ckanext.datapackager.logic.action.get import package_show_as_datapackage
 
-if toolkit.check_ckan_version(u'2.9'):
-    from ckanext.datapackager.plugin.flask_plugin import MixinPlugin
-    ckan_29_or_higher = True
-else:
-    from ckanext.datapackager.plugin.pylons_plugin import MixinPlugin
-    ckan_29_or_higher = False
-
 
-class DataPackagerPlugin(MixinPlugin, plugins.SingletonPlugin):
+class DataPackagerPlugin(plugins.SingletonPlugin):
     '''Plugin that adds importing/exporting datasets as Data Packages.
     '''
     plugins.implements(plugins.IActions)
     plugins.implements(plugins.IConfigurer)
+    plugins.implements(plugins.IBlueprint)
 
     def update_config(self, config):
         toolkit.add_template_directory(config, '../templates')
@@ -25,3 +22,26 @@ def get_actions(self):
             'package_create_from_datapackage': package_create_from_datapackage,
             'package_show_as_datapackage': package_show_as_datapackage,
         }
+
+    def get_blueprint(self):
+        blueprint = Blueprint("datapackager", __name__)
+        # As long as the URL for import_datapackage_view and import_datapackage are the same, reverse lookups from import_datapackage will work
+        blueprint.add_url_rule(
+            "/import_datapackage",
+            view_func=datapackage.new,
+            endpoint="import_datapackage",
+            methods=["GET"],
+        )
+        blueprint.add_url_rule(
+            "/import_datapackage",
+            view_func=datapackage.import_datapackage,
+            endpoint="import_datapackage_post",
+            methods=["POST"],
+        )
+        blueprint.add_url_rule(
+            "/dataset/<package_id>/datapackage.json",
+            view_func=datapackage.export_datapackage,
+            endpoint="export_datapackage",
+            methods=["GET"],
+        )
+        return blueprint

ckanext/datapackager/templates/datapackage/import_datapackage.html

@@ -3,18 +3,13 @@
 {% block subtitle %}{{ _('Import Data Package') }}{% endblock %}
 
 {% block breadcrumb_content %}
-      {% if h.ckan_version().split('.')|map('int')|list >= [2, 9, 0] %}
-        <li>{% link_for _('Datasets'), named_route='dataset.search' %}</li>
-      {% else %}
-        <li>{% link_for _('Datasets'), controller='package', action='search' %}</li>
-      {% endif %}
-
+  <li>{% link_for _('Datasets'), named_route='dataset.search' %}</li>
   <li class="active"><a href="">{{ _('Import Data Package') }}</a></li></a></li>
 {% endblock %}
 
 {% block info_module %}
   <section class="module module-narrow module-shallow">
-    <h2 class="module-heading"><i class="icon-info-sign"></i> {{ _('What are Data Packages?') }}</h2>
+    <h2 class="module-heading"><i class="fa fa-info-circle"></i> {{ _('What are Data Packages?') }}</h2>
     <div class="module-content">
       <p>
         {% trans %}

ckanext/datapackager/templates/datapackage/snippets/import_datapackage_form.html

@@ -1,7 +1,8 @@
 {% import 'macros/form.html' as form %}
-{% set action = h.url_for('datapackager.import_datapackage') if h.ckan_version().split('.')|map('int')|list >= [2, 9, 0] else h.url_for('import_datapackage') %}
+{% set action = h.url_for('datapackager.import_datapackage') %}
 
 <form class="dataset-form form-horizontal" method="post" action="{{ action }}" data-module="basic-form" enctype="multipart/form-data">
+    {{ h.csrf_input() if 'csrf_input' in h }}
     {% block errors %}{{ form.errors(error_summary) }}{% endblock %}
 
     {% block basic_fields %}

ckanext/datapackager/templates/organization/read.html

@@ -3,7 +3,7 @@
 {% block page_primary_action %}
   {{ super() }}
   {% if h.check_access('package_create', {'owner_org': c.group_dict.id}) %}
-  <a class="btn btn-primary" href="{{ h.url_for('datapackager.import_datapackage', group=c.group_dict.id) if h.ckan_version().split('.')|map('int')|list >= [2, 9, 0] else h.url_for('import_datapackage', group=c.group_dict.id) }}">
+  <a class="btn btn-primary" href="{{ h.url_for('datapackager.import_datapackage', group=c.group_dict.id) }}">
     <i class="fa fa-upload"></i>
     {{ _('Import Data Package') }}
   </a>

ckanext/datapackager/templates/package/read.html

@@ -2,14 +2,8 @@
 
 {% block package_resources %}
   <div style="float:right; margin-right: -7px;">
-
-    {% if h.ckan_version().split('.')|map('int')|list >= [2, 9, 0] %}
-        <a href="{% url_for 'datapackager.export_datapackage', package_id=pkg.id %}" class='btn' id='export_datapackage_button'>
-    {% else %}
-        <a href="{% url_for 'export_datapackage', package_id=pkg.id %}" class='btn' id='export_datapackage_button'>
-    {% endif %}
-
-       <i class='icon-download'></i>Download Data Package
+    <a href="{% url_for 'datapackager.export_datapackage', package_id=pkg.id %}" class='btn btn-default btn-light' id='export_datapackage_button'>
+       <i class='fa fa-arrow-circle-down fa-arrow-circle-o-down'></i>Download Data Package
     </a>
   </div>
   <div class="clearfix"></div>