[Fixes #13471] Synchronization of the services UI with the harvesting processes (#13473)

[Fixes #13471] Synchronization of the services UI with the harvesting processes (#13473)

Author: Gpetrak

geonode/services/templates/services/service_detail.html

@@ -14,43 +14,22 @@ <h3><strong>{{service.title|default:service.name}}</strong></h3>
         <p><strong>SERVICE NOTES:</strong> The service is accessed by Basic auth via the user <strong>{{service.username}}</strong></p>
       {% endif %}
     {% autoescape off %}
-        <h3>{% trans "Service Resources" %} <span class="badge">{{ total_resources }}</span></h3>
-        {% if service.harvester %}
-        {% with service.harvester.latest_harvesting_session.get_progress_percentage as harvesting_progress %}
-            {% if harvesting_progress >= 0 and harvesting_progress < 100.0 %}
-            <p>
-            <i class="fa fa-gear"></i> <i>{% trans "Resource harvesting in progress..." %}</i>
-            <div class="progress">
-                <div class="progress-bar" role="progressbar" style="width: {{ harvesting_progress }}%;" aria-valuenow="{{ harvesting_progress }}" aria-valuemin="0" aria-valuemax="100">{{ harvesting_progress }}%</div>
+        <div id="harvestProgressWrapper" style="display:none;">
+            <i class="fa fa-gear"></i> <i id="harvestProgressText"></i>
+            <div class="progress" id="harvestProgressContainer">
+                <div class="progress-bar" id="harvestProgressBar" role="progressbar"
+                    style="width:0%;" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100">
+                    0%
+                </div>
             </div>
-            </p>
+        </div>
+        <div id="resourcesContainer">
+            {% if resources %}
+                {% include "services/service_resources_partial.html" %}
+            {% else %}
+                <p>{% trans "No resources have been imported yet." %}</p>
             {% endif %}
-        {% endwith %}
-        {% endif %}
-        {% if total_resources == 0 %}
-            <p>{% trans "No resources have been imported yet." %}</p>
-        {% else %}
-            <div class="row">
-            <table class="table">
-                <thead>
-                <th>{% trans "Title" %}</th>
-                <th>{% trans "Description" %}</th>
-                </thead>
-                {% for dataset in datasets %}
-                    <tr>
-                        <td><a href='{{ dataset.get_absolute_url }}'>{{dataset.title|striptags}}</a></td>
-                        <td>
-                            <div class="row">
-                                <div class="col-md-9">
-                                    {{dataset.abstract|striptags}}
-                                </div>
-                            </div>
-                        </td>
-                    </tr>
-                {% endfor %}
-            </table>
-            </div>
-        {% endif %}
+        </div>
     {% if resources.paginator.num_pages > 1 %}
         <div class="row">
             <nav aria-label="importable resources pages">
@@ -109,27 +88,64 @@ <h3>{% trans "Service Resources" %} <span class="badge">{{ total_resources }}</s
     {% endif %}
 {% endblock %}
 {% block extra_script %}
-    {{ block.super }}
-    <script type="text/javascript">
-        $(document).ready(function () {
-            $('#harvestResources').on("click", function() {
-                $("#progressModal").modal("show");
-            });
-            $('button[name^=retry]').on('click', function () {
-                const resourceId = this.name.replace("retry-", "");
-                const retryUrl = "/services/{{ service.id }}/harvest/" + resourceId;
-                const retryForm = document.createElement('form');
-                retryForm.setAttribute("method", "post");
-                retryForm.setAttribute("action", retryUrl);
-                const csrfInputElem = document.createElement("input");
-                csrfInputElem.type = "hidden";
-                csrfInputElem.name = "csrfmiddlewaretoken";
-                csrfInputElem.value = "{{ csrf_token }}";
-                retryForm.appendChild(csrfInputElem);
-                document.body.appendChild(retryForm);
-                retryForm.submit();
+{{ block.super }}
+<script type="text/javascript">
+    $(document).ready(function () {
+        $('#harvestResources').on("click", function() {
+            $("#progressModal").modal("show");
+        });
 
-            });
+        $('button[name^=retry]').on('click', function () {
+            const resourceId = this.name.replace("retry-", "");
+            const retryUrl = "/services/{{ service.id }}/harvest/" + resourceId;
+            const retryForm = document.createElement('form');
+            retryForm.setAttribute("method", "post");
+            retryForm.setAttribute("action", retryUrl);
+            const csrfInputElem = document.createElement("input");
+            csrfInputElem.type = "hidden";
+            csrfInputElem.name = "csrfmiddlewaretoken";
+            csrfInputElem.value = "{{ csrf_token }}";
+            retryForm.appendChild(csrfInputElem);
+            document.body.appendChild(retryForm);
+            retryForm.submit();
         });
-    </script>
+
+        
+    let lastStatus = null;
+    let resourcesUpdated = false;
+
+    function updateHarvestProgress() {
+        $.get("{% url 'service_harvest_progress' service.id %}", function(data) {
+            let progress = data.progress || 0;
+
+            if (data.in_progress) {
+                $("#harvestProgressWrapper").show();
+                $("#harvestProgressText").text("Resource harvesting in progress...");
+                $("#harvestProgressBar")
+                    .css("width", progress + "%")
+                    .attr("aria-valuenow", progress)
+                    .text(Math.round(progress) + "%");
+                resourcesUpdated = false; // reset flag while harvesting
+            } else {
+                $("#harvestProgressWrapper").hide();
+                $("#harvestProgressText").text("");
+                $("#harvestProgressBar")
+                    .css("width", "0%")
+                    .attr("aria-valuenow", 0)
+                    .text("0%");
+
+                // Trigger partial API exactly once when in-progress → finished
+                if (lastStatus === "on-going" && !resourcesUpdated) {
+                    $("#resourcesContainer").load("{% url 'service_resources_partial' service.id %}");
+                    resourcesUpdated = true;
+                }
+            }
+
+        lastStatus = data.status;
+    });
+}
+    updateHarvestProgress();
+    setInterval(updateHarvestProgress, 7000)
+    });
+</script>
 {% endblock extra_script %}

geonode/services/templates/services/service_resources_partial.html

@@ -0,0 +1,31 @@
+{% load i18n %}
+
+<h3>
+    {% trans "Service Resources" %}
+    <span class="badge">{{ total_resources }}</span>
+</h3>
+
+{% if total_resources == 0 %}
+    <p>{% trans "No resources have been imported yet." %}</p>
+{% else %}
+    <div class="row">
+        <table class="table">
+            <thead>
+                <th>{% trans "Title" %}</th>
+                <th>{% trans "Description" %}</th>
+            </thead>
+            {% for resource in resources %}
+                <tr>
+                    <td><a href="{{ resource.get_absolute_url }}">{{ resource.title|striptags }}</a></td>
+                    <td>
+                        <div class="row">
+                            <div class="col-md-9">
+                                {{ resource.abstract|striptags }}
+                            </div>
+                        </div>
+                    </td>
+                </tr>
+            {% endfor %}
+        </table>
+    </div>
+{% endif %}

geonode/services/templatetags/services_tags.py

@@ -33,7 +33,7 @@ def get_dataset_count_by_services(service_id, user):
         if service.harvester:
             _h = service.harvester
             harvested_resources_ids = list(
-                _h.harvestable_resources.filter(should_be_harvested=True, geonode_resource__isnull=False).values_list(
+                _h.harvestable_resources.filter(geonode_resource__isnull=False).values_list(
                     "geonode_resource__id", flat=True
                 )
             )

geonode/services/urls.py

@@ -35,4 +35,14 @@
         views.harvest_single_resource,
         name="harvest_single_resource",
     ),
+    re_path(
+        r"^(?P<service_id>\d+)/progress/$",
+        views.service_harvest_progress,
+        name="service_harvest_progress",
+    ),
+    re_path(
+        r"^(?P<service_id>\d+)/resources_partial/$",
+        views.service_resources_partial,
+        name="service_resources_partial",
+    ),
 ]

geonode/services/views.py

@@ -22,7 +22,7 @@
 from django.contrib import messages
 from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator
 from django.urls import reverse
-from django.http import HttpResponse, HttpResponseRedirect
+from django.http import HttpResponse, HttpResponseRedirect, JsonResponse
 from django.http import Http404
 from django.shortcuts import get_object_or_404
 from django.shortcuts import redirect
@@ -241,46 +241,12 @@ def rescan_service(request, service_id):
 def service_detail(request, service_id):
     """This view shows the details of a service"""
 
-    services = Service.objects.filter(resourcebase_ptr_id=service_id)
-
-    if not services.exists():
-        messages.add_message(request, messages.ERROR, _("You dont have enougth rigths to see the resource detail"))
-        return redirect(reverse("services"))
-    service = services.first()
+    service, resources, total_resources = _get_service_and_resources(service_id, request.user, request.GET.get("page"))
 
     permissions_json = _perms_info_json(service)
 
     perms_list = permissions_registry.get_perms(instance=service, user=request.user)
 
-    harvested_resources_ids = []
-    if service.harvester:
-        _h = service.harvester
-        harvested_resources_ids = list(
-            _h.harvestable_resources.filter(should_be_harvested=True, geonode_resource__isnull=False).values_list(
-                "geonode_resource__id", flat=True
-            )
-        )
-    already_imported_datasets = get_visible_resources(
-        queryset=ResourceBase.objects.filter(id__in=harvested_resources_ids), user=request.user
-    )
-    resources_being_harvested = []
-
-    all_resources = list(resources_being_harvested) + list(already_imported_datasets)
-
-    paginator = Paginator(all_resources, getattr(settings, "CLIENT_RESULTS_LIMIT", 25), orphans=3)
-    page = request.GET.get("page")
-    try:
-        resources = paginator.page(page)
-    except PageNotAnInteger:
-        resources = paginator.page(1)
-    except EmptyPage:
-        resources = paginator.page(paginator.num_pages)
-
-    # pop the handler out of the session in order to free resources
-    # - we had stored the service handler on the session in order to
-    # speed up the register/harvest resources flow. However, for services
-    # with many resources, keeping the handler in the session leads to degraded
-    # performance
     try:
         request.session.pop(service.service_url)
     except KeyError:
@@ -291,14 +257,12 @@ def service_detail(request, service_id):
         template_name="services/service_detail.html",
         context={
             "service": service,
-            "datasets": already_imported_datasets,
-            # "resource_jobs": (r for r in resources if isinstance(r, HarvestJob)),
+            "resources": resources,
             "resource_jobs": (),
             "permissions_json": permissions_json,
             "permissions_list": perms_list,
             "can_add_resorces": request.user.has_perm("base.add_resourcebase"),
-            "resources": resources,
-            "total_resources": len(already_imported_datasets),
+            "total_resources": total_resources,
         },
     )
 
@@ -352,3 +316,71 @@ def remove_service(request, service_id):
         service.harvester.delete()
         messages.add_message(request, messages.INFO, _(f"Service {service.title} has been deleted"))
         return HttpResponseRedirect(reverse("services"))
+
+
+@login_required
+def service_harvest_progress(request, service_id):
+    try:
+        service = Service.objects.get(id=service_id)
+        session = None
+        if service.harvester:
+            # Get latest active session if any
+            active_session = service.harvester.latest_harvesting_session
+            if active_session and active_session.status in ["pending", "on-going"]:
+                session = active_session
+        if session:
+            progress = session.get_progress_percentage()
+            status = session.status
+            in_progress = status in ["pending", "on-going"]
+        else:
+            progress = None
+            status = "no-session"
+            in_progress = False
+        return JsonResponse({"progress": progress, "status": status, "in_progress": in_progress})
+    except Service.DoesNotExist:
+        return JsonResponse({"progress": 0, "status": "not_found", "in_progress": False})
+
+
+@login_required
+def service_resources_partial(request, service_id):
+    service, resources, total_resources = _get_service_and_resources(service_id, request.user, request.GET.get("page"))
+
+    return render(
+        request,
+        template_name="services/service_resources_partial.html",
+        context={
+            "service": service,
+            "resources": resources,
+            "total_resources": total_resources,
+        },
+    )
+
+
+def _get_service_and_resources(service_id, user, page):
+    """
+    Common helper to fetch a service and its paginated resources.
+    """
+    service = get_object_or_404(Service, id=service_id)
+
+    harvested_resources_ids = []
+    if service.harvester:
+        harvested_resources_ids = list(
+            service.harvester.harvestable_resources.filter(geonode_resource__isnull=False).values_list(
+                "geonode_resource__id", flat=True
+            )
+        )
+
+    datasets = get_visible_resources(queryset=ResourceBase.objects.filter(id__in=harvested_resources_ids), user=user)
+
+    resources_being_harvested = []
+    all_resources = list(resources_being_harvested) + list(datasets)
+
+    paginator = Paginator(all_resources, getattr(settings, "CLIENT_RESULTS_LIMIT", 25), orphans=3)
+    try:
+        resources = paginator.page(page)
+    except PageNotAnInteger:
+        resources = paginator.page(1)
+    except EmptyPage:
+        resources = paginator.page(paginator.num_pages)
+
+    return service, resources, len(datasets)