Add the ability to translate submissions (redux) (#4219)

This builds on the #4134 PR that initially introduced machine
translations into Hypha. This isolates the translation behavior; putting
pip dependencies in a separate `requirements-translate.txt` and will not
attempt any translate imports unless the setting for it is true.

Other small changes are also a full docs page explaining how to install
language packages & changing the setting once again from
`SUBMISSION_TRANSLATIONS_ENABLED` to `APPLICATION_TRANSLATIONS_ENABLED`
to reflect the system wide shift away from submission terminology.

Author: wes-otf

.github/workflows/hypha-ci.yml

@@ -37,6 +37,7 @@ jobs:
       DJANGO_SETTINGS_MODULE: hypha.settings.test
       SEND_MESSAGES: false
       PYTHONDONTWRITEBYTECODE: 1
+      APPLICATION_TRANSLATIONS_ENABLED: 1 # Run tests for machine translation logic
 
     services:
       postgres:
@@ -75,6 +76,7 @@ jobs:
         run: |
           uv venv
           uv pip install -r requirements-dev.txt
+          uv pip install -r requirements-translate.txt
 
       - name: Check Django migrations
         if: matrix.group == 1

docs/setup/administrators/configuration.md

@@ -245,6 +245,16 @@ Set this to enable Djangos settings for secure cookies.
 
     COOKIE_SECURE = env.bool('COOKIE_SECURE', False)
 
+----
+
+Machine translation settings for applications
+
+See [here](machine-translations.md) for more information on setting up machine translations
+
+    APPLICATION_TRANSLATIONS_ENABLED = env.bool("APPLICATION_TRANSLATIONS_ENABLED", False)
+
+----
+
 ## Slack settings
 
     SLACK_TOKEN = env.str('SLACK_TOKEN', None)

docs/setup/administrators/machine-translations.md

@@ -0,0 +1,29 @@
+# Machine translations
+
+Hypha has the ability to utilize [argostranslate](https://github.com/argosopentech/argos-translate) for machine translations of submitted application content. This is disabled by default and the dependencies are not installed to prevent unneeded bloat due to [PyTorch](https://pytorch.org/)'s large language models.
+
+
+## Installing dependencies
+
+As referenced in the [production deployment guide](../deployment/production/stand-alone.md), it is required to install the dependencies needed for machine translation dependencies via
+
+```bash
+python3 -m pip install -r requirements-translate.txt
+```
+
+This requirements file will specifically attempt to install the CPU version of [PyTorch](https://pytorch.org/) if available on the detected platform to play better with heroku (doesn't support GPU processing) and to minimize package bloat (CPU package is ~300MB less than the normal GPU). Depending on your use case, you may want to adjust this.
+
+
+## Installing languages
+
+Argostranslate handles translations via it's own packages - ie. Arabic -> English translation would be one package, while English -> Arabic would be another.
+
+Installing/uninstalling these packages can be done with the management commands `install_languages`/`uninstall_languages` respectively, utilizing the format of <from language code>_<to language code>. For example, installing the Arabic -> English & French -> English packages would look like:
+
+```bash
+python3 manage.py install_languages ar_en fr_en
+```
+
+## Enabling on the system
+
+To enable machine translations on an instance, the proper configuration variables need to be set. These can be found in the [configuration options](configuration.md#hypha-custom-settings)

docs/setup/deployment/development/stand-alone.md

@@ -278,6 +278,12 @@ source venv/bin/activate
 python3 -m pip install -r requirements-dev.txt
 ```
 
+If utilizing application machine translations, install the required dependencies:
+
+```shell
+python3 -m pip install -r requirements-translate.txt
+```
+
 Run:
 ```shell
 make serve-docs

docs/setup/deployment/production/stand-alone.md

@@ -105,6 +105,12 @@ Next, install the required packages using:
 python3 -m pip install -r requirements.txt
 ```
 
+If utilizing application machine translations, install the required dependencies:
+
+```shell
+python3 -m pip install -r requirements-translate.txt
+```
+
 ### Install Node packages
 
 All the needed Node packages are listed in `package.json`. Install them with this command.

hypha/apply/funds/templates/funds/applicationsubmission_admin_detail.html

@@ -1,5 +1,5 @@
 {% extends "funds/applicationsubmission_detail.html" %}
-{% load i18n static workflow_tags review_tags determination_tags heroicons %}
+{% load i18n static workflow_tags review_tags determination_tags translate_tags heroicons %}
 
 {% block extra_css %}
     <link rel="stylesheet" href="{% static 'css/fancybox.css' %}">
@@ -98,4 +98,8 @@ <h5 class="m-0">{% trans "Reminders" %}</h5>
     <script src="{% static 'js/jquery.fancybox.min.js' %}"></script>
     <script src="{% static 'js/fancybox-global.js' %}"></script>
     <script src="{% static 'js/behaviours/collapse.js' %}"></script>
+    <script src="{% static 'js/toggle-related.js' %}"></script>
+    {% if request.user|can_translate_submission %}
+        <script src="{% static 'js/translate-application.js' %}"></script>
+    {% endif %}
 {% endblock %}

hypha/apply/funds/templates/funds/applicationsubmission_detail.html

@@ -1,5 +1,5 @@
 {% extends "base-apply.html" %}
-{% load i18n static workflow_tags wagtailcore_tags statusbar_tags archive_tags submission_tags %}
+{% load i18n static workflow_tags wagtailcore_tags statusbar_tags archive_tags submission_tags translate_tags %}
 {% load heroicons %}
 {% load can from permission_tags %}
 
@@ -148,8 +148,15 @@ <h5>{% blocktrans with stage=object.previous.stage %}Your {{ stage }} applicatio
                                 {% endif %}
                             </div>
                         </header>
-
-                        {% include "funds/includes/rendered_answers.html" %}
+                        {% if request.user|can_translate_submission %}
+                            <div class="wrapper" hx-get="{% url 'funds:submissions:partial-translate-answers' object.id %}" hx-trigger="translateSubmission from:body" hx-indicator="#translate-card-loading" hx-vals='js:{fl: event.detail.from_lang, tl: event.detail.to_lang}'>
+                                {% include "funds/includes/rendered_answers.html" %}
+                            </div>
+                        {% else %}
+                            <div class="wrapper">
+                                {% include "funds/includes/rendered_answers.html" %}
+                            </div>
+                        {% endif %}
 
                     </article>
                 {% endif %}

hypha/apply/funds/templates/funds/includes/admin_primary_actions.html

@@ -1,5 +1,5 @@
 {% load i18n %}
-{% load heroicons primaryactions_tags %}
+{% load heroicons primaryactions_tags translate_tags %}
 
 <h5>{% trans "Actions to take" %}</h5>
 
@@ -84,6 +84,13 @@ <h5>{% trans "Actions to take" %}</h5>
         <summary class="sidebar__separator sidebar__separator--medium">{% trans "More actions" %}</summary>
         <a class="button button--white button--full-width button--bottom-space" href="{% url 'funds:submissions:revisions:list' submission_pk=object.id %}">{% trans "Revisions" %}</a>
 
+        {% if request.user|can_translate_submission %}
+            <button class="button button--white button--full-width button--bottom-space" hx-get="{% url 'funds:submissions:translate' pk=object.pk %}" hx-target="#htmx-modal">
+                {% heroicon_outline "language" aria_hidden="true" size=15 stroke_width=2 class="inline align-baseline me-1" %}
+                {% trans "Translate" %}
+            </button>
+        {% endif %}
+
         <button
             class="button button--white button--full-width button--bottom-space"
             hx-get="{% url 'funds:submissions:metaterms_update' pk=object.pk %}"

hypha/apply/funds/templates/funds/includes/rendered_answers.html

@@ -1,5 +1,23 @@
-{% load i18n wagtailusers_tags workflow_tags %}
-
+{% load i18n wagtailusers_tags workflow_tags translate_tags heroicons %}
+{% if request.user|can_translate_submission %}
+    {% if from_lang_name and to_lang_name %}
+        {# For active translations #}
+        <div class="w-full text-center my-2 py-5 border rounded-lg shadow-md">
+            <span>
+                {% heroicon_outline "language" aria_hidden="true" size=15 stroke_width=2 class="inline align-baseline me-1" %}
+                {% blocktrans %} This application is translated from {{from_lang_name}} to {{to_lang_name}}. {% endblocktrans %}
+                <a href="{% url 'funds:submissions:detail' object.id %}">
+                    {% trans "See original" %}
+                </a>
+            </span>
+        </div>
+    {% else %}
+        {# For a translation loading indicator #}
+        <div id="translate-card-loading" class="w-full text-center h-0 m-0 p-0 overflow-hidden content-center rounded-lg shadow-md animate-pulse htmx-indicator">
+            <span class="w-[490px] bg-gray-200 rounded-lg"></span>
+        </div>
+    {% endif %}
+{% endif %}
 <h3 class="text-xl border-b pb-2 font-bold">{% trans "Proposal Information" %}</h3>
 <div class="hypha-grid hypha-grid--proposal-info">
     {% if object.get_value_display != "-" %}
@@ -44,3 +62,18 @@ <h5 class="text-base">{% trans "Organization name" %}</h5>
 <div class="rich-text rich-text--answers">
     {{ object.output_answers }}
 </div>
+
+<style type="text/css">
+    #translate-card-loading.htmx-request.htmx-indicator{
+        height: 64px;
+        transition: height 0.25s ease-in;
+        margin-top: 0.5rem;
+        margin-bottom: 0.5rem;
+        border-width: 1px;
+    }
+
+    #translate-card-loading.htmx-request.htmx-indicator span {
+        display: inline-block;
+        height: 1rem;
+    }
+</style>

hypha/apply/funds/templates/funds/includes/translate_application_form.html

@@ -0,0 +1,125 @@
+{% load i18n static heroicons translate_tags %}
+{% modal_title %}{% trans "Translate" %}{% endmodal_title %}
+<form
+    class="px-2 pb-4 form"
+    id="translate_form"
+    method="POST"
+    action="{{ request.path }}"
+    hx-post="{{ request.path }}"
+>
+    {% csrf_token %}
+    {{ form.media }}
+    {% for hidden in form.hidden_fields %}
+        {{ hidden }}
+    {% endfor %}
+
+    <div>
+        {% if form.errors %}
+            {% for field in form %}
+                {% for error in field.errors %}
+                    <div class="alert alert-danger">
+                        <strong>{{ error|escape }}</strong>
+                    </div>
+                {% endfor %}
+            {% endfor %}
+            {% for error in form.non_field_errors %}
+                <div class="alert alert-danger">
+                    <strong>{{ error|escape }}</strong>
+                </div>
+            {% endfor %}
+        {% endif %}
+        <div class="flex mt-3 justify-center space-x-2">
+            <fieldset class="w-2/5">
+                <div>
+                    {{ form.from_lang }}
+                </div>
+            </fieldset>
+            <div class="flex flex-col justify-center">
+                {% heroicon_outline "arrow-right" aria_hidden="true" size=15 stroke_width=2 class="inline align-baseline me-1" %}
+            </div>
+            <fieldset class="w-2/5">
+                <div>
+                    {{ form.to_lang }}
+                </div>
+            </fieldset>
+        </div>
+    </div>
+
+    <div class="mt-5 sm:gap-4 sm:mt-4 sm:flex sm:flex-row-reverse">
+
+        {# Button text inserted below to prevent redundant translations #}
+        <button id="translate-btn" class="w-full button button--primary sm:w-auto" type="submit"></button>
+
+        <button
+            type="button"
+            class="inline-flex items-center justify-center w-full px-3 py-2 mt-3 text-sm font-semibold text-gray-900 bg-white rounded-sm shadow-sm ring-1 ring-inset ring-gray-300 hover:bg-gray-50 sm:mt-0 sm:w-auto"
+            @click="show = false"
+        >{% trans "Cancel" %}</button>
+        <span class="inline-block" data-tooltip="{% trans "Translations are an experimental feature and may be inaccurate" %}">{% heroicon_outline "information-circle" aria_hidden="true" size=15 stroke_width=2 class="inline align-baseline me-1" %}</span>
+    </div>
+</form>
+
+<script type="module">
+    import Choices from "{% static 'js/esm/choices.js-10-2-0.js' %}";
+
+    const choices = JSON.parse('{{ json_choices|safe }}')
+
+    {# Define translations for the button text #}
+    const CLEAR_TEXT = "{% trans "Clear" %}"
+    const TRANSLATE_TEXT = "{% trans "Translate" %}"
+
+    function getToChoices(from_lang) {
+        const selected = choices.find((choice) => choice.value === from_lang)
+        return selected ? selected.to : []
+    }
+
+    // Check if a given from/to lang combo is the active translation based on the JSON provided from the server
+    function isTranslationActive(newFromLang, newToLang) {
+        const active = choices.find((choice) => choice.selected === true);
+        if (!active) return false
+
+        const activeFrom = active.value;
+        const activeTo = active.to.find((to) => to.selected === true);
+
+        return (newFromLang === activeFrom && newToLang == activeTo)
+    }
+
+    // Change the button text to indicate the ability to clear the translation
+    function showClearBtn(show) {
+        translateBtn.textContent = show ? CLEAR_TEXT : TRANSLATE_TEXT
+    }
+
+    const selectFromLang = new Choices(document.getElementById('id_from_lang'), { allowHTML: true }).setChoices(choices);
+    const selectToLang = new Choices(document.getElementById('id_to_lang'), { allowHTML: true });
+    const translateBtn = document.getElementById('translate-btn');
+
+    // Initial setting of "to language" choices/disabling of field depending on starting "from language" values
+    if(selectFromLang.getValue()?.value) {
+        selectToLang.setChoices(getToChoices(selectFromLang.getValue().value))
+        showClearBtn(true)
+    } else {
+        showClearBtn(false)
+        selectToLang.disable();
+    }
+
+    // Event handler for when the "from language" selection is updated
+    selectFromLang.passedElement.element.addEventListener('change', (event) => {
+        const toLangChoices = getToChoices(event.detail.value)
+        if (toLangChoices.length > 0) {
+            selectToLang.setChoices(toLangChoices, 'value', 'label', true)
+            selectToLang.enable();
+            showClearBtn(isTranslationActive(event.detail.value, selectToLang.getValue().value));
+        } else {
+            selectToLang.disable();
+            showClearBtn(false);
+            translateBtn.disabled = true;
+        }
+    });
+
+    // Event handler for when "to language" selection is updated
+    selectToLang.passedElement.element.addEventListener('change', (event) => {
+        if (isTranslationActive(selectFromLang.getValue().value, event.detail.value)) {
+            showClearBtn(true);
+        }
+    })
+</script>