Merge branch 'main' into use-name-full

Author: BryceStevenWilley

.github/workflows/alkiln_tests.yml

@@ -8,6 +8,11 @@ on:
         description: 'Optional. Use a "tag expression" specify which tagged tests to run. See https://cucumber.io/docs/cucumber/api/#tag-expressions for syntax.'
         default: ''
         required: false
+      show_docker_output:
+        required: false
+        default: false
+        type: boolean
+        description: 'Show the docker logs while building the GitHub server container. It will also save the docker log artifact. This might show sensitive config information.'
   # To run your tests on a schedule, delete the first "#" symbol at the beginning of each line below.
   ## Also see https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule
   ## Also see https://crontab.guru/examples.html
@@ -20,12 +25,22 @@ jobs:
     name: Run end-to-end tests
     steps:
       - uses: actions/checkout@v3
-      - name: Use ALKiln to run end-to-end tests
+      - name: "ALKiln - Start the isolated temporary docassemble server on GitHub"
+        id: github_server
+        uses: suffolkLITLab/ALKiln/action_for_github_server@v5
+        with:
+          SHOW_DOCKER_OUTPUT: "${{ github.event.inputs.show_docker_output }}"
+      - run: echo "ALKiln finished starting the isolated GitHub docassemble server"
+      - name: Use ALKiln to run tests
         uses: SuffolkLITLab/ALKiln@v5
+        env:
+          ALKILN_TAG_EXPRESSION: "${{ (github.event.inputs.tags && format('{0}', github.event.inputs.tags)) }}"
         with:
-          SERVER_URL: "${{ secrets.SERVER_URL }}"
-          DOCASSEMBLE_DEVELOPER_API_KEY: "${{ secrets.DOCASSEMBLE_DEVELOPER_API_KEY }}"
-          SERVER_RELOAD_TIMEOUT_SECONDS: 120
+          SERVER_URL: "${{ steps.github_server.outputs.SERVER_URL }}"
+          DOCASSEMBLE_DEVELOPER_API_KEY: "${{ steps.github_server.outputs.DOCASSEMBLE_DEVELOPER_API_KEY }}"
+          INSTALL_METHOD: "server"
+          # To learn more, see https://assemblyline.suffolklitlab.org/docs/alkiln/writing/#optional-inputs
+          ALKILN_TAG_EXPRESSION: "${{ env.ALKILN_TAG_EXPRESSION }}"
       - run: echo "Finished running ALKiln end-to-end tests"
 
       ## To make a new issue in your repository when a test fails,

.github/workflows/da_build.yml

@@ -0,0 +1,12 @@
+name: DA Build
+on:
+  push:
+    branches: [ main, master ]
+  pull_request:
+    branches: [ main, master ]
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: SuffolkLITLab/ALActions/da_build@main

.github/workflows/valid_jinja2.yml

@@ -0,0 +1,11 @@
+name: Valid Jinja2
+on:
+  push:
+    branches:
+      - "**"
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: SuffolkLITLab/ALActions/valid_jinja2@main

.github/workflows/word_diff.yml

@@ -0,0 +1,9 @@
+name: Word Diff
+on:
+  pull_request:
+jobs:
+  diff:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: SuffolkLITLab/ALActions/word_diff@main

.gitignore

@@ -4,6 +4,7 @@ docassemble/assemblylinewizard/docassemble*
 .vscode/
 .hypothesis/
 *.egg-info
+.code/
 
 # alkiln testing
 tests/chromedriver

CHANGELOG.md

@@ -1,9 +1,11 @@
-# CHANGELOG
+# CHANGELOG (Deprecated)
 
-## For revisions after v1.9.0, see https://github.com/SuffolkLITLab/docassemble-ALWeaver/releases/
+## For releases after v1.9.0 (i.e. anything after 2023), see https://github.com/SuffolkLITLab/docassemble-ALWeaver/releases/
 
 ## Version v1.9.0
 
+May 15th, 2023.
+
 ### New
 
 * Radio buttons in weaver by @BryceStevenWilley in [PR 754](https://github.com/SuffolkLITLab/docassemble-ALWeaver/pull/754)

docassemble/ALWeaver/custom_values.py

@@ -1,5 +1,5 @@
-from typing import Any, Dict, List, Union, Optional
-from pathlib import Path
+from typing import Any, Dict, List, Optional, Union
+
 import ruamel.yaml as yaml
 from docassemble.base.util import log, DADict, DAList, DAStore, path_and_mimetype
 from packaging.version import Version
@@ -21,6 +21,16 @@
 ]
 
 
+_CAPABILITY_LOAD_DEFAULTS: Dict[str, Any] = {
+    "base": "docassemble.ALWeaver",
+    "minimum_version": "1.5",
+    "include_playground": False,
+}
+
+
+_capability_cache: Optional[Dict[str, Any]] = None
+
+
 def _package_name(package_name: Optional[str] = None):
     """Get package name without the name of the current module, like: docassemble.ALWeaver instead of
     docassemble.ALWeaver.advertise_capabilities"""
@@ -49,8 +59,10 @@ def __str__(self):
 
 
 def load_capabilities(
-    base: str = "docassemble.ALWeaver", minimum_version="1.5", include_playground=False
-):
+    base: str = "docassemble.ALWeaver",
+    minimum_version: str = "1.5",
+    include_playground: bool = False,
+) -> Dict[str, Any]:
     """
     Load and return a dictionary containing all advertised capabilities matching
     the specified minimum version, and optionally include capabilities that were
@@ -67,10 +79,9 @@ def load_capabilities(
         weaverdata.get("published_configuration_capabilities") or {}
     )
     try:
+        yaml_loader = yaml.YAML(typ="safe", pure=True)
         with open(this_yaml) as f:
-            this_yaml_contents = f.read()
-
-        first_file = list(yaml.safe_load_all(this_yaml_contents))[0]
+            first_file = yaml_loader.load(f)
 
         capabilities = {"Default configuration": first_file}
     except:
@@ -96,41 +107,45 @@ def load_capabilities(
                 f"{package_name}:data/sources/{published_configuration_capabilities[package_name][0]}"
             )[0]
             try:
+                yaml_loader = yaml.YAML(typ="safe", pure=True)
                 with open(path) as f:
-                    yaml_contents = f.read()
-                capabilities[package_name] = list(yaml.safe_load_all(yaml_contents))[0]
+                    capabilities[package_name] = yaml_loader.load(f)
             except:
                 log(f"Unable to load published Weaver configuration file {path}")
 
     return capabilities
 
 
-_al_weaver_capabilities = load_capabilities()
+def _get_capabilities(refresh: bool = False) -> Dict[str, Any]:
+    """Return cached capabilities, optionally refreshing from the datastore."""
+
+    global _capability_cache
+    if refresh or _capability_cache is None:
+        _capability_cache = load_capabilities(**_CAPABILITY_LOAD_DEFAULTS)
+    return _capability_cache
 
 
 def get_possible_deps_as_choices(dep_category=None):
     """Gets the possible yml files that the generated interview will depend on"""
 
     dep_choices = []
 
+    capabilities = _get_capabilities()
+
     # TODO: do we want to prefix the choice with the package name?
-    for capability in _al_weaver_capabilities:
+    for capability in capabilities:
         if dep_category == "organization":
             dep_choices.extend(
                 [
                     {item.get("include_name"): item.get("description")}
-                    for item in _al_weaver_capabilities[capability].get(
-                        "organization_choices", []
-                    )
+                    for item in capabilities[capability].get("organization_choices", [])
                 ]
             )
         elif dep_category == "jurisdiction":
             dep_choices.extend(
                 [
                     {item.get("include_name"): item.get("description")}
-                    for item in _al_weaver_capabilities[capability].get(
-                        "jurisdiction_choices", []
-                    )
+                    for item in capabilities[capability].get("jurisdiction_choices", [])
                 ]
             )
 
@@ -146,14 +161,14 @@ def get_pypi_deps_from_choices(choices: Union[List[str], DADict]):
     else:  # List
         choice_list = choices
 
-    for capability in _al_weaver_capabilities:
+    capabilities = _get_capabilities()
+
+    for capability in capabilities:
         pypi_deps.extend(
             [
                 choice.get("dependency")
-                for choice in _al_weaver_capabilities[capability].get(
-                    "organization_choices", []
-                )
-                + _al_weaver_capabilities[capability].get("jurisdiction_choices", [])
+                for choice in capabilities[capability].get("organization_choices", [])
+                + capabilities[capability].get("jurisdiction_choices", [])
                 if choice.get("dependency")
                 and choice.get("include_name") in choice_list
             ]
@@ -178,16 +193,14 @@ def get_full_dep_details(dep_category: Optional[str] = None) -> List:
     filtered and used as needed."""
     dep_choices = []
 
+    capabilities = _get_capabilities()
+
     # TODO: do we want to prefix the choice with the package name?
-    for capability in _al_weaver_capabilities:
+    for capability in capabilities:
         if dep_category == "organization":
-            dep_choices.extend(
-                _al_weaver_capabilities[capability].get("organization_choices", [])
-            )
+            dep_choices.extend(capabilities[capability].get("organization_choices", []))
         elif dep_category == "jurisdiction":
-            dep_choices.extend(
-                _al_weaver_capabilities[capability].get("jurisdiction_choices", [])
-            )
+            dep_choices.extend(capabilities[capability].get("jurisdiction_choices", []))
 
     return list(unique_everseen(dep_choices))
 
@@ -200,7 +213,9 @@ def get_matching_deps(
     dep_choices = []
 
     # TODO: do we want to prefix the choice with the package name?
-    for capability in _al_weaver_capabilities.values():
+    capabilities = _get_capabilities()
+
+    for capability in capabilities.values():
         if dep_category == "organization":
             dep_choices.extend(
                 [
@@ -234,7 +249,9 @@ def get_output_mako_choices() -> Dict[str, str]:
     the interview YAML file. It will be one deep.
     """
     choices = {}
-    for key, capability in _al_weaver_capabilities.items():
+    capabilities = _get_capabilities()
+
+    for key, capability in capabilities.items():
         if capability.get("output_mako"):
             if isinstance(capability["output_mako"], str):
                 choices[key] = capability["output_mako"]
@@ -309,3 +326,10 @@ def advertise_capabilities(
     weaverdata.set(
         "published_configuration_capabilities", published_configuration_capabilities
     )
+    global _CAPABILITY_LOAD_DEFAULTS
+    _CAPABILITY_LOAD_DEFAULTS = {
+        **_CAPABILITY_LOAD_DEFAULTS,
+        "base": base,
+        "minimum_version": minimum_version,
+    }
+    _get_capabilities(refresh=True)

docassemble/ALWeaver/data/questions/assembly_line.yml

@@ -59,10 +59,6 @@ code: |
         if yes_recognize_form_fields: # ask to add fields
           process_field_recognition
       if im_feeling_lucky:
-        if not formfyxer_available() and user_logged_in() and user_has_privilege(["admin"]) and install_en_core_web_lg:
-          install_spacy_background_task
-          if not install_spacy_background_task.ready():
-            install_spacy_background_waiting_screen
         nav.set_sections([
           {"nav_generate_interview": "Generate your interview"},
           {"nav_review": "Review your answers"},
@@ -1904,20 +1900,6 @@ subquestion: |
 
   % endfor
 
-  % if not formfyxer_available():
-  <div class="alert alert-warning" role="alert">
-  Automatic field grouping isn't set up. To use it, you need:
-
-  <ol>
-  <li>The `en_core_web_lg` installed, or</li>
-  <li>An API token for tools.suffolklitlab.org installed in your configuration.</li>
-  </ol>
-
-  You can still use auto drafting mode but all questions will be
-  placed on one screen. You can edit them in your playground later.
-  </div>
-  % endif
-
 fields: 
   - Do you want us to draft the interview for you?: im_feeling_lucky
     datatype: radio
@@ -1947,40 +1929,6 @@ fields:
     show if: interview.start_with_json
     accept: |
       "application/json"
-  - Try to install the `en_core_web_lg` package before using auto drafting mode: install_en_core_web_lg
-    help: |
-      Installing `en_core_web_lg` will allow you to use automatic field
-      grouping but it may use more memory on your server.
-
-      If you are a nonprofit, you can request an API token to use
-      auto field grouping without installing `en_core_web_lg`.
-    datatype: yesnoradio
-    show if:
-      variable: im_feeling_lucky
-      is: True
-      code: |
-        not formfyxer_available() and user_logged_in() and user_has_privilege(["admin"])
----
-code: |
-  install_spacy_background_task = background_action("install_spacy_background_action")
----
-code: |
-  install_spacy_model()
-  background_response()
-  install_spacy_background_action = True
----
-event: install_spacy_background_waiting_screen
-id: waiting screen
-question: |
-  Installing `en_core_web_lg`
-subquestion: |
-  Please wait while we install `en_core_web_lg`. This can take a
-  few minutes to complete.
-
-  <div class="spinner-border" role="status">
-    <span class="visually-hidden">Loading...</span>
-  </div>
-reload: True
 ---
 template: explain_auto_drafting
 subject: |