Merge branch 'master' of github.com:mideind/Greynir

sveinbjornt · sveinbjornt · commit f18fa5b95f8a · 2025-05-15T18:34:08.000Z
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
@@ -10,7 +10,6 @@ on:
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     # Service containers to run with `container-job`
@@ -37,7 +36,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ["3.9"]
+        python-version: ["3.9", "3.12", "pypy3.9", "pypy3.10"]
 
     steps:
     - uses: actions/checkout@v4
@@ -51,12 +50,9 @@ jobs:
           sudo update-locale
     - name: Install dependencies
       run: |
-        pip cache purge
-        pip install --upgrade pip wheel setuptools pytest curlylint
-        pip install --upgrade git+https://github.com/mideind/Tokenizer#egg=tokenizer
-        pip install --upgrade git+https://github.com/mideind/GreynirEngine#egg=reynir
-        pip install --upgrade git+https://github.com/mideind/Icespeak#egg=icespeak
-        pip install --upgrade -r requirements.txt
+        pip install uv
+        uv pip install --system --upgrade wheel setuptools pytest curlylint
+        uv pip install --system --upgrade -r requirements.txt
         sudo npm i -g jshint
         sudo apt-get install shellcheck
     - name: Set up database
diff --git a/.gitignore b/.gitignore
@@ -125,8 +125,13 @@ p3*/
 p7*/
 pypy/
 pypy*
+pypy39/*
+pypy310/*
 cpython*/
 venv
+p312/*
+p313/*
+p39/*
 
 # Installer logs
 pip-log.txt
diff --git a/README.md b/README.md
@@ -29,9 +29,11 @@ database and processed by grammatical pattern matching modules to obtain stateme
 of fact and relations between stated facts.
 
 An overview of the technology behind Greynir can be found in the paper
-[A Wide-Coverage Context-Free Grammar for Icelandic
-and an Accompanying Parsing System](https://acl-bg.org/proceedings/2019/RANLP%202019/pdf/RANLP160.pdf)
-by Vilhjálmur Þorsteinsson, Hulda Óladóttir and Hrafn Loftsson *(Proceedings of Recent Advances in Natural Language Processing, pages 1397–1404, Varna, Bulgaria, Sep 2–4, 2019).*
+[A Wide-Coverage Context-Free Grammar for Icelandic and an Accompanying
+Parsing System](https://acl-bg.org/proceedings/2019/RANLP%202019/pdf/RANLP160.pdf)
+by Vilhjálmur Þorsteinsson, Hulda Óladóttir and Hrafn Loftsson
+*(Proceedings of Recent Advances in Natural Language Processing,
+pages 1397–1404, Varna, Bulgaria, Sep 2–4, 2019).*
 
 <a href="https://raw.githubusercontent.com/mideind/Greynir/master/static/img/tree-example.png" title="Greynir parse tree">
 <img src="static/img/tree-example-small.png" width="400" height="450" alt="Greynir parse tree">
@@ -57,10 +59,11 @@ structured, recursive trees that directly correspond to the original grammar.
 These trees can then be further processed and acted upon by sets of Python
 functions that are linked to grammar nonterminals.
 
-**Greynir is currently able to parse about *90%* of sentences** in a typical news article from the web,
-and many well-written articles can be parsed completely. It presently has well over a million parsed articles
-in its database, containing more than 16 million parsed sentences. A relatively recent (2021) version of this
-database is available via the [GreynirCorpus](https://github.com/mideind/GreynirCorpus) project.
+**Greynir is currently able to parse about *90%* of sentences** in a typical news article
+from the web, and many well-written articles can be parsed completely. It presently has about
+1.3 million parsed articles in its database, containing about 18 million parsed sentences.
+A 2021 version of this database is available via the
+[GreynirCorpus](https://github.com/mideind/GreynirCorpus) project.
 
 Greynir supports natural language querying of its databases. Users can ask about person names, titles and
 entity definitions and get appropriate replies. The HTML5 Web Speech API is supported to allow
@@ -235,7 +238,7 @@ See [Contributing to Greynir](CONTRIBUTING.md).
 
 ## License
 
-Greynir is Copyright &copy; 2023 [Miðeind ehf.](https://mideind.is)  
+Greynir is Copyright &copy; 2025 [Miðeind ehf.](https://mideind.is)  
 The original author of this software is *Vilhjálmur Þorsteinsson*.
 
 <a href="https://mideind.is"><img src="static/img/mideind-horizontal-small.png" alt="Miðeind ehf."
diff --git a/db/models.py b/db/models.py
@@ -39,7 +39,7 @@
     Float,
     Sequence,
     Boolean,
-    DateTime,  # type: ignore # Imported for re-export
+    DateTime,  # type: ignore # noqa: F401 # Imported for re-export
     UniqueConstraint,
     Index,
     ForeignKey,
diff --git a/images.py b/images.py
@@ -327,8 +327,7 @@ def get_staticmap_image(
         return None
 
     if r.status_code == 200:
-        r.raw.decode_content = True
-        return BytesIO(r.raw.data)
+        return BytesIO(r.content)
 
     logging.warning(f"Status {r.status_code} when requesting static map image")
     return None
diff --git a/main.py b/main.py
@@ -63,7 +63,7 @@
     QUERIES_UTIL_GRAMMAR_DIR,
 )
 
-from reynir.version import __version__ as greynir_version
+from reynir import __version__ as greynir_version
 from tokenizer import __version__ as tokenizer_version
 
 
@@ -101,7 +101,7 @@
 app.config["CACHE"] = Cache(app, config={"CACHE_TYPE": cache_type})
 
 # Register blueprint routes
-from routes import routes, max_age  # type: ignore
+from routes import routes, max_age  # type: ignore  # noqa: E402
 
 app.register_blueprint(routes)
 
@@ -288,8 +288,7 @@ def inject_nn_bools() -> Dict[str, Union[str, bool]]:
     except socket_error as e:
         if e.errno == errno.EADDRINUSE:  # Address already in use
             logging.error(
-                f"Another application is already running at"
-                " host {Settings.HOST}:{Settings.PORT}"
+                f"Another application is already running at host {Settings.HOST}:{Settings.PORT}"
             )
             sys.exit(1)
         else:
diff --git a/processors/disabled/attribs.py b/processors/disabled/attribs.py
@@ -43,8 +43,6 @@
 
 """
 
-from __future__ import annotations
-
 from queries import QueryStateDict
 from tree import Node, ParamList, Result, TreeStateDict
 
diff --git a/queries/builtin.py b/queries/builtin.py
@@ -26,8 +26,7 @@
 
 """
 
-from typing import Callable, Dict, Iterable, Optional, List, Any, Tuple, cast
-from typing_extensions import TypedDict
+from typing import Callable, Dict, Iterable, Optional, List, Any, Tuple, cast, TypedDict
 
 import math
 from datetime import datetime, timezone
diff --git a/queries/currency.py b/queries/currency.py
@@ -29,11 +29,13 @@
 
 from typing import Any, Dict, List, Mapping, Optional, Sequence, cast
 
-import cachetools  # type: ignore
+import cachetools
 import random
 import logging
+import xml.etree.ElementTree as ET
 
 from icespeak import gssml
+import requests
 
 from queries import Query, QueryStateDict
 from queries.util import (
@@ -199,18 +201,19 @@ def QCurAmountConversion(node: Node, params: QueryStateDict, result: Result) ->
     result.op = "convert"
 
 
+_ARION_EXCHR_XML_API_URL = "https://www.arionbanki.is/markadir/gjaldmidlar/gengi/xml-export"
+
 _CURR_API_URL = "https://apis.is/currency/arion"
 _CURR_CACHE_TTL = 3600  # seconds
 
 
+def _fetch_arion_exchange_rates() -> Optional[Dict[str, float]]:
+    """Fetch exchange rate data from apis.is and cache it."""
+
+
+
 @cachetools.cached(cachetools.TTLCache(1, _CURR_CACHE_TTL))
 def _fetch_exchange_rates() -> Optional[Dict[str, float]]:
-<<<<<<< Updated upstream
-    """Fetch exchange rate data from apis.is and cache it."""
-    res = query_json_api(_CURR_API_URL)
-    if not isinstance(res, dict) or "results" not in res:
-        logging.warning(f"Unable to fetch exchange rate data from {_CURR_API_URL}")
-=======
     """Fetch exchange rate data from Arion banki and cache it."""
     try:
         resp = requests.get(_ARION_EXCHR_XML_API_URL, timeout=5)
@@ -229,13 +232,7 @@ def _fetch_exchange_rates() -> Optional[Dict[str, float]]:
         return exchange_rates
     except Exception as e:
         logging.warning(f"Error fetching exchange rate data from {_ARION_EXCHR_XML_API_URL}: {e}")
->>>>>>> Stashed changes
         return None
-    return {
-        c["shortName"]: c["value"]
-        for c in res["results"]
-        if "shortName" in c and "value" in c
-    }
 
 
 def fetch_exchange_rates() -> Optional[Dict[str, float]]:
diff --git a/queries/flights.py b/queries/flights.py
@@ -26,8 +26,7 @@
 # TODO: Map country to capital city, e.g. "Svíþjóð" -> "Stokkhólmur"
 # TODO: Fetch more than one flight using "flight_count"?
 
-from typing import Any, List, Dict, Optional, cast
-from typing_extensions import TypedDict
+from typing import Any, List, Dict, Optional, cast, TypedDict
 
 import re
 import random
diff --git a/queries/gpt.py b/queries/gpt.py
@@ -36,8 +36,9 @@
     Type,
     Union,
     cast,
+    TypedDict,
 )
-from typing_extensions import TypedDict, NotRequired
+from typing_extensions import NotRequired
 
 import abc
 import inspect
diff --git a/queries/petrol.py b/queries/petrol.py
@@ -111,23 +111,23 @@ def QPetrolClosestCheapestStation(
 _COMPANY_NAME_FIXES = {"Costco Iceland": "Costco"}
 
 
-_PETROL_API = "https://apis.is/petrol"
+_PETROL_API = "https://raw.githubusercontent.com/gasvaktin/gasvaktin/refs/heads/master/vaktin/gas.min.json"
 _PETROL_CACHE_TTL = 3600  # seconds, ttl 1 hour
 
 
 @cachetools.cached(cachetools.TTLCache(1, _PETROL_CACHE_TTL))
 def _get_petrol_station_data() -> Optional[List]:
     """Fetch list of petrol stations w. prices from apis.is (Gasvaktin)"""
     pd = query_json_api(_PETROL_API)
-    if not isinstance(pd, dict) or "results" not in pd:
+    if not isinstance(pd, dict) or "stations" not in pd:
         return None
 
     # Fix company names
-    for s in pd["results"]:
+    for s in pd["stations"]:
         name = s.get("company", "")
         s["company"] = _COMPANY_NAME_FIXES.get(name, name)
 
-    return pd["results"]
+    return pd["stations"]
 
 
 def _stations_with_distance(loc: Optional[LatLonTuple]) -> Optional[List]:
diff --git a/queries/play.py b/queries/play.py
@@ -23,16 +23,13 @@
 
 """
 
-from __future__ import annotations
-
-from typing import Optional, List, Any, Match
-from typing_extensions import TypedDict
+from typing import Optional, List, Any, Match, TypedDict
 
 import re
 import logging
 from random import choice
 
-from pyyoutube import Api, SearchListResponse  # type: ignore
+from pyyoutube import Api
 
 from queries import Query
 from queries.util import gen_answer
diff --git a/queries/schedules.py b/queries/schedules.py
@@ -27,8 +27,7 @@
 # TODO: Channels provided by Síminn
 # TODO: "Rás tveir" vandamál, ætti að vera "Rás tvö"
 
-from typing import List, Dict, Optional, Tuple, Any, cast
-from typing_extensions import TypedDict
+from typing import List, Dict, Optional, Tuple, Any, cast, TypedDict
 from queries import Query, QueryStateDict
 from tree import Node, TerminalNode, ParamList, Result
 
diff --git a/queries/util/__init__.py b/queries/util/__init__.py
@@ -21,8 +21,6 @@
 
 """
 
-from __future__ import annotations
-
 from typing import (
     Any,
     Mapping,
@@ -33,8 +31,8 @@
     Tuple,
     Union,
     cast,
+    TypedDict,
 )
-from typing_extensions import TypedDict
 
 import logging
 import requests
diff --git a/queries/util/openai_gpt.py b/queries/util/openai_gpt.py
@@ -23,17 +23,15 @@
 
 """
 
-from __future__ import annotations
-
 from typing import (
     Any,
     Dict,
     Iterable,
     List,
     Mapping,
     Tuple,
+    TypedDict,
 )
-from typing_extensions import TypedDict
 
 import re
 import os
diff --git a/queries/weather.py b/queries/weather.py
@@ -44,8 +44,6 @@
 # TODO: "Hvernig er færðin"
 # TODO: Er rigning úti? Er sól úti? Er sól á Húsavík? Er rigning í Reykjavík?
 
-from __future__ import annotations
-
 from typing import Dict, Mapping, Optional, Union
 
 import os
diff --git a/requirements.txt b/requirements.txt
@@ -1,33 +1,37 @@
 # Flask
 werkzeug>=2.3.7
 Flask==2.3.3
-flask-caching>=2.0.2
-Flask-Cors>=4.0.0
+flask-caching>=2.3.1
+Flask-Cors>=5.0.1
 # DB
 psycopg2cffi==2.9.0
-SQLAlchemy==1.4.50
-sqlalchemy2-stubs>=0.0.2a35
+SQLAlchemy==1.4.54
+sqlalchemy2-stubs>=0.0.2a38
+sqlalchemy-stubs>=0.4
 # Util
-cachetools>=5.3.1
-requests>=2.31.0
-typing-extensions>=4.7.1
-python-dotenv>=1.0.0
+cachetools>=5.5.2
+requests>=2.32.3
+typing-extensions>=4.13.2
+python-dotenv>=1.1.0
 # Scraping
-beautifulsoup4>=4.12.2
-feedparser>=6.0.10
+beautifulsoup4==4.13.4
+feedparser==6.0.11
 # Query modules
-pytz>=2023.3
-timezonefinder>=6.2.0
-rjsmin>=1.2.1
-python-youtube>=0.9.1
-country-list>=1.0.0
+pytz==2025.2
+timezonefinder==6.5.9
+rjsmin>=1.2.4
+python-youtube==0.9.1
+country-list>=1.1.0
 # For OpenAI GPT support
 openai==1.14.3
+httpx==0.27.2 # to address issue in old version of OpenAI API
 # Ours
-reynir>=3.5.4
-islenska>=0.4.8
-icespeak<0.4.0
-iceaddr>=0.5.6
+reynir>=3.5.7
+islenska>=1.0.3
+# Pinning old version of Icespeak since newer require
+# grpcio, which doesn't work under PyPy. :/
+icespeak @ git+https://github.com/mideind/Icespeak@greynir
+iceaddr>=0.5.8
 iceweather>=0.2.3
 cityloc>=0.1.1
 straeto>=1.4.1
diff --git a/routes/loc.py b/routes/loc.py
diff --git a/scraper.py b/scraper.py
diff --git a/scrapers/default.py b/scrapers/default.py
diff --git a/search.py b/search.py
diff --git a/templates/about.html b/templates/about.html
diff --git a/templates/container.html b/templates/container.html
diff --git a/tnttagger.py b/tnttagger.py
diff --git a/tree/__init__.py b/tree/__init__.py
