Merge pull request #1273 from d-v-b/impr/modernize-pre-commit

Impr/modernize pre commit

Author: dimitri-yatsenko

.codespellrc

@@ -23,7 +23,7 @@ jobs:
         extra_args: codespell --all-files
     - uses: pre-commit/[email protected]
       with:
-        extra_args: black --all-files
+        extra_args: ruff --all-files
     - uses: pre-commit/[email protected]
       with:
-        extra_args: flake8 --all-files
+        extra_args: ruff-format --all-files

.github/workflows/lint.yaml

@@ -20,37 +20,17 @@ repos:
   rev: v2.4.1
   hooks:
     - id: codespell
-- repo: https://github.com/pycqa/isort
-  rev: 6.0.1  # Use the latest stable version
+      args: [--toml, pyproject.toml]
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.8.4
   hooks:
-    - id: isort
-      args:
-        - --profile=black  # Optional, makes isort compatible with Black
-- repo: https://github.com/psf/black
-  rev: 25.1.0 # matching versions in pyproject.toml and github actions
-  hooks:
-  - id: black
-    args: ["-v", "src", "tests", "--diff"] # --required-version is conflicting with pre-commit
-- repo: https://github.com/PyCQA/flake8
-  rev: 7.3.0
-  hooks:
-  # syntax tests
-  - id: flake8
-    args:
-      - --select=E9,F63,F7,F82
-      - --count
-      - --show-source
-      - --statistics
-    files: src/ # a lot of files in tests are not compliant
-  # style tests
-  - id: flake8
-    args:
-      - --ignore=E203,E722,W503
-      - --count
-      - --max-complexity=62
-      - --max-line-length=127
-      - --statistics
-    files: src/ # a lot of files in tests are not compliant
+    # Run the linter
+    - id: ruff
+      args: [--fix]
+      files: ^(src/|tests/)
+    # Run the formatter
+    - id: ruff-format
+      files: ^(src/|tests/)
 - repo: https://github.com/rhysd/actionlint
   rev: v1.7.7
   hooks:

.pre-commit-config.yaml

@@ -9,9 +9,7 @@
 nav = mkdocs_gen_files.Nav()
 for path in sorted(Path(package).glob("**/*.py")):
     with mkdocs_gen_files.open(f"api/{path.with_suffix('')}.md", "w") as f:
-        module_path = ".".join(
-            [p for p in path.with_suffix("").parts if p != "__init__"]
-        )
+        module_path = ".".join([p for p in path.with_suffix("").parts if p != "__init__"])
         print(f"::: {module_path}", file=f)
     nav[path.parts] = f"{path.with_suffix('')}.md"
 

docs/src/api/make_pages.py

@@ -229,9 +229,7 @@
     "                home_city=city,\n",
     "                home_state=state,\n",
     "                home_zip=zipcode,\n",
-    "                date_of_birth=str(\n",
-    "                    fake.date_time_between(start_date=\"-35y\", end_date=\"-15y\").date()\n",
-    "                ),\n",
+    "                date_of_birth=str(fake.date_time_between(start_date=\"-35y\", end_date=\"-15y\").date()),\n",
     "                home_phone=fake.phone_number()[:20],\n",
     "            )"
    ]
@@ -261,9 +259,7 @@
     "\n",
     "StudentMajor.insert(\n",
     "    {**s, **d, \"declare_date\": fake.date_between(start_date=datetime.date(1999, 1, 1))}\n",
-    "    for s, d in zip(\n",
-    "        Student.fetch(\"KEY\"), random.choices(Department.fetch(\"KEY\"), k=len(Student()))\n",
-    "    )\n",
+    "    for s, d in zip(Student.fetch(\"KEY\"), random.choices(Department.fetch(\"KEY\"), k=len(Student())))\n",
     "    if random.random() < 0.75\n",
     ")\n",
     "\n",
@@ -318,17 +314,11 @@
     "    ]\n",
     ")\n",
     "\n",
-    "Term.insert(\n",
-    "    dict(term_year=year, term=term)\n",
-    "    for year in range(1999, 2019)\n",
-    "    for term in [\"Spring\", \"Summer\", \"Fall\"]\n",
-    ")\n",
+    "Term.insert(dict(term_year=year, term=term) for year in range(1999, 2019) for term in [\"Spring\", \"Summer\", \"Fall\"])\n",
     "\n",
     "Term().fetch(order_by=(\"term_year DESC\", \"term DESC\"), as_dict=True, limit=1)[0]\n",
     "\n",
-    "CurrentTerm().insert1(\n",
-    "    {**Term().fetch(order_by=(\"term_year DESC\", \"term DESC\"), as_dict=True, limit=1)[0]}\n",
-    ")\n",
+    "CurrentTerm().insert1({**Term().fetch(order_by=(\"term_year DESC\", \"term DESC\"), as_dict=True, limit=1)[0]})\n",
     "\n",
     "\n",
     "def make_section(prob):\n",
@@ -372,10 +362,7 @@
     "            sections = ((Section & term) - (Course & (Enroll & student))).fetch(\"KEY\")\n",
     "            if sections:\n",
     "                Enroll.insert(\n",
-    "                    {**student, **section}\n",
-    "                    for section in random.sample(\n",
-    "                        sections, random.randrange(min(5, len(sections)))\n",
-    "                    )\n",
+    "                    {**student, **section} for section in random.sample(sections, random.randrange(min(5, len(sections))))\n",
     "                )\n",
     "\n",
     "# assign random grades\n",
@@ -385,10 +372,7 @@
     "random.shuffle(grade_keys)\n",
     "grade_keys = grade_keys[: len(grade_keys) * 9 // 10]\n",
     "\n",
-    "Grade.insert(\n",
-    "    {**key, \"grade\": grade}\n",
-    "    for key, grade in zip(grade_keys, random.choices(grades, k=len(grade_keys)))\n",
-    ")"
+    "Grade.insert({**key, \"grade\": grade} for key, grade in zip(grade_keys, random.choices(grades, k=len(grade_keys))))"
    ]
   },
   {
@@ -544,9 +528,7 @@
     }
    ],
    "source": [
-    "(Grade * LetterGrade) & \"term_year='2018'\" & dj.Top(\n",
-    "    limit=5, order_by=\"points DESC\", offset=5\n",
-    ")"
+    "(Grade * LetterGrade) & \"term_year='2018'\" & dj.Top(limit=5, order_by=\"points DESC\", offset=5)"
    ]
   },
   {
@@ -566,11 +548,7 @@
     }
    ],
    "source": [
-    "(\n",
-    "    (LetterGrade * Grade)\n",
-    "    & \"term_year='2018'\"\n",
-    "    & dj.Top(limit=10, order_by=\"points DESC\", offset=0)\n",
-    ").make_sql()"
+    "((LetterGrade * Grade) & \"term_year='2018'\" & dj.Top(limit=10, order_by=\"points DESC\", offset=0)).make_sql()"
    ]
   },
   {
@@ -590,11 +568,7 @@
     }
    ],
    "source": [
-    "(\n",
-    "    (Grade * LetterGrade)\n",
-    "    & \"term_year='2018'\"\n",
-    "    & dj.Top(limit=20, order_by=\"points DESC\", offset=0)\n",
-    ").make_sql()"
+    "((Grade * LetterGrade) & \"term_year='2018'\" & dj.Top(limit=20, order_by=\"points DESC\", offset=0)).make_sql()"
    ]
   },
   {
@@ -800,9 +774,7 @@
     }
    ],
    "source": [
-    "(Grade * LetterGrade) & \"term_year='2018'\" & dj.Top(\n",
-    "    limit=20, order_by=\"points DESC\", offset=0\n",
-    ")"
+    "(Grade * LetterGrade) & \"term_year='2018'\" & dj.Top(limit=20, order_by=\"points DESC\", offset=0)"
    ]
   },
   {
@@ -1008,9 +980,7 @@
     }
    ],
    "source": [
-    "(LetterGrade * Grade) & \"term_year='2018'\" & dj.Top(\n",
-    "    limit=20, order_by=\"points DESC\", offset=0\n",
-    ")"
+    "(LetterGrade * Grade) & \"term_year='2018'\" & dj.Top(limit=20, order_by=\"points DESC\", offset=0)"
    ]
   },
   {

docs/src/tutorials/dj-top.ipynb

@@ -91,17 +91,47 @@ test = [
 [project.optional-dependencies]
 dev = [
   "pre-commit",
-  "black==24.2.0",
-  "flake8",
-  "isort",
+  "ruff",
   "codespell",
   # including test
   "pytest",
   "pytest-cov",
 ]
 
-[tool.isort]
-profile = "black"
+[tool.ruff]
+# Equivalent to flake8 configuration
+line-length = 127
+target-version = "py39"
+
+[tool.ruff.lint]
+# Enable specific rule sets equivalent to flake8 configuration
+select = [
+    "E",  # pycodestyle errors
+    "W",  # pycodestyle warnings
+    "F",  # pyflakes
+    "C90", # mccabe complexity
+]
+
+# Ignore specific rules (equivalent to flake8 --ignore)
+ignore = [
+    "E203",  # whitespace before ':'
+    "E722",  # bare except
+]
+
+# Per-file ignores (equivalent to flake8 --per-file-ignores)
+[tool.ruff.lint.per-file-ignores]
+"datajoint/diagram.py" = ["C901"]  # function too complex
+"tests/test_blob_matlab.py" = ["E501"]  # SQL hex strings cannot be broken across lines
+
+[tool.ruff.lint.mccabe]
+# Maximum complexity (equivalent to flake8 --max-complexity)
+max-complexity = 62
+
+[tool.ruff.format]
+# Use black-compatible formatting
+quote-style = "double"
+indent-style = "space"
+line-ending = "auto"
 
 [tool.setuptools]
 packages = ["datajoint"]
@@ -110,6 +140,13 @@ package-dir = {"" = "src"}
 [tool.setuptools.dynamic]
 version = { attr = "datajoint.version.__version__"}
 
+[tool.codespell]
+skip = ".git,*.pdf,*.svg,*.csv,*.ipynb,*.drawio"
+# Rever -- nobody knows
+# numer -- numerator variable
+# astroid -- Python library name (not "asteroid")
+ignore-words-list = "rever,numer,astroid"
+
 [tool.pytest_env]
 # Default values - pytest fixtures will override with actual container details
 DJ_USER="root"

pyproject.toml

@@ -20,18 +20,14 @@ def set_password(new_password=None, connection=None, update_config=None):
             logger.warning("Failed to confirm the password! Aborting password change.")
             return
 
-    if version.parse(
-        connection.query("select @@version;").fetchone()[0]
-    ) >= version.parse("5.7"):
+    if version.parse(connection.query("select @@version;").fetchone()[0]) >= version.parse("5.7"):
         # SET PASSWORD is deprecated as of MySQL 5.7 and removed in 8+
         connection.query("ALTER USER user() IDENTIFIED BY '%s';" % new_password)
     else:
         connection.query("SET PASSWORD = PASSWORD('%s')" % new_password)
     logger.info("Password updated.")
 
-    if update_config or (
-        update_config is None and user_choice("Update local setting?") == "yes"
-    ):
+    if update_config or (update_config is None and user_choice("Update local setting?") == "yes"):
         config["database.password"] = new_password
         config.save_local(verbose=True)
 
@@ -67,17 +63,10 @@ def kill(restriction=None, connection=None, order_by=None):
     while True:
         print("  ID USER         HOST          STATE         TIME    INFO")
         print("+--+ +----------+ +-----------+ +-----------+ +-----+")
-        cur = (
-            {k.lower(): v for k, v in elem.items()}
-            for elem in connection.query(query, as_dict=True)
-        )
+        cur = ({k.lower(): v for k, v in elem.items()} for elem in connection.query(query, as_dict=True))
         for process in cur:
             try:
-                print(
-                    "{id:>4d} {user:<12s} {host:<12s} {state:<12s} {time:>7d}  {info}".format(
-                        **process
-                    )
-                )
+                print("{id:>4d} {user:<12s} {host:<12s} {state:<12s} {time:>7d}  {info}".format(**process))
             except TypeError:
                 print(process)
         response = input('process to kill or "q" to quit > ')
@@ -111,15 +100,11 @@ def kill_quick(restriction=None, connection=None):
     if connection is None:
         connection = conn()
 
-    query = (
-        "SELECT * FROM information_schema.processlist WHERE id <> CONNECTION_ID()"
-        + ("" if restriction is None else " AND (%s)" % restriction)
+    query = "SELECT * FROM information_schema.processlist WHERE id <> CONNECTION_ID()" + (
+        "" if restriction is None else " AND (%s)" % restriction
     )
 
-    cur = (
-        {k.lower(): v for k, v in elem.items()}
-        for elem in connection.query(query, as_dict=True)
-    )
+    cur = ({k.lower(): v for k, v in elem.items()} for elem in connection.query(query, as_dict=True))
     nkill = 0
     for process in cur:
         connection.query("kill %d" % process["id"])

src/datajoint/admin.py

@@ -45,20 +45,14 @@ def get_adapter(context, adapter_name):
     try:
         adapter = context[adapter_name]
     except KeyError:
-        raise DataJointError(
-            "Attribute adapter '{adapter_name}' is not defined.".format(
-                adapter_name=adapter_name
-            )
-        )
+        raise DataJointError("Attribute adapter '{adapter_name}' is not defined.".format(adapter_name=adapter_name))
     if not isinstance(adapter, AttributeAdapter):
         raise DataJointError(
             "Attribute adapter '{adapter_name}' must be an instance of datajoint.AttributeAdapter".format(
                 adapter_name=adapter_name
             )
         )
-    if not isinstance(adapter.attribute_type, str) or not re.match(
-        r"^\w", adapter.attribute_type
-    ):
+    if not isinstance(adapter.attribute_type, str) or not re.match(r"^\w", adapter.attribute_type):
         raise DataJointError(
             "Invalid attribute type {type} in attribute adapter '{adapter_name}'".format(
                 type=adapter.attribute_type, adapter_name=adapter_name