Merge branch 'master' into enh/remove-numpy-pin

Author: lkstrp

.github/workflows/codeql.yml

@@ -0,0 +1,92 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+  schedule:
+    - cron: '19 21 * * 6'
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - language: python
+          build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        build-mode: ${{ matrix.build-mode }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    # If the analyze step fails for one of the languages you are analyzing with
+    # "We were unable to automatically build your code", modify the matrix above
+    # to set the build mode to "manual" for that language. Then modify this step
+    # to build your code.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - if: matrix.build-mode == 'manual'
+      shell: bash
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/test.yml

@@ -93,3 +93,35 @@ jobs:
     - name: Upload code coverage report
       if: matrix.os == 'ubuntu-latest'
       uses: codecov/codecov-action@v4
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+
+  check-types:
+    name: Check types
+    needs: [build]
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0 # Needed for setuptools_scm
+
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.12
+
+    - name: Download package
+      uses: actions/download-artifact@v4
+      with:
+        name: Packages
+        path: dist
+
+    - name: Install package and dependencies
+      run: |
+        python -m pip install uv
+        uv pip install --system "$(ls dist/*.whl)[dev]"
+
+    - name: Run type checker (mypy)
+      run: |
+        mypy .

.pre-commit-config.yaml

@@ -1,12 +1,15 @@
+ci:
+  autoupdate_schedule: monthly
+
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
     -   id: check-merge-conflict
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.7
+    rev: v0.7.2
     hooks:
     -   id: ruff
         args: [--fix]
@@ -32,9 +35,3 @@ repos:
     hooks:
     -   id: jupyter-notebook-cleanup
         exclude: examples/solve-on-remote.ipynb
--   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.11.1
-    hooks:
-    -   id: mypy
-        files: ^(linopy|test)/
-        additional_dependencies: [numpy, pandas, xarray, types-paramiko]

README.md

@@ -12,7 +12,7 @@
         **I**nteger\
         **N**on-linear\
         **O**ptimization in\
-        **P Y**thon
+        **PY**thon
 
 **linopy** is an open-source python package that facilitates **optimization** with **real world data**. It builds a bridge between data analysis packages like [xarray](https://github.com/pydata/xarray) & [pandas](https://pandas.pydata.org/) and problem solvers like [cbc](https://projects.coin-or.org/Cbc), [gurobi](https://www.gurobi.com/) (see the full list below). **Linopy** supports **Linear, Integer, Mixed-Integer and Quadratic Programming** while aiming to make linear programming in Python easy, highly-flexible and performant.
 

benchmark/notebooks/plot-benchmarks.py.ipynb

@@ -12,8 +12,8 @@
     "import seaborn as sns\n",
     "\n",
     "sns.set_theme(\"paper\", \"white\")\n",
-    "#plt.rc('text', usetex=True)\n",
-    "#plt.rc('font', family='sans-serif')"
+    "# plt.rc('text', usetex=True)\n",
+    "# plt.rc('font', family='sans-serif')"
    ]
   },
   {
@@ -24,8 +24,8 @@
    "outputs": [],
    "source": [
     "data = pd.read_csv(snakemake.input[0], index_col=0)\n",
-    "cols = ['Time', 'Memory']\n",
-    "df = data.melt(id_vars=data.columns.drop(cols), value_vars=cols, var_name='kind')"
+    "cols = [\"Time\", \"Memory\"]\n",
+    "df = data.melt(id_vars=data.columns.drop(cols), value_vars=cols, var_name=\"kind\")"
    ]
   },
   {
@@ -35,21 +35,31 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "if snakemake.wildcards[\"kind\"] == 'overhead':\n",
-    "    labels = ['Overhead time (s)', 'Overhead memory (MB)']\n",
+    "if snakemake.wildcards[\"kind\"] == \"overhead\":\n",
+    "    labels = [\"Overhead time (s)\", \"Overhead memory (MB)\"]\n",
     "else:\n",
-    "    labels = ['Time (s)', 'Memory (MB)']\n",
+    "    labels = [\"Time (s)\", \"Memory (MB)\"]\n",
     "\n",
-    "g = sns.FacetGrid(data=df, row=\"kind\", sharey=False, height=2., aspect=2)\n",
-    "g.map_dataframe(sns.lineplot, x=\"Number of Variables\", y='value', hue='API', style='API',\n",
-    "                   marker='.', legend='full', zorder=8)\n",
+    "g = sns.FacetGrid(data=df, row=\"kind\", sharey=False, height=2.0, aspect=2)\n",
+    "g.map_dataframe(\n",
+    "    sns.lineplot,\n",
+    "    x=\"Number of Variables\",\n",
+    "    y=\"value\",\n",
+    "    hue=\"API\",\n",
+    "    style=\"API\",\n",
+    "    marker=\".\",\n",
+    "    legend=\"full\",\n",
+    "    zorder=8,\n",
+    ")\n",
     "for ax, label in zip(g.axes.ravel(), labels):\n",
     "    ax.set_ylabel(label)\n",
     "    ax.set_title(\"\")\n",
-    "    ax.grid(axis='y', lw=0.2, color='grey', zorder=3, alpha=0.4)\n",
+    "    ax.grid(axis=\"y\", lw=0.2, color=\"grey\", zorder=3, alpha=0.4)\n",
     "g.fig.tight_layout()\n",
     "g.add_legend()\n",
-    "g.fig.savefig(snakemake.output.time_memory, bbox_inches='tight', pad_inches=0.1, dpi=300)"
+    "g.fig.savefig(\n",
+    "    snakemake.output.time_memory, bbox_inches=\"tight\", pad_inches=0.1, dpi=300\n",
+    ")"
    ]
   },
   {
@@ -59,25 +69,33 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "if snakemake.wildcards[\"kind\"] == 'overhead':\n",
-    "    label = 'Computational overhead [MBs]'\n",
+    "if snakemake.wildcards[\"kind\"] == \"overhead\":\n",
+    "    label = \"Computational overhead [MBs]\"\n",
     "else:\n",
-    "    label = 'Computational resource [MBs]'\n",
+    "    label = \"Computational resource [MBs]\"\n",
     "\n",
-    "df = data.assign(Resource = data[\"Time\"] * data[\"Memory\"])\n",
-    "cols = ['Resource']\n",
-    "df = df.melt(id_vars=df.columns.drop(cols), value_vars=cols, var_name='kind')\n",
+    "df = data.assign(Resource=data[\"Time\"] * data[\"Memory\"])\n",
+    "cols = [\"Resource\"]\n",
+    "df = df.melt(id_vars=df.columns.drop(cols), value_vars=cols, var_name=\"kind\")\n",
     "\n",
     "fig, ax = plt.subplots(figsize=(6, 3))\n",
-    "sns.lineplot(data=df, x=\"Number of Variables\", y='value', hue='API', style='API',\n",
-    "                   marker='.', legend='full', zorder=8)\n",
+    "sns.lineplot(\n",
+    "    data=df,\n",
+    "    x=\"Number of Variables\",\n",
+    "    y=\"value\",\n",
+    "    hue=\"API\",\n",
+    "    style=\"API\",\n",
+    "    marker=\".\",\n",
+    "    legend=\"full\",\n",
+    "    zorder=8,\n",
+    ")\n",
     "sns.despine()\n",
     "ax.set_ylabel(label)\n",
     "ax.set_title(\"\")\n",
-    "plt.ticklabel_format(axis='both', style='sci', scilimits=(3,3))\n",
-    "ax.grid(axis='y', lw=0.2, color='grey', zorder=3, alpha=0.4)\n",
+    "plt.ticklabel_format(axis=\"both\", style=\"sci\", scilimits=(3, 3))\n",
+    "ax.grid(axis=\"y\", lw=0.2, color=\"grey\", zorder=3, alpha=0.4)\n",
     "fig.tight_layout()\n",
-    "fig.savefig(snakemake.output.resource, bbox_inches='tight', pad_inches=0.1, dpi=300)"
+    "fig.savefig(snakemake.output.resource, bbox_inches=\"tight\", pad_inches=0.1, dpi=300)"
    ]
   }
  ],

codecov.yml

@@ -0,0 +1 @@
+comment: false

doc/release_notes.rst

@@ -4,6 +4,35 @@ Release Notes
 Upcoming Version
 ----------------
 
+* The method :meth:`model.to_file <linopy.model.Model.to_file>` now includes a progress argument to enable or disable the progress bar while writing.
+
+
+Version 0.4.2
+--------------
+
+* Fix the file handler to properly close the file when reading the sense from a problem file.
+
+Version 0.4.1
+--------------
+
+* Fix the `slice_size` argument in the `solve` function. The argument was not properly passed to the `to_file` function.
+* Fix the slicing of constraints in case the term dimension is larger than the leading constraint coordinate dimension.
+
+Version 0.4.0
+--------------
+
+* When writing out an LP file, large variables and constraints are now chunked to avoid memory issues. This is especially useful for large models with constraints with many terms. The chunk size can be set with the `slice_size` argument in the `solve` function.
+* Constraints which of the form `<= infinity` and `>= -infinity` are now automatically filtered out when solving. The `solve` function now has a new argument `sanitize_infinities` to control this feature. Default is set to `True`.
+* The representation of linopy objects with multiindexed coordinates was improved to be more readable.
+* Grouping expressions is now supported on dimensions called "group" and dimensions that have the same name as the grouping object.
+* Grouping dimensions which have multiindexed coordinates is now supported.
+* See full list of changes `here <https://github.com/PyPSA/linopy/releases/tag/v0.4.0>`__.
+
+Version 0.3.15
+--------------
+
+* The group dimension when grouping by a pandas dataframe is now always `group`. This fixes the case that the dataframe contains a column named `name`.
+
 Version 0.3.14
 --------------
 

examples/create-a-model-with-coordinates.ipynb

@@ -71,10 +71,14 @@
    "source": [
     "import pandas as pd\n",
     "\n",
-    "time = pd.Index(range(10), name='time')\n",
+    "time = pd.Index(range(10), name=\"time\")\n",
     "\n",
-    "x = m.add_variables(lower=0, coords=[time], name='x', )\n",
-    "y = m.add_variables(lower=0, coords=[time], name='y')"
+    "x = m.add_variables(\n",
+    "    lower=0,\n",
+    "    coords=[time],\n",
+    "    name=\"x\",\n",
+    ")\n",
+    "y = m.add_variables(lower=0, coords=[time], name=\"y\")"
    ]
   },
   {
@@ -97,7 +101,7 @@
    "source": [
     "factor = pd.Series(time, index=time)\n",
     "\n",
-    "3*x + 7*y >= 10*factor"
+    "3 * x + 7 * y >= 10 * factor"
    ]
   },
   {
@@ -115,8 +119,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "con1 = m.add_constraints(3*x + 7*y >= 10*factor, name='con1')\n",
-    "con2 = m.add_constraints(5*x + 2*y >= 3*factor, name='con2')\n",
+    "con1 = m.add_constraints(3 * x + 7 * y >= 10 * factor, name=\"con1\")\n",
+    "con2 = m.add_constraints(5 * x + 2 * y >= 3 * factor, name=\"con2\")\n",
     "m"
    ]
   },
@@ -135,7 +139,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "obj = (x + 2*y).sum()\n",
+    "obj = (x + 2 * y).sum()\n",
     "m.add_objective(obj)"
    ]
   },
@@ -164,7 +168,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "m.solution.to_dataframe().plot(grid=True, ylabel='Optimal Value');"
+    "m.solution.to_dataframe().plot(grid=True, ylabel=\"Optimal Value\");"
    ]
   },
   {