Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions esmvalcore/_main.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,15 @@
import sys
from importlib.metadata import entry_points
from pathlib import Path
from typing import TYPE_CHECKING

import fire

from esmvalcore.config._config import warn_if_old_extra_facets_exist

if TYPE_CHECKING:
from esmvalcore.config import Session

# set up logging
logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -80,7 +84,7 @@ def parse_resume(resume, recipe):
return resume


def process_recipe(recipe_file: Path, session):
def process_recipe(recipe_file: Path, session: Session) -> None:
"""Process recipe."""
import datetime
import shutil
Expand Down Expand Up @@ -318,7 +322,7 @@ def _copy_config_file(
in_file: Path,
out_file: Path,
overwrite: bool,
):
) -> None:
"""Copy a configuration file."""
import shutil

Expand Down Expand Up @@ -634,7 +638,7 @@ def _create_session_dir(session):
def _run(
self,
recipe: Path,
session,
session: Session,
cli_config_dir: Path | None,
) -> None:
"""Run `recipe` using `session`."""
Expand Down Expand Up @@ -697,7 +701,7 @@ def _clean_preproc(session):
shutil.rmtree(session.preproc_dir)

@staticmethod
def _get_recipe(recipe) -> Path:
def _get_recipe(recipe: str) -> Path:
from esmvalcore.config._diagnostics import DIAGNOSTICS

if not os.path.isfile(recipe):
Expand Down
8 changes: 4 additions & 4 deletions esmvalcore/_provenance.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ def __init__(
attributes: dict[str, Any] | None = None,
ancestors: Iterable[TrackedFile] | None = None,
prov_filename: str | None = None,
):
) -> None:
"""Create an instance of a file with provenance tracking.

Arguments
Expand Down Expand Up @@ -182,7 +182,7 @@ def attributes(self) -> dict[str, Any]:
return self._attributes

@attributes.setter
def attributes(self, value: dict[str, Any] | None):
def attributes(self, value: dict[str, Any] | None) -> None:
"""Set attributes describing the file."""
self._attributes = value

Expand All @@ -194,11 +194,11 @@ def __repr__(self) -> str:
"""Return representation string (e.g., used by ``pformat``)."""
return f"{self.__class__.__name__}: {self.filename}"

def __eq__(self, other) -> bool:
def __eq__(self, other: object) -> bool:
"""Check if `other` equals `self`."""
return hasattr(other, "filename") and self.filename == other.filename

def __lt__(self, other) -> bool:
def __lt__(self, other: object) -> bool:
"""Check if `other` should be sorted before `self`."""
return hasattr(other, "filename") and self.filename < other.filename

Expand Down
6 changes: 3 additions & 3 deletions esmvalcore/_recipe/check.py
Original file line number Diff line number Diff line change
Expand Up @@ -364,7 +364,7 @@ def _verify_span_value(span: str) -> None:
raise RecipeError(msg)


def _verify_groupby(groupby: Any) -> None:
def _verify_groupby(groupby: list[str]) -> None:
"""Raise error if groupby arguments cannot be verified."""
if not isinstance(groupby, list):
msg = (
Expand All @@ -375,7 +375,7 @@ def _verify_groupby(groupby: Any) -> None:
raise RecipeError(msg)


def _verify_keep_input_datasets(keep_input_datasets: Any) -> None:
def _verify_keep_input_datasets(keep_input_datasets: bool) -> None:
if not isinstance(keep_input_datasets, bool):
msg = (
f"Invalid value encountered for `keep_input_datasets`."
Expand All @@ -385,7 +385,7 @@ def _verify_keep_input_datasets(keep_input_datasets: Any) -> None:
raise RecipeError(msg)


def _verify_ignore_scalar_coords(ignore_scalar_coords: Any) -> None:
def _verify_ignore_scalar_coords(ignore_scalar_coords: bool) -> None:
if not isinstance(ignore_scalar_coords, bool):
msg = (
f"Invalid value encountered for `ignore_scalar_coords`."
Expand Down
5 changes: 3 additions & 2 deletions esmvalcore/_recipe/from_datasets.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import itertools
import logging
import re
from collections.abc import Iterable, Mapping, Sequence
from collections.abc import Iterable, Mapping
from functools import partial
from typing import TYPE_CHECKING, Any

Expand All @@ -16,6 +16,7 @@
from ._io import _load_recipe

if TYPE_CHECKING:
from collections.abc import Sequence
from pathlib import Path

from esmvalcore.dataset import Dataset
Expand Down Expand Up @@ -114,7 +115,7 @@ def _to_frozen(item):
return item


def _move_one_level_up(base: dict, level: str, target: str):
def _move_one_level_up(base: dict, level: str, target: str) -> None:
"""Move datasets one level up in the recipe."""
groups = base[level]
if not groups:
Expand Down
16 changes: 11 additions & 5 deletions esmvalcore/_recipe/recipe.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

from esmvalcore import __version__, esgf
from esmvalcore._provenance import get_recipe_provenance
from esmvalcore._task import BaseTask, DiagnosticTask, ResumeTask, TaskSet
from esmvalcore._task import DiagnosticTask, ResumeTask, TaskSet
from esmvalcore.config._config import TASKSEP
from esmvalcore.config._dask import validate_dask_config
from esmvalcore.config._diagnostics import TAGS
Expand Down Expand Up @@ -59,6 +59,9 @@
if TYPE_CHECKING:
from collections.abc import Iterable, Sequence

from prov.model import ProvEntity

from esmvalcore._task import BaseTask
from esmvalcore.config import Session
from esmvalcore.io.protocol import DataElement
from esmvalcore.typing import Facets
Expand Down Expand Up @@ -964,7 +967,10 @@ def _need_ncl(raw_diagnostics: dict[str, Diagnostic]) -> bool:
return True
return False

def _initialize_provenance(self, raw_documentation: dict[str, Any]):
def _initialize_provenance(
self,
raw_documentation: dict[str, Any],
) -> ProvEntity:
"""Initialize the recipe provenance."""
doc = deepcopy(raw_documentation)

Expand Down Expand Up @@ -1380,10 +1386,10 @@ def write_html_summary(self) -> None:
RecipeOutput,
)

output = self.get_output()

try:
output = RecipeOutput.from_core_recipe_output(output)
output = RecipeOutput.from_core_recipe_output(
self.get_output(),
)
except LookupError as error:
# See https://github.com/ESMValGroup/ESMValCore/issues/28
logger.warning("Could not write HTML report: %s", error)
Expand Down
3 changes: 2 additions & 1 deletion esmvalcore/_recipe/to_datasets.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from __future__ import annotations

import logging
from collections.abc import Iterable, Iterator, Sequence
from collections.abc import Iterable
from copy import deepcopy
from typing import TYPE_CHECKING, Any

Expand All @@ -23,6 +23,7 @@
from ._io import _load_recipe

if TYPE_CHECKING:
from collections.abc import Iterator, Sequence
from pathlib import Path

from esmvalcore.config import Session
Expand Down
12 changes: 7 additions & 5 deletions esmvalcore/_task.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
"""ESMValtool task definition."""

from __future__ import annotations

import abc
import contextlib
import datetime
Expand Down Expand Up @@ -283,7 +285,7 @@ def flatten(self):
tasks.add(self)
return tasks

def run(self, input_files=None):
def run(self, input_files: list[str] | None = None) -> None:
"""Run task."""
if not self.output_files:
if input_files is None:
Expand All @@ -308,7 +310,7 @@ def run(self, input_files=None):
return self.output_files

@abc.abstractmethod
def _run(self, input_files):
def _run(self, input_files: list[str]) -> list[str]:
"""Run task."""

def get_product_attributes(self) -> dict:
Expand Down Expand Up @@ -362,7 +364,7 @@ def __init__(self, prev_preproc_dir, preproc_dir, name):

super().__init__(ancestors=None, name=name, products=products)

def _run(self, _):
def _run(self, _: list[str]) -> list[str]:
"""Return the result of a previous run."""
metadata = self.get_product_attributes()

Expand Down Expand Up @@ -810,11 +812,11 @@ def available_cpu_count() -> int:
class TaskSet(set):
"""Container for tasks."""

def flatten(self) -> "TaskSet":
def flatten(self) -> TaskSet:
"""Flatten the list of tasks."""
return TaskSet(t for task in self for t in task.flatten())

def get_independent(self) -> "TaskSet":
def get_independent(self) -> TaskSet:
"""Return a set of independent tasks."""
independent_tasks = TaskSet()
all_tasks = self.flatten()
Expand Down
7 changes: 6 additions & 1 deletion esmvalcore/cmor/_fixes/cmip5/ec_earth.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
"""Fixes for EC-Earth model."""

from collections.abc import Iterable
from __future__ import annotations

from typing import TYPE_CHECKING

import iris
import numpy as np
Expand All @@ -12,6 +14,9 @@
cube_to_aux_coord,
)

if TYPE_CHECKING:
from collections.abc import Iterable


class Sic(Fix):
"""Fixes for sic."""
Expand Down
7 changes: 6 additions & 1 deletion esmvalcore/cmor/_fixes/cmip6/cesm2.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
"""Fixes for CESM2 model."""

from pathlib import Path
from __future__ import annotations

from shutil import copyfile
from typing import TYPE_CHECKING

import iris
import iris.coords
Expand All @@ -18,6 +20,9 @@
fix_ocean_depth_coord,
)

if TYPE_CHECKING:
from pathlib import Path


class Cl(Fix):
"""Fixes for ``cl``."""
Expand Down
7 changes: 6 additions & 1 deletion esmvalcore/cmor/_fixes/cmip6/e3sm_1_1.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,15 @@
"""Fixes for E3SM-1-1 model."""

from iris.cube import Cube
from __future__ import annotations

from typing import TYPE_CHECKING

from esmvalcore.cmor.fix import Fix
from esmvalcore.preprocessor._shared import get_array_module

if TYPE_CHECKING:
from iris.cube import Cube


def _mask_greater(cube: Cube, value: float) -> Cube:
"""Mask all data of cube which is greater than ``value``."""
Expand Down
9 changes: 7 additions & 2 deletions esmvalcore/cmor/_fixes/cmip6/ec_earth3_veg_lr.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,17 @@
"""Fixes for EC-Earth3-Veg-LR model."""

from collections.abc import Sequence
from __future__ import annotations

import iris.cube
from typing import TYPE_CHECKING

from esmvalcore.cmor._fixes.common import OceanFixGrid
from esmvalcore.cmor._fixes.fix import Fix

if TYPE_CHECKING:
from collections.abc import Sequence

import iris.cube


class AllVars(Fix):
"""Fixes for all variables."""
Expand Down
12 changes: 7 additions & 5 deletions esmvalcore/cmor/_fixes/fix.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,13 @@
import logging
import tempfile
from pathlib import Path
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any

import dask
import numpy as np
from cf_units import Unit
from iris.coords import Coord, CoordExtent
from iris.cube import Cube, CubeList
from iris.coords import CoordExtent
from iris.cube import CubeList
from iris.exceptions import UnitConversionError
from iris.util import reverse

Expand All @@ -37,6 +37,8 @@

import ncdata
import xarray as xr
from iris.coords import Coord
from iris.cube import Cube

from esmvalcore.cmor.table import CoordinateInfo, VariableInfo
from esmvalcore.config import Session
Expand Down Expand Up @@ -421,12 +423,12 @@ def _msg_suffix(cube: Cube) -> str:
return f"\n(for file {cube.attributes['source_file']})"
return f"\n(for variable {cube.var_name})"

def _debug_msg(self, cube: Cube, msg: str, *args) -> None:
def _debug_msg(self, cube: Cube, msg: str, *args: Any) -> None:
"""Print debug message."""
msg += self._msg_suffix(cube)
generic_fix_logger.debug(msg, *args)

def _warning_msg(self, cube: Cube, msg: str, *args) -> None:
def _warning_msg(self, cube: Cube, msg: str, *args: Any) -> None:
"""Print debug message."""
msg += self._msg_suffix(cube)
generic_fix_logger.warning(msg, *args)
Expand Down
10 changes: 7 additions & 3 deletions esmvalcore/cmor/_fixes/icon/_base_fixes.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from pathlib import Path
from shutil import copyfileobj
from tempfile import NamedTemporaryFile
from typing import Any
from typing import TYPE_CHECKING, Any
from urllib.parse import urlparse

import iris
Expand All @@ -19,15 +19,19 @@
import requests
from cf_units import Unit
from iris import NameConstraint
from iris.coords import AuxCoord, Coord, DimCoord
from iris.cube import Cube, CubeList
from iris.coords import AuxCoord, DimCoord
from iris.cube import CubeList
from iris.mesh import Connectivity, MeshXY

import esmvalcore.local
from esmvalcore.cmor._fixes.native_datasets import NativeDatasetFix
from esmvalcore.config._data_sources import _get_data_sources
from esmvalcore.iris_helpers import add_leading_dim_to_cube, date2num

if TYPE_CHECKING:
from iris.coords import Coord
from iris.cube import Cube

logger = logging.getLogger(__name__)


Expand Down
Loading