veriflow.configuration.utils#

A module for frequently used config elements in the context of verification.

Classes

BaseZarrConfig(*, path[, group, ...])

Configuration for connecting to a single Zarr store.

LeadTimes(*, unit, values)

A lead times config element.

LocalFile(*, directory, filename)

Configuration pointing to a local file.

LocalFiles(*, directory, filename_glob)

Config for multiple local files using pathlib.Path.glob.

Range(*, start, end, step)

A range.

TimePeriod(*, start, end)

A time period config element.

VerificationPair(*, id, ...)

Configuration for a verification pair.

VerificationPeriod(*, start, end[, dimension])

Definition of the verification period.

class veriflow.configuration.utils.BaseZarrConfig(*, path, group=None, auth_config=None, storage_options=None, consolidated=None)[source]#

Configuration for connecting to a single Zarr store.

The store may live on the local filesystem or on remote object storage (e.g. S3), and is read via xr.open_zarr / written via xr.Dataset.to_zarr / xr.DataTree.to_zarr. This is a shared base: it’s used directly for the veriflow cache (ZarrCacheConfig), for reading Zarr datasources (ZarrConfig), and for writing Zarr datasinks (CFCompliantZarrConfig).

Parameters:
  • path (Annotated[str, MinLen(min_length=1)])

  • group (str | None)

  • auth_config (S3AuthConfig | None)

  • storage_options (dict[str, str] | None)

  • consolidated (bool | None)

path: Annotated[str, FieldInfo(annotation=NoneType, required=True, description="Path to a single Zarr store. Local filesystem path (absolute or relative) or a remote URL such as 's3://bucket/key/store.zarr'.", metadata=[MinLen(min_length=1)])]#
group: Annotated[str | None, FieldInfo(annotation=NoneType, required=False, default=None, description="Group within the Zarr store to open. Corresponds to the 'group' parameter in xr.open_zarr.")]#
auth_config: Annotated[S3AuthConfig | None, FieldInfo(annotation=NoneType, required=False, default=None, description="Authentication configuration for remote stores. Only consulted when 'path' points to an 's3://' location. When the path is remote and this is left unset, credentials are loaded automatically from S3_-prefixed environment variables, so configuring 'auth_config: {}' in YAML is not required.")]#
storage_options: Annotated[dict[str, str] | None, FieldInfo(annotation=NoneType, required=False, default=None, description="Additional storage_options forwarded to xr.open_zarr. Merged on top of the options derived from 'auth_config'. Use this for advanced fsspec / s3fs settings not exposed by S3AuthConfig.")]#
consolidated: Annotated[bool | None, FieldInfo(annotation=NoneType, required=False, default=None, description="Whether to use consolidated metadata when opening the store. Forwarded to xr.open_zarr. Default ('None') lets xarray auto-detect.")]#
is_remote_path()[source]#

Return True if path looks like a remote/fsspec URL (e.g. s3://).

Return type:

bool

property resolved_storage_options: dict[str, object] | None#

Build the final storage_options for xr.open_zarr/to_zarr.

Merges ‘auth_config’ (auto-populated from S3_-prefixed environment variables for remote paths) with the explicit ‘storage_options’ override. Returns None for local paths so that xarray opens the store directly from the local filesystem. This is the single place that should ever build storage_options from a BaseZarrConfig - duplicating this logic elsewhere risks silently dropping ‘auth_config’ (and thus falling back to boto3’s default credential discovery instead of the configured S3_-prefixed credentials).

validate_zarr_path_accessible()[source]#

Check that a local cache dir exists, or initialize S3 auth for remote paths.

Return type:

Self

class veriflow.configuration.utils.LeadTimes(*, unit, values)[source]#

A lead times config element.

Parameters:
unit: TimeUnits#
values: <lambda>, json_schema_input_type=PydanticUndefined)]#
classmethod convert_range_to_list(v)[source]#

Convert range to list.

Parameters:

v (Range | list[int])

Return type:

list[int]

property timedelta64: list[timedelta64]#

As numpy timedelta64.

property stdlib_timedelta: list[timedelta]#

As datetime timedelta.

property max: timedelta#

Get the maximum lead time.

property min: timedelta#

Get the minimum lead time.

class veriflow.configuration.utils.LocalFile(*, directory, filename)[source]#

Configuration pointing to a local file.

Parameters:
directory: Path#
filename: str#
class veriflow.configuration.utils.LocalFiles(*, directory, filename_glob)[source]#

Config for multiple local files using pathlib.Path.glob.

Parameters:
  • directory (str)

  • filename_glob (str)

directory: str#
filename_glob: Annotated[str, FieldInfo(annotation=NoneType, required=True, description="A valid filename glob, like '*.nc' for all netcdf files.")]#
property paths: Iterator[Path]#

Return all filepaths as Path objects in a deterministic sorted order.

class veriflow.configuration.utils.Range(*, start, end, step)[source]#

A range.

Parameters:
start: int#
end: int#
step: int#
to_list()[source]#

Convert to list.

Return type:

list[int]

class veriflow.configuration.utils.TimePeriod(*, start, end)[source]#

A time period config element.

Parameters:
start: Annotated[datetime, FieldInfo(annotation=NoneType, required=True, description='YYYY-MM-DD[T]HH:MM[:SS[.ffffff]][Z or [±]HH[:]MM], see: https://docs.pydantic.dev/2.0/usage/types/datetime/#validation-of-datetime-types')]#
end: Annotated[datetime, FieldInfo(annotation=NoneType, required=True, description='YYYY-MM-DD[T]HH:MM[:SS[.ffffff]][Z or [±]HH[:]MM], see: https://docs.pydantic.dev/2.0/usage/types/datetime/#validation-of-datetime-types')]#
classmethod to_utc_naive_numpy_datetime(v)[source]#

Convert timezone aware datetimes to naive UTC; leave naive untouched (assumed UTC).

Parameters:

v (datetime)

Return type:

datetime

property start_datetime64: datetime64#

Get start as numpy format.

property end_datetime64: datetime64#

Get start as numpy format.

as_formatted_string()[source]#

Get the time period as a formatted string.

Return type:

str

class veriflow.configuration.utils.VerificationPair(*, id, observations_source_id, simulations_source_id, variable)[source]#

Configuration for a verification pair.

The id uniquely identifies the verification pair. The observations_source_id and simulations_source_id fields reference the respective data sources and should match with the configured source_id in the datasource configuration. The variable field specifies the (internal; i.e. after id mapping) physical variable to be verified.

Parameters:
  • id (Annotated[str, MinLen(min_length=1)])

  • observations_source_id (Annotated[str, StringConstraints(strip_whitespace=None, to_upper=None, to_lower=None, strict=None, min_length=1, max_length=None, pattern=^[A-Za-z][A-Za-z0-9_]*$, ascii_only=None), MinLen(min_length=1)])

  • simulations_source_id (Annotated[str, StringConstraints(strip_whitespace=None, to_upper=None, to_lower=None, strict=None, min_length=1, max_length=None, pattern=^[A-Za-z][A-Za-z0-9_]*$, ascii_only=None), MinLen(min_length=1)])

  • variable (Annotated[str, MinLen(min_length=1)])

id: Annotated[str, FieldInfo(annotation=NoneType, required=True, description="Unique identifier for the verification pair. All strings are allowed, but 'input_data' is reserved. And can not be used.", metadata=[MinLen(min_length=1)])]#
observations_source_id: ')]#
simulations_source_id: ')]#
variable: Annotated[str, FieldInfo(annotation=NoneType, required=True, description='The variable name references a physical variable to be verified. Must match the variable definition in the datasource. IdMapping can be set in the general config, to map external variables to an internal definition. This is needed when you want to verify data from different sources, where the variable definition is not equal.', metadata=[MinLen(min_length=1)]), FieldInfo(annotation=NoneType, required=True, description='The (internal) physical variable to be verified.')]#
classmethod id_not_reserved(v)[source]#

Reject ‘input_data’, reserved as the DataTree node name for raw input datasets.

Parameters:

v (str)

Return type:

str

class veriflow.configuration.utils.VerificationPeriod(*, start, end, dimension='forecast_reference_time')[source]#

Definition of the verification period.

Parameters:
dimension: Annotated[Literal['forecast_reference_time', 'time'], FieldInfo(annotation=NoneType, required=True, description="The dimension along which the verification period is defined. Using 'forecast_reference_time' allows for a forecast-centric verification run, whereas 'time' allows for an observation-centric verification run.")]#
as_formatted_string()#

Get the time period as a formatted string.

Return type:

str

property end_datetime64: datetime64#

Get start as numpy format.

property start_datetime64: datetime64#

Get start as numpy format.

classmethod to_utc_naive_numpy_datetime(v)#

Convert timezone aware datetimes to naive UTC; leave naive untouched (assumed UTC).

Parameters:

v (datetime)

Return type:

datetime

start#
end#