Source code for pyml_datasets._types

"""Public types of the package: ``Dataset`` and ``DatasetInfo``."""

import re
from collections.abc import Iterator, Mapping
from dataclasses import asdict, dataclass, field
from datetime import date
from typing import Any, Final, Literal, get_args

import numpy as np
from numpy.typing import NDArray

LicenseId = Literal["CC0-1.0", "CC-BY-4.0"]
Task = Literal["classification", "regression"]

LICENSE_IDS: Final[tuple[str, ...]] = get_args(LicenseId)
TASKS: Final[tuple[str, ...]] = get_args(Task)

_NAME_PATTERN: Final = re.compile(r"[a-z][a-z0-9_]*")
_DOI_PATTERN: Final = re.compile(r"10\.\d{4,9}/\S+")


def _plain(value: Any) -> Any:
    """Convert tuples and mappings into JSON-friendly lists and dicts.

    Parameters
    ----------
    value : object
        A value produced by ``dataclasses.asdict``.

    Returns
    -------
    object
        The same value with tuples replaced by lists, recursively.
    """
    if isinstance(value, Mapping):
        return {str(key): _plain(item) for key, item in value.items()}
    if isinstance(value, tuple | list):
        return [_plain(item) for item in value]
    return value


[docs] @dataclass(frozen=True, kw_only=True) class DatasetInfo: """Provenance and license of a dataset. Every bundled dataset has exactly one ``DatasetInfo``. Everything that talks about a dataset's origin (the description stored in its file, the ``NOTICE.md`` file, the tables of the README and of the documentation) is generated from it, so a fact is written down only once. Invalid values are rejected when the object is created. Attributes ---------- name : str Short identifier, lowercase letters, digits and underscores. It is also the name of the data file (``<name>.npz``) and of the loader (``load_<name>``). title : str Human readable name of the dataset. task : {"classification", "regression"} Kind of the prediction problem. It decides the dtype of ``y``. summary : str One or two sentences about what the dataset contains. license : {"CC0-1.0", "CC-BY-4.0"} SPDX identifier of the license the data is redistributed under. Only licenses that allow redistribution with at most an attribution duty are accepted. license_url : str Address of the license text. source : str Name of the original publisher of the data. source_url : str Page of the original publisher that describes the dataset and its license. doi : str or None DOI of the dataset or of the release that was used, without the resolver prefix. creators : tuple of str The people or institutions that created the dataset, as the source names them. citation : str The citation the source asks users to give, copied as the source writes it. obtained_from : str Where the bytes in this repository really came from. It can differ from ``source`` when a copy of the dataset was used. changes : tuple of str Every change made to the data, as the attribution duty of the license requires. verified_on : str ISO date (``YYYY-MM-DD``) on which the license was last checked at the source. references : tuple of str, default=() Papers and records that describe the dataset or that the source asks to cite. categorical : mapping of str to tuple of str, default={} Categorical features. Each key is a feature name, each value the category names in the order of the integer codes stored in ``X`` (code ``0`` is the first name). has_missing : bool, default=False Whether ``X`` contains missing values. They are stored as ``NaN``. notes : tuple of str, default=() Warnings and remarks that users of the dataset should see. """ name: str title: str task: Task summary: str license: LicenseId license_url: str source: str source_url: str doi: str | None creators: tuple[str, ...] citation: str obtained_from: str changes: tuple[str, ...] verified_on: str references: tuple[str, ...] = () categorical: Mapping[str, tuple[str, ...]] = field(default_factory=dict) has_missing: bool = False notes: tuple[str, ...] = () def __post_init__(self) -> None: """Reject values that would make the provenance of the dataset unclear. Raises ------ ValueError If the name, the task, the license, a URL, the DOI or the verification date is malformed, or if the creators, the citation or the changes are empty. """ name = self.name if not _NAME_PATTERN.fullmatch(name): raise ValueError(f"{name!r} is not a valid dataset name.") if self.task not in TASKS: raise ValueError(f"{name}: task must be one of {TASKS}, got {self.task!r}.") if self.license not in LICENSE_IDS: raise ValueError(f"{name}: license must be one of {LICENSE_IDS}, got {self.license!r}.") for label, url in (("license_url", self.license_url), ("source_url", self.source_url)): if not url.startswith("https://"): raise ValueError(f"{name}: {label} must be an https URL, got {url!r}.") if self.doi is not None and not _DOI_PATTERN.fullmatch(self.doi): raise ValueError(f"{name}: {self.doi!r} is not a valid DOI.") if not self.creators: raise ValueError(f"{name}: at least one creator is required.") if not self.citation.strip(): raise ValueError(f"{name}: the citation must not be empty.") if not self.changes: raise ValueError(f"{name}: list the changes made to the data.") try: date.fromisoformat(self.verified_on) except ValueError as error: message = f"{name}: verified_on must be an ISO date, got {self.verified_on!r}." raise ValueError(message) from error for feature, categories in self.categorical.items(): if not categories: raise ValueError(f"{name}: categorical feature {feature!r} has no categories.")
[docs] def to_dict(self) -> dict[str, Any]: """Return the information as a dict of plain Python values. Returns ------- dict of str to object Strings, lists, dicts, booleans and ``None`` only, so the result can be written as JSON. """ return dict(_plain(asdict(self)))
[docs] @dataclass(frozen=True, eq=False) class Dataset: """Dataset with its data matrix, targets and metadata. Iterating over a ``Dataset`` yields ``X`` and ``y``, so ``X, y = load_iris()`` works as well as ``data = load_iris()``. Attributes ---------- X : ndarray of shape (n_samples, n_features) Data matrix. Missing values, if the dataset has any, are ``NaN``. y : ndarray of shape (n_samples,) or (n_samples, n_targets) Target values. feature_names : tuple of str Names of the columns of ``X``. target_names : tuple of str or None Class names for classification or target names for multi-output regression, ``None`` if the source does not provide them. description : str Description of the dataset: what it is, where it comes from, how to cite it, which license applies and which changes were made. info : DatasetInfo The structured form of the provenance and license information. """ X: NDArray[np.float64] y: NDArray[Any] feature_names: tuple[str, ...] target_names: tuple[str, ...] | None description: str info: DatasetInfo def __iter__(self) -> Iterator[NDArray[Any]]: """Yield ``X`` and ``y`` in this order.""" yield self.X yield self.y @property def has_missing(self) -> bool: """Whether ``X`` contains at least one missing value (``NaN``).""" return bool(np.isnan(self.X).any()) @property def categorical_features(self) -> tuple[int, ...]: """Column indices of the categorical features, stored as integer codes. The category names of a column are in ``info.categorical`` under the feature name. """ return tuple( index for index, name in enumerate(self.feature_names) if name in self.info.categorical )