Source code for static_frame.core.store_config

from __future__ import annotations

import dataclasses
from collections.abc import Iterable, Mapping
from typing import ClassVar

import typing_extensions as tp

from static_frame.core.exception import (
    ErrorInitStoreConfig,
)
from static_frame.core.frame import Frame
from static_frame.core.store_filter import STORE_FILTER_DEFAULT
from static_frame.core.util import DTYPE_STR_KINDS

if tp.TYPE_CHECKING:
    from static_frame.core.generic_aliases import TFrameAny
    from static_frame.core.store_filter import StoreFilter
    from static_frame.core.util import (
        TDepthLevel,
        TDtypeSpecifier,
        TDtypesSpecifier,
        TIndexCtor,
        TIndexCtorSpecifiers,
        TLabel,
        TMpContext,
    )


def label_encode_tuple(source: tuple[tp.Any, ...]) -> str:
    """For encoding tuples of NumPy scalars in strings that can use literal_eval to re-evaluate"""
    parts = []
    for obj in source:
        if dt := getattr(obj, 'dtype', None):  # a NumPy scalar
            if dt.kind in DTYPE_STR_KINDS:
                parts.append(f"'{obj}'")
            else:  # str, not repr, must be used
                parts.append(str(obj))
        elif isinstance(obj, str):
            parts.append(repr(obj))
        else:
            parts.append(str(obj))
    return f'({", ".join(parts)})'


def _depth_specifier_pickleable(
    depth_specifier: TDepthLevel | None,
) -> int | tuple[int, ...] | None:
    if isinstance(depth_specifier, Iterable):
        return tuple(depth_specifier)

    return depth_specifier


def _index_ctor_specifiers_pickleable(
    ctor_specifier: TIndexCtorSpecifiers | None,
) -> TIndexCtor | tuple[TIndexCtor, ...] | None:
    if isinstance(ctor_specifier, Iterable):
        return tuple(ctor_specifier)  # pyright: ignore

    return ctor_specifier  # type: ignore


def _dtypes_specifier_pickleable(
    dtypes_specifier: TDtypesSpecifier,
) -> None | TDtypeSpecifier | tuple[TDtypeSpecifier, ...] | dict[TLabel, TDtypeSpecifier]:
    if dtypes_specifier is None:
        return dtypes_specifier

    if isinstance(dtypes_specifier, dict):
        return dtypes_specifier

    if isinstance(dtypes_specifier, Iterable):
        return tuple(dtypes_specifier)

    return dtypes_specifier


_ATTR_TO_PICKLEABLE: dict[str, tp.Callable[[tp.Any], tp.Any]] = dict(
    index_name_depth_level=_depth_specifier_pickleable,
    index_constructors=_index_ctor_specifiers_pickleable,
    columns_name_depth_level=_depth_specifier_pickleable,
    columns_constructors=_index_ctor_specifiers_pickleable,
    columns_select=lambda v: None if v is None else tuple(v),
    dtypes=_dtypes_specifier_pickleable,
)


[docs] @dataclasses.dataclass(frozen=True, kw_only=True) class StoreConfigBase: """ A read-only container used by :obj:`Store` subclasses for reading from and writing to multi-table storage formats. This base class contains arguments common to all storage formats. """ label_encoder: tp.Callable[[TLabel], str] | None = None label_decoder: tp.Callable[[str], TLabel] | None = None read_frame_filter: tp.Callable[[TLabel, TFrameAny], TFrameAny] | None = None read_max_workers: int | None = None read_chunksize: int = 1 write_max_workers: int | None = None write_chunksize: int = 1 mp_context: TMpContext | None = None _CONSTRUCTOR: ClassVar[tp.Callable[..., TFrameAny]]
[docs] def to_frame_ctor_config(self) -> tp.Self: # All frame-construction-relevant fields are defined in subclasses. to_replace = dict.fromkeys( field.name for field in dataclasses.fields(StoreConfigBase) ) undefined = object() for attr, handler in _ATTR_TO_PICKLEABLE.items(): if (value := getattr(self, attr, undefined)) is not undefined: to_replace[attr] = handler(value) return dataclasses.replace(self, **to_replace) # type: ignore
[docs] def label_encode(self, label: TLabel) -> str: if self.label_encoder is str and isinstance(label, tuple): # with NumPy2, str() of a tuple of NumPy scalars returns (np.str_('a'), np.int64(1)), not ('a', 1) label = label_encode_tuple(label) elif self.label_encoder: label = self.label_encoder(label) if not isinstance(label, str): raise RuntimeError( f'Store label {label!r} is not a string; provide a label_encoder to {self.__class__.__name__}.' ) return label
[docs] def label_decode(self, label: str) -> TLabel: if self.label_decoder: return self.label_decoder(label) return label
@dataclasses.dataclass(frozen=True, kw_only=True) class StoreConfigFromFrameMixin: # Constructors index_depth: int = 0 # this default does not permit round trip columns_depth: int = 1 # Exporters include_index: bool = True include_index_name: bool = True include_columns: bool = True include_columns_name: bool = False @classmethod def from_frame(cls, frame: TFrameAny) -> tp.Self: """Derive a config from a Frame.""" include_index = frame.index.depth > 1 or frame.index._map is not None # type: ignore include_columns = frame.columns.depth > 1 or frame.columns._map is not None # type: ignore index_depth = 0 if not include_index else frame.index.depth columns_depth = 0 if not include_columns else frame.columns.depth return cls( index_depth=index_depth, columns_depth=columns_depth, include_index=include_index, include_columns=include_columns, ) @dataclasses.dataclass(frozen=True, kw_only=True) class StoreConfigDelimited(StoreConfigFromFrameMixin, StoreConfigBase): # Constructors index_name_depth_level: TDepthLevel | None = None index_constructors: TIndexCtorSpecifiers = None columns_name_depth_level: TDepthLevel | None = None columns_constructors: TIndexCtorSpecifiers = None dtypes: TDtypesSpecifier = None consolidate_blocks: bool = False # Both store_filter: StoreFilter | None = STORE_FILTER_DEFAULT @dataclasses.dataclass(frozen=True, kw_only=True) class StoreConfigTSV(StoreConfigDelimited): _CONSTRUCTOR = Frame.from_tsv @dataclasses.dataclass(frozen=True, kw_only=True) class StoreConfigCSV(StoreConfigDelimited): _CONSTRUCTOR = Frame.from_csv @dataclasses.dataclass(frozen=True, kw_only=True) class StoreConfigPickle(StoreConfigBase): _CONSTRUCTOR = Frame.from_pickle @dataclasses.dataclass(frozen=True, kw_only=True) class StoreConfigNPZ(StoreConfigBase): # Exporters include_index: bool = True include_columns: bool = True consolidate_blocks: bool = False _CONSTRUCTOR = Frame.from_npz @dataclasses.dataclass(frozen=True, kw_only=True) class StoreConfigParquet(StoreConfigFromFrameMixin, StoreConfigBase): # Constructors index_name_depth_level: TDepthLevel | None = None index_constructors: TIndexCtorSpecifiers = None columns_name_depth_level: TDepthLevel | None = None columns_constructors: TIndexCtorSpecifiers = None columns_select: tp.Iterable[str] | None = None dtypes: TDtypesSpecifier = None consolidate_blocks: bool = False _CONSTRUCTOR = Frame.from_parquet @dataclasses.dataclass(frozen=True, kw_only=True) class StoreConfigNPY(StoreConfigBase): # Exporters include_index: bool = True include_columns: bool = True consolidate_blocks: bool = False @dataclasses.dataclass(frozen=True, kw_only=True) class StoreConfigXLSX(StoreConfigFromFrameMixin, StoreConfigBase): # Constructors index_name_depth_level: TDepthLevel | None = None index_constructors: TIndexCtorSpecifiers = None columns_name_depth_level: TDepthLevel | None = None columns_constructors: TIndexCtorSpecifiers = None columns_select: tp.Iterable[str] | None = None dtypes: TDtypesSpecifier = None consolidate_blocks: bool = False skip_header: int = 0 skip_footer: int = 0 trim_nadir: bool = False # Exporters merge_hierarchical_labels: bool = True # Both store_filter: StoreFilter | None = STORE_FILTER_DEFAULT _CONSTRUCTOR = Frame.from_xlsx @dataclasses.dataclass(frozen=True, kw_only=True) class StoreConfigSQLite(StoreConfigFromFrameMixin, StoreConfigBase): # Constructors index_constructors: TIndexCtorSpecifiers = None columns_select: tp.Iterable[str] | None = None columns_constructors: TIndexCtorSpecifiers = None dtypes: TDtypesSpecifier = None consolidate_blocks: bool = False _CONSTRUCTOR = Frame.from_sqlite TVStoreConfig = tp.TypeVar( 'TVStoreConfig', bound=StoreConfigBase, default=StoreConfigBase ) class FromFrameProtocol(tp.Protocol[TVStoreConfig]): @classmethod def from_frame(cls, frame: TFrameAny) -> TVStoreConfig: ... @tp.final class StoreConfigMap(tp.Generic[TVStoreConfig]): """ Immutable defaultdict-like mapping to StoreConfigs. """ __slots__ = ( '_map', '_default', ) _ALIGN_WITH_DEFAULT_ATTRS = tuple( field.name for field in dataclasses.fields(StoreConfigBase) ) @staticmethod def from_frames( frames: tp.Iterable[TFrameAny], *, config_class: FromFrameProtocol[TVStoreConfig] ) -> StoreConfigMap[TVStoreConfig]: config_map: dict[tp.Any, TVStoreConfig] = {} for f in frames: config_map[f.name] = config_class.from_frame(f) return StoreConfigMap[TVStoreConfig](config_map=config_map) @staticmethod def from_initializer( initializer: TVStoreConfigMapInitializer[TVStoreConfig], ) -> StoreConfigMap[TVStoreConfig]: if initializer is None: # NOTE: This branch will fail! To reduce repeated code, the failure happens in init return StoreConfigMap[TVStoreConfig]() if isinstance(initializer, StoreConfigBase): return StoreConfigMap[TVStoreConfig](default=initializer) if isinstance(initializer, StoreConfigMap): return initializer if not isinstance(initializer, Mapping): raise ErrorInitStoreConfig( f'Unsupported initializer type: {type(initializer)}' ) return StoreConfigMap[TVStoreConfig](config_map=initializer) def __init__( self, config_map: tp.Mapping[tp.Any, TVStoreConfig] | None = None, *, default: TVStoreConfig | None = None, ) -> None: self._map: tp.Mapping[tp.Any, TVStoreConfig] = {} if config_map: config_types: set[type[TVStoreConfig]] = set(map(type, config_map.values())) if len(config_types) == 1: config_type = config_types.pop() if not isinstance(config_type, type) or not issubclass( config_type, StoreConfigBase ): raise ErrorInitStoreConfig( 'Mapping config values must be subclasses of StoreConfigBase!' ) # Better default! if default is None: default = config_type() elif not isinstance(default, config_type): raise ErrorInitStoreConfig( 'Default & Mapping config classes must be the same!' ) else: raise ErrorInitStoreConfig('Multiple config types present!') for label, config in config_map.items(): for attr in self._ALIGN_WITH_DEFAULT_ATTRS: if getattr(config, attr) != getattr(default, attr): raise ErrorInitStoreConfig( f'config {label!r} has {attr} inconsistent with default; align values and/or pass a default StoreConfigBase.' ) self._map[label] = config if default is None: raise ErrorInitStoreConfig( "Disallowed construction; cannot infer StoreConfigMap's default type from None. " 'To get a default, simply construct the StoreConfigBase subclass without args (e.g. StoreConfigCSV())' ) if not isinstance(default, StoreConfigBase): raise ErrorInitStoreConfig( 'Default config must be a StoreConfigBase instance!' ) self._default = default def __getitem__(self, key: TLabel | None) -> TVStoreConfig: return self._map.get(key, self._default) @property def default(self) -> TVStoreConfig: return self._default TVStoreConfigMapInitializer = tp.TypeAliasType( 'TVStoreConfigMapInitializer', TVStoreConfig | StoreConfigMap[TVStoreConfig] | None | tp.Mapping[tp.Any, TVStoreConfig], type_params=(TVStoreConfig,), )