Skip to content

omnipy.components.pandas.serializers

Tar-file serializer for pandas-backed Omnipy datasets.

CLASS DESCRIPTION
PandasDatasetToTarFileSerializer

Serialize pandas datasets to and from gzipped tar archives.

PandasDatasetToTarFileSerializer

Bases: TarFileSerializer[PandasDataset]


              flowchart BT
              omnipy.components.pandas.serializers.PandasDatasetToTarFileSerializer[PandasDatasetToTarFileSerializer]
              omnipy.data.serializer.TarFileSerializer[TarFileSerializer]
              omnipy.data.serializer.Serializer[Serializer]

                              omnipy.data.serializer.TarFileSerializer --> omnipy.components.pandas.serializers.PandasDatasetToTarFileSerializer
                                omnipy.data.serializer.Serializer --> omnipy.data.serializer.TarFileSerializer
                



              click omnipy.components.pandas.serializers.PandasDatasetToTarFileSerializer href "" "omnipy.components.pandas.serializers.PandasDatasetToTarFileSerializer"
              click omnipy.data.serializer.TarFileSerializer href "" "omnipy.data.serializer.TarFileSerializer"
              click omnipy.data.serializer.Serializer href "" "omnipy.data.serializer.Serializer"

Serialize pandas datasets to and from gzipped tar archives.

METHOD DESCRIPTION
deserialize

Deserialize a gzipped tar archive back into a dataset.
get_dataset_cls_for_new

Return the dataset class created during deserialization.
get_output_file_suffix

Return the file suffix used for serialized dataset members.
is_dataset_directly_supported

Return whether a dataset is a pandas dataset.
serialize

Serialize a dataset into a gzipped tar archive.
Source code in src/omnipy/components/pandas/serializers.py 
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
class PandasDatasetToTarFileSerializer(TarFileSerializer[PandasDataset]):
    """Serialize pandas datasets to and from gzipped tar archives."""
    @classmethod
    def is_dataset_directly_supported(cls, dataset: IsDataset) -> bool:
        """Return whether a dataset is a pandas dataset."""

        return isinstance(dataset, PandasDataset)

    @classmethod
    def get_dataset_cls_for_new(cls) -> Type[IsDataset]:
        # %% Original docstring (managed by expand_docstr_macros.py) %%
        # {{SERIALIZER_GET_DATASET_CLS_FOR_NEW_SUMMARY}}
        """Return the dataset class created during deserialization."""

        return PandasDataset

    @classmethod
    def get_output_file_suffix(cls) -> str:
        # %% Original docstring (managed by expand_docstr_macros.py) %%
        # {{SERIALIZER_GET_OUTPUT_FILE_SUFFIX_SUMMARY}}
        """Return the file suffix used for serialized dataset members."""

        return 'csv'

    @classmethod
    def serialize(cls, dataset: PandasDataset) -> bytes | memoryview:
        # %% Original docstring (managed by expand_docstr_macros.py) %%
        # {{SERIALIZE_GZIPPED_TAR_SUMMARY}}
        """Serialize a dataset into a gzipped tar archive."""

        assert isinstance(dataset, PandasDataset)

        def pandas_encode_func(pandas_data: 'pd.DataFrame') -> memoryview:
            csv_bytes = BytesIO()
            pandas_data.to_csv(csv_bytes, encoding='utf8', mode='wb', index=False)
            return csv_bytes.getbuffer()

        return cls.create_tarfile_from_dataset(dataset, data_encode_func=pandas_encode_func)

    @classmethod
    def deserialize(cls, serialized: bytes, any_file_suffix=False) -> PandasDataset:
        # %% Original docstring (managed by expand_docstr_macros.py) %%
        # {{DESERIALIZE_GZIPPED_TAR_SUMMARY}}
        """Deserialize a gzipped tar archive back into a dataset."""

        pandas_dataset = PandasDataset()

        def csv_decode_func(file_stream: IO[bytes]) -> 'pd.DataFrame':
            from .lazy_import import pd
            return pd.read_csv(file_stream, encoding='utf8')

        def python_dictify_object(data_file: str, obj_val: Any) -> dict:
            return {data_file: obj_val}

        cls.create_dataset_from_tarfile(
            pandas_dataset,
            serialized,
            data_decode_func=csv_decode_func,
            dictify_object_func=python_dictify_object,
            import_method='from_data',
            any_file_suffix=any_file_suffix,
        )  # noqa

        return pandas_dataset

deserialize classmethod 

deserialize(serialized: bytes, any_file_suffix=False) -> PandasDataset

Deserialize a gzipped tar archive back into a dataset.

Source code in src/omnipy/components/pandas/serializers.py 
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
@classmethod
def deserialize(cls, serialized: bytes, any_file_suffix=False) -> PandasDataset:
    # %% Original docstring (managed by expand_docstr_macros.py) %%
    # {{DESERIALIZE_GZIPPED_TAR_SUMMARY}}
    """Deserialize a gzipped tar archive back into a dataset."""

    pandas_dataset = PandasDataset()

    def csv_decode_func(file_stream: IO[bytes]) -> 'pd.DataFrame':
        from .lazy_import import pd
        return pd.read_csv(file_stream, encoding='utf8')

    def python_dictify_object(data_file: str, obj_val: Any) -> dict:
        return {data_file: obj_val}

    cls.create_dataset_from_tarfile(
        pandas_dataset,
        serialized,
        data_decode_func=csv_decode_func,
        dictify_object_func=python_dictify_object,
        import_method='from_data',
        any_file_suffix=any_file_suffix,
    )  # noqa

    return pandas_dataset

get_dataset_cls_for_new classmethod 

get_dataset_cls_for_new() -> Type[IsDataset]

Return the dataset class created during deserialization.

Source code in src/omnipy/components/pandas/serializers.py 
24
25
26
27
28
29
30
@classmethod
def get_dataset_cls_for_new(cls) -> Type[IsDataset]:
    # %% Original docstring (managed by expand_docstr_macros.py) %%
    # {{SERIALIZER_GET_DATASET_CLS_FOR_NEW_SUMMARY}}
    """Return the dataset class created during deserialization."""

    return PandasDataset

get_output_file_suffix classmethod 

get_output_file_suffix() -> str

Return the file suffix used for serialized dataset members.

Source code in src/omnipy/components/pandas/serializers.py 
32
33
34
35
36
37
38
@classmethod
def get_output_file_suffix(cls) -> str:
    # %% Original docstring (managed by expand_docstr_macros.py) %%
    # {{SERIALIZER_GET_OUTPUT_FILE_SUFFIX_SUMMARY}}
    """Return the file suffix used for serialized dataset members."""

    return 'csv'

is_dataset_directly_supported classmethod 

is_dataset_directly_supported(dataset: IsDataset) -> bool

Return whether a dataset is a pandas dataset.

Source code in src/omnipy/components/pandas/serializers.py 
18
19
20
21
22
@classmethod
def is_dataset_directly_supported(cls, dataset: IsDataset) -> bool:
    """Return whether a dataset is a pandas dataset."""

    return isinstance(dataset, PandasDataset)

serialize classmethod 

serialize(dataset: PandasDataset) -> bytes | memoryview

Serialize a dataset into a gzipped tar archive.

Source code in src/omnipy/components/pandas/serializers.py 
40
41
42
43
44
45
46
47
48
49
50
51
52
53
@classmethod
def serialize(cls, dataset: PandasDataset) -> bytes | memoryview:
    # %% Original docstring (managed by expand_docstr_macros.py) %%
    # {{SERIALIZE_GZIPPED_TAR_SUMMARY}}
    """Serialize a dataset into a gzipped tar archive."""

    assert isinstance(dataset, PandasDataset)

    def pandas_encode_func(pandas_data: 'pd.DataFrame') -> memoryview:
        csv_bytes = BytesIO()
        pandas_data.to_csv(csv_bytes, encoding='utf8', mode='wb', index=False)
        return csv_bytes.getbuffer()

    return cls.create_tarfile_from_dataset(dataset, data_encode_func=pandas_encode_func)