File size: 441 Bytes
e0fcef4 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 |
"""
Data processing modules
"""
from axolotl.utils.data.dpo import load_prepare_dpo_datasets # noqa: F401
from axolotl.utils.data.pretraining import ( # noqa: F401
encode_pretraining,
wrap_pretraining_dataset,
)
from axolotl.utils.data.sft import ( # noqa: F401
get_dataset_wrapper,
load_prepare_datasets,
load_tokenized_prepared_datasets,
prepare_dataset,
)
from axolotl.utils.data.utils import md5 # noqa: F401
|