data_meta_config.py 1.7 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344
  1. # Copyright (c) Alibaba, Inc. and its affiliates.
  2. class DataMetaConfig(object):
  3. """Modelscope data-meta config class.
  4. Attributes:
  5. dataset_scripts(str): The local path of dataset scripts.
  6. dataset_formation(:obj:`enum.Enum`): Dataset formation, refer to modelscope.utils.constant.DatasetFormations.
  7. meta_cache_dir(str): Meta cache path.
  8. meta_data_files(dict): Meta data mapping, Example: {'test': 'https://xxx/mytest.csv'}
  9. zip_data_files(dict): Data files mapping, Example: {'test': 'pictures.zip'}
  10. meta_args_map(dict): Meta arguments mapping, Example: {'test': {'file': 'pictures.zip'}, ...}
  11. target_dataset_structure(dict): Dataset Structure, like
  12. {
  13. "default":{
  14. "train":{
  15. "meta":"my_train.csv",
  16. "file":"pictures.zip"
  17. }
  18. },
  19. "subsetA":{
  20. "test":{
  21. "meta":"mytest.csv",
  22. "file":"pictures.zip"
  23. }
  24. }
  25. }
  26. dataset_py_script(str): The python script path of dataset.
  27. meta_type_map(dict): The custom dataset mapping in meta data,
  28. Example: {"type": "MovieSceneSegmentationCustomDataset",
  29. "preprocessor": "movie-scene-segmentation-preprocessor"}
  30. """
  31. def __init__(self):
  32. self.dataset_scripts = None
  33. self.dataset_formation = None
  34. self.meta_cache_dir = None
  35. self.meta_data_files = None
  36. self.zip_data_files = None
  37. self.meta_args_map = None
  38. self.target_dataset_structure = None
  39. self.dataset_py_script = None
  40. self.meta_type_map = {}