feat: delete interval_size in InputArgs.py

Author: shijinpjlab

dingo/io/input/InputArgs.py

@@ -24,7 +24,6 @@ class InputArgs(BaseModel):
     # Resume settings
     start_index: int = 0
     end_index: int = -1
-    interval_size: int = 1000
 
     # Concurrent settings
     max_workers: int = 1
@@ -89,10 +88,6 @@ def check_args(self):
         if self.end_index >= 0 and self.end_index < self.start_index:
             raise ValueError("if end_index is non negative, end_index must be greater than start_index")
 
-        # check interval size
-        if self.interval_size <= 0:
-            raise ValueError("interval_size must be positive.")
-
         # check max workers
         if self.max_workers <= 0:
             raise ValueError("max_workers must be a positive integer.")

dingo/run/cli.py

@@ -30,8 +30,6 @@ def parse_args():
                         default=None, help="The number of data start to check.")
     parser.add_argument("--end_index", type=int,
                         default=None, help="The number of data end to check.")
-    parser.add_argument("--interval_size", type=int,
-                        default=None, help="The number of size to save while checking.")
     parser.add_argument("--max_workers", type=int,
                         default=None, help="The number of max workers to concurrent check. ")
     parser.add_argument("--batch_size", type=int,
@@ -112,8 +110,6 @@ def parse_args():
             input_data['start_index'] = args.start_index
         if args.end_index:
             input_data['end_index'] = args.end_index
-        if args.interval_size:
-            input_data['interval_size'] = args.interval_size
         if args.max_workers:
             input_data['max_workers'] = args.max_workers
         if args.batch_size:

docs/config.md

@@ -17,7 +17,6 @@
 | --save_raw                | bool |              False               |    No    | whether save raw data.                                                                       |
 | --start_index             | int  |                0                 |    No    | the number of data start to check.                                                           |
 | --end_index               | int  |                -1                |    No    | the number of data end to check. if it's negative, include the data from start_index to end. |
-| --interval_size           | int  |               1000               |    No    | the number of size to save while checking.                                                   |
 | --max_workers             | int  |                1                 |    No    | the number of max workers to concurrent check.                                               |
 | --batch_size              | int  |                1                 |    No    | the number of max data for concurrent check.                                                 |
 | --dataset                 | str  |          "hugging_face"          |   Yes    | dataset type, in ['hugging_face', 'local']                                                   |
@@ -46,7 +45,6 @@
 | save_raw                | bool                  |              False               |    No    | whether save raw data.                                                                       |
 | start_index             | int                   |                0                 |    No    | the number of data start to check.                                                           |
 | end_index               | int                   |                -1                |    No    | the number of data end to check. if it's negative, include the data from start_index to end. |
-| interval_size           | int                   |               1000               |    No    | the number of size to save while checking.                                                   |
 | max_workers             | int                   |                1                 |    No    | the number of max workers to concurrent check.                                               |
 | batch_size              | int                   |                1                 |    No    | the number of max data for concurrent check.                                                 |
 | dataset                 | str                   |          "hugging_face"          |   Yes    | dataset type, in ['hugging_face', 'local']                                                   |