fix: remove packing from func def

Signed-off-by: Will Johnson <mwjohnson728@gmail.com>
foundation-model-stack · Jan 23, 2025 · b72003d · b72003d
1 parent 3107fbf
commit b72003d
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/tuning/data/setup_dataprocessor.py b/tuning/data/setup_dataprocessor.py
@@ -74,7 +74,7 @@ def _process_dataconfig_file(
 
 
 # Data Format 1: Pretokenized Data
-def _get_pretokenized_dataset_handlers(data_args, packing, is_eval_tokenized):
+def _get_pretokenized_dataset_handlers(data_args, is_eval_tokenized):
 
     # if the provided train dataset is pretokenized
     # however user provides formatting flags, error out
@@ -258,7 +258,7 @@ def _process_raw_data_args(
     if is_traindata_tokenized:
         # Data Format 1: Pretokenized Data
         handlers, dataset_text_field = _get_pretokenized_dataset_handlers(
-            data_args, packing, (is_eval_dataset_present and not is_evaldata_tokenized)
+            data_args, (is_eval_dataset_present and not is_evaldata_tokenized)
         )
     elif data_args.instruction_template and data_args.response_template:
         # Data Format 2: Chat dataset with instruction and response template