Oct 16, 2025 · Oct 16, 2025 · Oct 16, 2025 · Oct 16, 2025 · Oct 16, 2025 · Oct 16, 2025
diff --git a/open_instruct/dataset_transformation.py b/open_instruct/dataset_transformation.py
    row.pop(sft_messages_key)
    return row


 def rlvr_tokenize_v3(
    row: Dict[str, Any],
    tokenizer: PreTrainedTokenizer,
    return max_prompt_token_length_ok and max_token_length_ok and (contain_some_labels or not need_contain_labels)


 def rlvr_filter_v2(
    row: Dict[str, Any],
    tokenizer: PreTrainedTokenizer,
    max_prompt_token_length: Optional[int] = None,
 ):
 def rlvr_filter_v2(row: Dict[str, Any], tokenizer: PreTrainedTokenizer, max_prompt_token_length: Optional[int] = None):
    max_prompt_token_length_ok = True
    if max_prompt_token_length is not None:
        max_prompt_token_length_ok = len(row[INPUT_IDS_PROMPT_KEY]) <= max_prompt_token_length
diff --git a/open_instruct/grpo_fast.py b/open_instruct/grpo_fast.py

 # isort: on
 import asyncio
 import copy
 import json
 import logging
 import math

    """Set up training and evaluation datasets."""
    transform_fn_args = [
        {
            "system_prompt_override": system_prompt_override,
        },
        {"system_prompt_override": system_prompt_override},
        {"max_prompt_token_length": args.max_prompt_token_length},
    ]
    train_dataset = get_cached_dataset_tulu(
Original file line number	Diff line number	Diff line change
Expand Up		@@ -1235,6 +1235,7 @@ def rlvr_tokenize_v2(
		row.pop(sft_messages_key)
		return row


		def rlvr_tokenize_v3(
		row: Dict[str, Any],
		tokenizer: PreTrainedTokenizer,
Expand DownExpand Up		@@ -1296,11 +1297,7 @@ def rlvr_filter_v1(
		return max_prompt_token_length_ok and max_token_length_ok and (contain_some_labels or not need_contain_labels)


		def rlvr_filter_v2(
		row: Dict[str, Any],
		tokenizer: PreTrainedTokenizer,
		max_prompt_token_length: Optional[int] = None,
		):
		def rlvr_filter_v2(row: Dict[str, Any], tokenizer: PreTrainedTokenizer, max_prompt_token_length: Optional[int] = None):
		max_prompt_token_length_ok = True
		if max_prompt_token_length is not None:
		max_prompt_token_length_ok = len(row[INPUT_IDS_PROMPT_KEY]) <= max_prompt_token_length
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -48,7 +48,6 @@

		# isort: on
		import asyncio
		import copy
		import json
		import logging
		import math
Expand DownExpand Up		@@ -2174,9 +2173,7 @@ def setup_datasets(args: Args, tc: TokenizerConfig, tokenizer: PreTrainedTokeniz

		"""Set up training and evaluation datasets."""
		transform_fn_args = [
		{
		"system_prompt_override": system_prompt_override,
		},
		{"system_prompt_override": system_prompt_override},
		{"max_prompt_token_length": args.max_prompt_token_length},
		]
		train_dataset = get_cached_dataset_tulu(
Expand Down