update training

This commit is contained in:
Ubuntu
2025-09-01 09:33:16 +00:00
parent a520d9cae5
commit d3bd2806e8
7 changed files with 437 additions and 130 deletions

View File

@@ -115,13 +115,13 @@ def generate_teacher_logits(processor, llm, data_list, config):
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--config", type=str, required=True)
# --- MODIFICATION: Added arguments to define the data chunk ---
# arguments to define the data chunk ---
parser.add_argument("--start_index", type=int, required=True)
parser.add_argument("--end_index", type=int, required=True)
args = parser.parse_args()
config = json.load(open(args.config))
# --- MODIFICATION: The main logic is now simpler ---
logging.info(f"Processing chunk from index {args.start_index} to {args.end_index}")
full_data_list = read_json_field(config["dataset"]["instruction_path"])