update training
This commit is contained in:
@@ -115,13 +115,13 @@ def generate_teacher_logits(processor, llm, data_list, config):
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--config", type=str, required=True)
|
||||
# --- MODIFICATION: Added arguments to define the data chunk ---
|
||||
# arguments to define the data chunk ---
|
||||
parser.add_argument("--start_index", type=int, required=True)
|
||||
parser.add_argument("--end_index", type=int, required=True)
|
||||
args = parser.parse_args()
|
||||
config = json.load(open(args.config))
|
||||
|
||||
# --- MODIFICATION: The main logic is now simpler ---
|
||||
|
||||
logging.info(f"Processing chunk from index {args.start_index} to {args.end_index}")
|
||||
full_data_list = read_json_field(config["dataset"]["instruction_path"])
|
||||
|
||||
|
Reference in New Issue
Block a user