lphatnguyen/distillation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix white box max length dismatch

Browse Source
This commit is contained in:
玖烛
2025-06-09 16:57:41 +08:00
parent 82e59971d9
commit f8727a45ee
1 changed files with 2 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
configs/kd_white_box.json
View File

@@ -19,7 +19,7 @@
"top_logits_num": 10 "top_logits_num": 10
}, },
"distillation": { "distillation": {
"kd_ratio": 0.5, "kd_ratio": 0.1,
"max_seq_length": 512, "max_seq_length": 512,
"distillation_type": "forward_kld" "distillation_type": "forward_kld"
}, },
@@ -32,6 +32,7 @@
"num_train_epochs": 3, "num_train_epochs": 3,
"per_device_train_batch_size": 1, "per_device_train_batch_size": 1,
"gradient_accumulation_steps": 8, "gradient_accumulation_steps": 8,
"max_length":512,
"save_steps": 1000, "save_steps": 1000,
"logging_steps": 1, "logging_steps": 1,
"learning_rate": 2e-5, "learning_rate": 2e-5,