From 0165f28f3f384248ce1c1cb356b53cd6e0419944 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=86=8A=E5=85=AE?= Date: Mon, 23 Jun 2025 17:01:37 +0800 Subject: [PATCH] feat: update instruct eval --- configs/instruct_eval_api.json | 12 ++ easydistill/eval/cot_eval.py | 177 ---------------------- easydistill/eval/data_eval.py | 268 +++++++++++++++++++++++++++++++++ 3 files changed, 280 insertions(+), 177 deletions(-) create mode 100644 configs/instruct_eval_api.json delete mode 100644 easydistill/eval/cot_eval.py create mode 100644 easydistill/eval/data_eval.py diff --git a/configs/instruct_eval_api.json b/configs/instruct_eval_api.json new file mode 100644 index 0000000..44121d0 --- /dev/null +++ b/configs/instruct_eval_api.json @@ -0,0 +1,12 @@ +{ + "job_type": "instruct_eval_api", + "dataset": { + "input_path": "instruct_input.json", + "output_path": "instruct_output.json" + }, + "inference":{ + "base_url": "ENDPOINT", + "api_key": "TOKEN", + "max_new_tokens": 8196 + } +} \ No newline at end of file diff --git a/easydistill/eval/cot_eval.py b/easydistill/eval/cot_eval.py deleted file mode 100644 index 726050c..0000000 --- a/easydistill/eval/cot_eval.py +++ /dev/null @@ -1,177 +0,0 @@ - -# Copyright 2024 Alibaba Group Holding Limited. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -import json, jsonlines -import argparse -import logging -import os -import re -from tqdm import tqdm -from openai import OpenAI - - -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') - - -def build_cot_prompts(instruction, output): - rv_prompt_template = ( - "You are an expert judge tasked with evaluating the Reasoning Verbosity of a Chain-of-Thought (CoT) " - "for a given problem and its answer. Reasoning Verbosity Evaluation Focus: Assess how well the CoT’s " - "length and step complexity match the problem’s inherent difficulty. An optimal chain is neither " - "missing essential steps nor padded with needless digressions. A simple question should be solved " - "with a brief, direct chain; a challenging one may justifiably require a longer path with reflection " - "and error-checking. Scoring Guidelines (0-9):\n" - "0-1 Minimal verbosity, straightforward expression with little to no elaboration.\n" - "2-3 Clear and concise reasoning with necessary explanations.\n" - "4-5 Moderate verbosity with detailed explanations and thorough reasoning.\n" - "6-7 Extensive verbosity with comprehensive justification and exploration of complex connections.\n" - "8-9 High verbosity with deep, exhaustive exploration of reasoning; involves extensive elaboration, nested justifications, " - "and consideration of counterarguments or alternative perspectives.\n" - "Given Problem, Answer with hain-of-Thought, you will:\n" - "1. Analyze the Reasoning Verbosity\n" - "2. Determine score using the above criteria\n" - "3. Output ONLY the integer score (0-9), place your score in \n" - f"Problem: {instruction}\n" - f"Answer with Chain-of-Thought: {output}" - ) - cd_prompt_template = ( - "You are an expert judge assessing the Cognitive Difficulty of a Chain-of-Thought (CoT) " - "for a given problem and its answer. Cognitive Difficulty Evaluation Focus: The level of " - "reasoning competence required for a model to follow and reproduce the chain faithfully. " - "Judge the reasoning approach, techniques, and overall difficulty. Higher scores correspond " - "to more advanced concepts, abstractions, or multi-layer reasoning patterns. " - "Scoring Guidelines (0-9):\n" - "0-1 Elementary facts or a single trivial operation.\n" - "2-3 Multi-step arithmetic, explicit enumeration, basic rule chaining.\n" - "4-5 Early-undergraduate logic/algebra; one non-obvious insight.\n" - "6-7 Advanced undergraduate techniques (determinants, dynamic programming, layered code reasoning, etc).\n" - "8-9 Graduate-level abstraction, nested proofs, intricate algorithmic analysis.\n" - "Given Problem, Answer with hain-of-Thought, you will:\n" - "1. Analyze the Cognitive Difficulty\n" - "2. Determine score using the above criteria\n" - "3. Output ONLY the integer score (0-9), place your score in \n" - f"Problem: {instruction}\n" - f"Answer with Chain-of-Thought: {output}" - ) - lc_prompt_template = ( - "You are a rigorous logical validator analyzing problem-solving components. " - "Your task is to separately assess the validity of the reasoning process and final solution. " - "Given Problem, Answer with hain-of-Thought, you will:\n" - "1. Verify stepwise logical coherence and soundness\n" - "2. Confirm all critical problem constraints are properly addressed\n" - "3. Check for self-contradictions or unsupported leaps in logic\n" - "4. Verify the process can actually derive the proposed solution\n" - "5. Output ONLY the 1/0 answer (1 for true, 0 for false) for logical correctness, place your answer in \n" - f"Problem: {instruction}\n" - f"Answer with Chain-of-Thought: {output}" - ) - return rv_prompt_template, cd_prompt_template, lc_prompt_template - - -def extract_score(text): - match = re.search(r"(\d+)", text) - if match: - return int(match.group(1)) - else: - return -1 - - -def read_json_fields(filename): - try: - with open(filename, 'r') as file: - data = json.load(file) - return data - except FileNotFoundError: - logging.error("The file was not found.") - except json.JSONDecodeError: - logging.error("There was an error decoding the JSON file.") - except Exception as e: - logging.error(f"An error occurred: {e}") - - -def write_data_to_json_file(data, file_path): - try: - with open(file_path, 'w') as file: - json.dump(data, file, ensure_ascii=False, indent=4) - logging.info(f"Data successfully written to {file_path}") - except Exception as e: - logging.error(f"An error occurred: {e}") - - -def generate_teacher_response_api(data_list, config): - client = OpenAI( - api_key = config["inference"]["api_key"], - base_url = config["inference"]["base_url"] - ) - models = client.models.list() - model = models.data[0].id - logging.info(model) - outcomes = [] - for sample in tqdm(data_list, desc="Call remote model and generating responses"): - instruction = sample["instruction"] - output = sample["output"] - rv_prompt_template, cd_prompt_template, lc_prompt_template = build_cot_prompts(instruction, output) - - def generate_score(sample, model, config): - message = [ - {'role': 'user', 'content': sample} - ] - completion = client.chat.completions.create( - messages = message, - model = model, - max_completion_tokens = config["inference"]["max_new_tokens"] - ) - result = completion.choices[0].message.content - score = extract_score(result) - return score - - rv_score = generate_score(rv_prompt_template, model, config) - cd_score = generate_score(cd_prompt_template, model, config) - lc_score = generate_score(lc_prompt_template, model, config) - if lc_score == 1: - lc_score = True - else: - lc_score =False - - outcomes.append( - { - 'instruction': instruction, - 'output': output, - "reasoning_verbosity": rv_score, - "cognitive_difficulty": cd_score, - "logical_correctness": lc_score - } - ) - write_data_to_json_file(outcomes, config["dataset"]["output_path"]) - - -def infer_with_teacher_model(config): - logging.info('Generating distillation data from the teacher model!') - data_list = read_json_fields(config["dataset"]["input_path"]) - generate_teacher_response_api(data_list, config) - - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--config', type=str, required=True, help='path to the json config file') - args = parser.parse_args() - config = json.load(open(args.config)) - infer_with_teacher_model(config) - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/easydistill/eval/data_eval.py b/easydistill/eval/data_eval.py new file mode 100644 index 0000000..b16d4ff --- /dev/null +++ b/easydistill/eval/data_eval.py @@ -0,0 +1,268 @@ + +# Copyright 2024 Alibaba Group Holding Limited. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import json +import argparse +import logging +import os +import re +from tqdm import tqdm +from openai import OpenAI + + +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') + + +def build_cot_prompts(instruction, output): + rv_prompt_template = ( + "You are an expert judge tasked with evaluating the Reasoning Verbosity of a Chain-of-Thought (CoT) " + "for a given problem and its answer. Reasoning Verbosity Evaluation Focus: Assess how well the CoT’s " + "length and step complexity match the problem’s inherent difficulty. An optimal chain is neither " + "missing essential steps nor padded with needless digressions. A simple question should be solved " + "with a brief, direct chain; a challenging one may justifiably require a longer path with reflection " + "and error-checking. Scoring Guidelines (0-9):\n" + "0-1 Minimal verbosity, straightforward expression with little to no elaboration.\n" + "2-3 Clear and concise reasoning with necessary explanations.\n" + "4-5 Moderate verbosity with detailed explanations and thorough reasoning.\n" + "6-7 Extensive verbosity with comprehensive justification and exploration of complex connections.\n" + "8-9 High verbosity with deep, exhaustive exploration of reasoning; involves extensive elaboration, nested justifications, " + "and consideration of counterarguments or alternative perspectives.\n" + "Given Problem, Answer with hain-of-Thought, you will:\n" + "1. Analyze the Reasoning Verbosity\n" + "2. Determine score using the above criteria\n" + "3. Output ONLY the integer score (0-9), place your score in \n" + f"Problem: {instruction}\n" + f"Answer with Chain-of-Thought: {output}" + ) + cd_prompt_template = ( + "You are an expert judge assessing the Cognitive Difficulty of a Chain-of-Thought (CoT) " + "for a given problem and its answer. Cognitive Difficulty Evaluation Focus: The level of " + "reasoning competence required for a model to follow and reproduce the chain faithfully. " + "Judge the reasoning approach, techniques, and overall difficulty. Higher scores correspond " + "to more advanced concepts, abstractions, or multi-layer reasoning patterns. " + "Scoring Guidelines (0-9):\n" + "0-1 Elementary facts or a single trivial operation.\n" + "2-3 Multi-step arithmetic, explicit enumeration, basic rule chaining.\n" + "4-5 Early-undergraduate logic/algebra; one non-obvious insight.\n" + "6-7 Advanced undergraduate techniques (determinants, dynamic programming, layered code reasoning, etc).\n" + "8-9 Graduate-level abstraction, nested proofs, intricate algorithmic analysis.\n" + "Given Problem, Answer with hain-of-Thought, you will:\n" + "1. Analyze the Cognitive Difficulty\n" + "2. Determine score using the above criteria\n" + "3. Output ONLY the integer score (0-9), place your score in \n" + f"Problem: {instruction}\n" + f"Answer with Chain-of-Thought: {output}" + ) + lc_prompt_template = ( + "You are a rigorous logical validator analyzing problem-solving components. " + "Your task is to separately assess the validity of the reasoning process and final solution. " + "Given Problem, Answer with hain-of-Thought, you will:\n" + "1. Verify stepwise logical coherence and soundness\n" + "2. Confirm all critical problem constraints are properly addressed\n" + "3. Check for self-contradictions or unsupported leaps in logic\n" + "4. Verify the process can actually derive the proposed solution\n" + "5. Output ONLY the 1/0 answer (1 for true, 0 for false) for logical correctness, place your answer in \n" + f"Problem: {instruction}\n" + f"Answer with Chain-of-Thought: {output}" + ) + return rv_prompt_template, cd_prompt_template, lc_prompt_template + + +def build_instruct_prompts(instruction, output): + informativeness_template = ( + "You are an expert judge tasked with evaluating the Informativeness of a response generated by an instruction-following model " + "for a given user instruction. Informativeness Evaluation Focus: Assess how thoroughly and accurately the response addresses " + "the user’s instruction, providing relevant details, facts, and explanations without omissions or irrelevant additions. " + "An informative response fully satisfies the query with meaningful content, whereas a less informative one may be vague, " + "incomplete, or superficial. Scoring Guidelines (0-9):\n" + "0-1 Very low informativeness; the response is irrelevant or nearly empty.\n" + "2-3 Low informativeness; addresses the instruction minimally with significant missing information.\n" + "4-5 Moderate informativeness; covers some key points but lacks depth or completeness.\n" + "6-7 High informativeness; provides detailed and mostly comprehensive information relevant to the instruction.\n" + "8-9 Exceptional informativeness; thoroughly and accurately covers all relevant aspects with rich and precise details.\n" + "Given Instruction and Model Response, you will:\n" + "1. Analyze the Informativeness of the response\n" + "2. Determine a score using the above criteria\n" + "3. Output ONLY the integer score (0-9), place your score in \n" + f"Instruction: {instruction}\n" + f"Response: {output}" + ) + helpfulness_template = ( + "You are an expert judge tasked with evaluating the Helpfulness of a response generated by an instruction-following model " + "for a given user instruction. Helpfulness Evaluation Focus: Assess how well the response assists the user in accomplishing " + "their goal, providing clear, actionable, and relevant information or guidance. A helpful response should be easy " + "to understand and effectively address the user’s needs without unnecessary confusion or missing key details.\n" + "Scoring Guidelines (0-9):\n" + "0-1 Not helpful; response is irrelevant, confusing, or fails to address the instruction.\n" + "2-3 Slightly helpful; responds partially but lacks clarity or important elements.\n" + "4-5 Moderately helpful; response addresses the instruction but may be incomplete or somewhat unclear.\n" + "6-7 Mostly helpful; provides clear and relevant information that adequately assists the user.\n" + "8-9 Extremely helpful; offers comprehensive, clear, and precise guidance or information that fully satisfies the user’s instruction.\n" + "Given Instruction and Model Response, you will:\n" + "1. Analyze the Helpfulness of the response\n" + "2. Determine a score using the above criteria\n" + "3. Output ONLY the integer score (0-9), place your score in \n" + f"Instruction: {instruction}\n" + f"Response: {output}" + ) + generalization_template = ( + "You are an expert judge tasked with evaluating the Potential for Generalization of a response generated by an " + "instruction-following model to similar but unseen tasks. Generalization Evaluation Focus: Assess how well the response " + "demonstrates understanding and reasoning that can be effectively adapted or transferred to other related instructions or " + "problems beyond the specific input. A response with high generalization ability " + "captures underlying principles or strategies rather than relying on shallow, task-specific heuristics.\n" + "Scoring Guidelines (0-9):\n" + "0-1 Very poor generalization; response is overly specific, rigid, or fails to show adaptable reasoning.\n" + "2-3 Limited generalization; response applies partly to related tasks but is mostly narrow or shallow.\n" + "4-5 Moderate generalization; response reflects some transferable understanding but may lack depth or clarity.\n" + "6-7 Strong generalization; response shows clear reasoning patterns or concepts that can extend to similar tasks.\n" + "8-9 Exceptional generalization; response exhibits deep, abstract, and flexible comprehension applicable across a broad range of related instructions.\n" + "Given Instruction and Model Response, you will:\n" + "1. Analyze the Potential for Generalization to Similar Tasks\n" + "2. Determine a score using the above criteria\n" + "3. Output ONLY the integer score (0-9), place your score in \n" + f"Instruction: {instruction}\n" + f"Response: {output}" + ) + correctness_template = ( + "You are a meticulous correctness evaluator tasked with assessing whether the response to a user instruction " + "is factually accurate and logically sound.\n" + "Your evaluation should determine:\n" + "1. Whether the response correctly addresses the instruction\n" + "2. Whether any factual claims or data are accurate\n" + "3. Whether the reasoning, if present, is logically valid and free of errors\n" + "4. Whether the final answer is consistent with the evidence or instructions provided\n" + "You will:\n" + "Output ONLY '1' if the response is correct and accurate, or '0' if it contains factual errors, logical flaws, " + "or fails to correctly address the instruction.\n" + "Place your answer in tags.\n" + f"Instruction: {instruction}\n" + f"Response: {output}" + ) + return informativeness_template, helpfulness_template, generalization_template, correctness_template + + + +def extract_score(text): + match = re.search(r"(\d+)", text) + if match: + return int(match.group(1)) + else: + return -1 + + +def read_json_fields(filename): + try: + with open(filename, 'r') as file: + data = json.load(file) + return data + except FileNotFoundError: + logging.error("The file was not found.") + except json.JSONDecodeError: + logging.error("There was an error decoding the JSON file.") + except Exception as e: + logging.error(f"An error occurred: {e}") + + +def write_data_to_json_file(data, file_path): + try: + with open(file_path, 'w') as file: + json.dump(data, file, ensure_ascii=False, indent=4) + logging.info(f"Data successfully written to {file_path}") + except Exception as e: + logging.error(f"An error occurred: {e}") + + +def generate_teacher_response_api(data_list, config, is_cot_model): + client = OpenAI( + api_key = config["inference"]["api_key"], + base_url = config["inference"]["base_url"] + ) + models = client.models.list() + model = models.data[0].id + logging.info(model) + outcomes = [] + for sample in tqdm(data_list, desc="Call remote model and generating responses"): + instruction = sample["instruction"] + output = sample["output"] + + def generate_score(sample, model, config): + message = [ + {'role': 'user', 'content': sample} + ] + completion = client.chat.completions.create( + messages = message, + model = model, + max_completion_tokens = config["inference"]["max_new_tokens"] + ) + result = completion.choices[0].message.content + score = extract_score(result) + return score + + if is_cot_model: + rv_prompt_template, cd_prompt_template, lc_prompt_template = build_cot_prompts(instruction, output) + rv_score = generate_score(rv_prompt_template, model, config) + cd_score = generate_score(cd_prompt_template, model, config) + lc_score = generate_score(lc_prompt_template, model, config) + lc_score = (lc_score == 1) + outcomes.append( + { + 'instruction': instruction, + 'output': output, + "reasoning_verbosity": rv_score, + "cognitive_difficulty": cd_score, + "logical_correctness": lc_score + } + ) + else: + informativeness_temp, helpfulness_temp, generalization_temp, correctness_temp = build_instruct_prompts(instruction, output) + informativeness = generate_score(informativeness_temp, model, config) + helpfulness = generate_score(helpfulness_temp, model, config) + generalization = generate_score(generalization_temp, model, config) + correctness = generate_score(correctness_temp, model, config) + correctness = (correctness == 1) + outcomes.append( + { + 'instruction': instruction, + 'output': output, + "informativeness": informativeness, + "helpfulness": helpfulness, + "generalization": generalization, + "correctness": correctness + } + ) + + write_data_to_json_file(outcomes, config["dataset"]["output_path"]) + + +def infer_with_teacher_model(config): + logging.info('Generating distillation data from the teacher model!') + data_list = read_json_fields(config["dataset"]["input_path"]) + job_type = config["job_type"] + is_cot_model = "cot" in job_type + generate_teacher_response_api(data_list, config, is_cot_model) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--config', type=str, required=True, help='path to the json config file') + args = parser.parse_args() + config = json.load(open(args.config)) + infer_with_teacher_model(config) + + +if __name__ == "__main__": + main() \ No newline at end of file