File size: 3,713 Bytes
5374a2d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 |
import os
from dotenv import load_dotenv
from evoagentx.benchmark import MBPPPLUS, AFlowMBPPPLUS
from evoagentx.optimizers import AFlowOptimizer
from evoagentx.models import LiteLLMConfig, LiteLLM, OpenAILLMConfig, OpenAILLM
import os
EXPERIMENTAL_CONFIG = {
"humaneval": {
"question_type": "code",
"operators": ["Custom", "CustomCodeGenerate", "Test", "ScEnsemble"]
},
"mbpp": {
"question_type": "code",
"operators": ["Custom", "CustomCodeGenerate", "Test", "ScEnsemble"]
},
"hotpotqa": {
"question_type": "qa",
"operators": ["Custom", "AnswerGenerate", "QAScEnsemble"]
},
"gsm8k": {
"question_type": "math",
"operators": ["Custom", "ScEnsemble", "Programmer"]
},
"math": {
"question_type": "math",
"operators": ["Custom", "ScEnsemble", "Programmer"]
}
}
class MBPPSplits(AFlowMBPPPLUS):
def _load_data(self):
# load the original MBPP data
mbpp_test_data = AFlowMBPPPLUS().get_dev_data()
# split the data into dev and test
import numpy as np
np.random.seed(42)
permutation = np.random.permutation(len(mbpp_test_data))
# radnomly select 50 samples for dev and 100 samples for test (be consistent with other models)
dev_data_task_ids = [mbpp_test_data[idx]["task_id"] for idx in permutation[:30]]
super()._load_data()
full_data = self._dev_data
self._dev_data = [example for example in full_data if example["task_id"] in dev_data_task_ids]
def main():
from evoagentx.models import OpenAILLMConfig, OpenAILLM,AzureOpenAIConfig,LiteLLMConfig,LiteLLM
from evoagentx.workflow import SEWWorkFlowGraph
from evoagentx.agents import AgentManager
from evoagentx.benchmark import ClassEval
from evoagentx.evaluators import Evaluator
from evoagentx.optimizers import SEWOptimizer
from evoagentx.core.callbacks import suppress_logger_info
os.environ["AZURE_OPENAI_DEPLOYMENT_NAME"] = "gpt-4o-mini"
os.environ["AZURE_OPENAI_ENDPOINT"] = "https://75244-mfztkr7x-eastus2.cognitiveservices.azure.com/"
os.environ["AZURE_OPENAI_KEY"] = "8PNMdsUYGdMPsCfl0baO0hjtnGE2m40zJTrUGC3vKnHdpjnkOgeQJQQJ99BIACHYHv6XJ3w3AAAAACOG7VZI"
os.environ["AZURE_OPENAI_API_VERSION"] = "2024-12-01-preview"
# llm_config = OpenAILLMConfig(model="gpt-4o-mini-2024-07-18", openai_key=OPENAI_API_KEY, top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)
llm_config = LiteLLMConfig(model="azure/" + os.getenv("AZURE_OPENAI_DEPLOYMENT_NAME"), # Azure model format
azure_endpoint=os.getenv("AZURE_OPENAI_ENDPOINT"),
azure_key=os.getenv("AZURE_OPENAI_KEY"),
api_version=os.getenv("AZURE_OPENAI_API_VERSION", "2024-12-01-preview"), top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)
executor_llm = LiteLLM(config=llm_config)
optimizer_llm = LiteLLM(config=llm_config)
# load benchmark
mbpp_input = MBPPSplits()
mbpp = AFlowMBPPPLUS()
mbpp._dev_data = mbpp_input._dev_data
mbpp.error_list = {}
# create optimizer
optimizer = AFlowOptimizer(
graph_path = "examples/aflow/code_generation",
optimized_path = "examples/aflow/mbppplus_full_new/optimized",
optimizer_llm=optimizer_llm,
executor_llm=executor_llm,
validation_rounds=5,
eval_rounds=1,
max_rounds=10,
**EXPERIMENTAL_CONFIG["mbpp"]
)
# run optimization
optimizer.optimize(mbpp)
# run test
optimizer.test(mbpp) # use `test_rounds: List[int]` to specify the rounds to test
if __name__ == "__main__":
main() |