add llama3 evaluator in config generation

Gapminder · May 24, 2024 · 2c85d47 · 2c85d47
1 parent 2feba0f
commit 2c85d47
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 0 deletions.
diff --git a/automation-api/yival_experiments/experiment_defaults.yaml b/automation-api/yival_experiments/experiment_defaults.yaml
@@ -17,6 +17,10 @@ custom_evaluators:
     class: ./custom_configuration/vertex_ai_evaluator.VertexAIEvaluator
     config_cls: ./custom_configuration/vertex_ai_evaluator_config.VertexAIEvaluatorConfig
 
+  llama3_evaluator:
+    class: ./custom_configuration/llama3_evaluator.Llama3Evaluator
+    config_cls: ./custom_configuration/llama3_evaluator_config.Llama3EvaluatorConfig
+
 
 custom_function: model_compare.model_compare
 

diff --git a/automation-api/yival_experiments/scripts/generate_experiment_config.py b/automation-api/yival_experiments/scripts/generate_experiment_config.py
@@ -45,6 +45,9 @@ def get_evaluators(ai_eval_sheet: AiEvalData, evaluator_model="gpt4"):
     elif evaluator_model == "claude":
         evaluator_name = "vertex_ai_evaluator"
         model_name = "vertex_ai/claude-3-opus@20240229"
+    elif evaluator_model == "llama":
+        evaluator_name = "llama3_evaluator"
+        model_name = "replicate/meta/meta-llama-3-70b-instruct"
 
     for m in metrics:
         metric: Dict[str, Any] = dict()