Merge pull request #296 from reecelikesramen/feature-config-autocommi…

…t-model

Feature: configure autocommit model

src/ell/configurator.py

@@ -38,6 +38,7 @@ class Config(BaseModel):
     lazy_versioning: bool = Field(default=True, description="If True, enables lazy versioning for improved performance.")
     default_api_params: Dict[str, Any] = Field(default_factory=dict, description="Default parameters for language models.")
     default_client: Optional[openai.Client] = Field(default=None, description="The default OpenAI client used when a specific model client is not found.")
+    autocommit_model: str = Field(default="gpt-4o-mini", description="When set, changes the default autocommit model from GPT 4o mini.")
     providers: Dict[Type, Provider] = Field(default_factory=dict, description="A dictionary mapping client types to provider classes.")
     def __init__(self, **data):
         super().__init__(**data)
@@ -148,7 +149,8 @@ def init(
     autocommit: bool = True,
     lazy_versioning: bool = True,
     default_api_params: Optional[Dict[str, Any]] = None,
-    default_client: Optional[Any] = None
+    default_client: Optional[Any] = None,
+    autocommit_model: str = "gpt-4o-mini"
 ) -> None:
     """
     Initialize the ELL configuration with various settings.
@@ -165,6 +167,8 @@ def init(
     :type default_api_params: Dict[str, Any], optional
     :param default_openai_client: Set the default OpenAI client.
     :type default_openai_client: openai.Client, optional
+    :param autocommit_model: Set the model used for autocommitting.
+    :type autocommit_model: str
     """
     # XXX: prevent double init
     config.verbose = verbose
@@ -183,6 +187,9 @@ def init(
     if default_client is not None:
         config.default_client = default_client
 
+    if autocommit_model is not None:
+        config.autocommit_model = autocommit_model
+
 # Existing helper functions
 def get_store() -> Union[Store, None]:
     return config.store

src/ell/util/differ.py

@@ -1,42 +1,52 @@
-
+from ell.configurator import config
 from ell.lmp.simple import simple
 import difflib
 
 # Todo: update this for single change stuff so that it doesn't summarize small chage but says it specifically.
-@simple("gpt-4o-mini", temperature=0.2, exempt_from_tracking=True)
+@simple(config.autocommit_model, temperature=0.2, exempt_from_tracking=True, max_tokens=500)
 def write_commit_message_for_diff(old : str, new : str) -> str:
-    """You are an expert programmer who's goal is to write commit messages based on diffs. 
-
-You will be given two version of source code. 
-You will be expected to write a commit message that describes the changes between the two versions. Your commit message should be at most one sentence and highly specific to the changes made. Don't just discuss the functions changed but how they were specifically changed.
-Your commit message cannot be more than 10 words so use sentence fragments and be concise.
-The @ell.simple decorator turns a function into a call to a language model: 
-    the docstring is the system prompt and the string returned in the user prompt. 
-It is extremely important that if these are change: your commit message must say what specifically changed in the user or system prompt rather than saying they were updated or changed geneircally.
-It is extremely important that you never refer to a @ell.simple docstring as a docstring; it is a system prompt. 
-Don't say why a change was done but what specifically changed.
-Consider all changes ot the program including the globals and free variables
-Respond in the following format:
-<commit_message summarizing all changes with specificity>:
-<* bulleted list of all changes>."""
+    """You are an expert programmer whose goal is to write commit messages based on diffs.
+You will be given two version of source code and their unified diff.
+You will be expected to write a commit message that describes the changes between the two versions.
+
+Follow these guidelines:
+1. Your commit message should be at most one sentence and highly specific to the changes made. Don't just discuss the functions changed but how they were specifically changed.
+2. Your commit message cannot be more than 10 words so use sentence fragments and be concise.
+3. The @ell.simple decorator turns a function into a call to a language model program: the function's docstring is the system prompt and the string returned is the user prompt. 
+4. It is extremely important that if the system prompt or user prompt changes, your commit message must say what specifically changed, rather than vaguely saying they were updated or changed.
+5. It is extremely important that you never refer to a @ell.simple docstring as a docstring: it is a system prompt. 
+6. Do NOT say why a change was done, say what specifically changed.
+7. Consider all changes ot the program including the globals and free variables
+
+Example response:
+'''
+Update model temperature and refine system prompt wording:
+* Changed temperature from 0.5 to 0.7.
+* Updated "with specificity, brevity, and good grammar" to "clearly and concisely" in system prompt.
+* The `questions` param was assigned type List[Question]
+'''
+Response format:
+<Short commit message summarizing all the changes with specificity>:
+* <Bulleted list of each specific change>.
+"""
     clean_program_of_all_bv_tags = lambda program : program.replace("#<BV>", "").replace("#</BV>", "").replace("#<BmV>", "").replace("#</BmV>", "")
     old_clean = clean_program_of_all_bv_tags(old)
     new_clean = clean_program_of_all_bv_tags(new)
 
-    diff = difflib.unified_diff(old_clean.splitlines(), new_clean.splitlines(), lineterm='')
+    diff = "\n".join(difflib.unified_diff(old_clean.splitlines(), new_clean.splitlines(), lineterm=''))
 
     return f"""Write a commit message succinctly and specifically describing the changes between these two versions of a program.
-OLD VERISON:
+Old version:
 ```
 {old_clean}
 ```
 
-NEW VERSION:
+New version:
 ```
 {new_clean}
 ```
 
-DIFF:
+Unified diff:
 {diff}
 """
 

tests/test_autocommit_model.py

@@ -0,0 +1,130 @@
+import ell
+from ell.configurator import config
+import anthropic
+
+ell.init(verbose=True, autocommit_model="gpt-4o-mini")
+# ell.init(verbose=True, autocommit_model="claude-3-haiku-20240307")
+
+test1_v1 = '''import ell
+import numpy as np
+
+@ell.simple(model="gpt-4o-mini")
+def come_up_with_a_premise_for_a_joke_about(topic : str):
+    """You are an incredibly funny comedian. Come up with a premise for a joke about topic"""
+    return f"come up with a premise for a joke about {topic}"
+
+def get_random_length():
+    return int(np.random.beta(2, 5) * 300)
+
+@ell.simple(model="gpt-4o-mini")
+def joke(topic : str):
+    """You are a funny comedian. You respond in scripts for a standup comedy skit."""
+    return f"Act out a full joke. Make your script {get_random_length()} words long. Here's the premise: {come_up_with_a_premise_for_a_joke_about(topic)}"'''
+
+test1_v2 = '''import ell
+import numpy as np
+
+@ell.simple(model="gpt-4o-mini")
+def come_up_with_a_premise_for_a_joke_about(topic : str):
+    """You are an incredibly funny comedian. Come up with a premise for a joke about topic"""
+    return f"come up with a premise for a joke about {topic}"
+
+def get_random_length():
+    return int(np.random.beta(2, 5) * 200)
+
+@ell.simple(model="gpt-4o-mini")
+def joke(topic : str):
+    """You are a funny comedian. You respond in scripts for skits."""
+    return f"Act out a full joke. Make your script {get_random_length()} words long. Here's the premise: {come_up_with_a_premise_for_a_joke_about(topic)}"'''
+
+test2_v1 = """CHORD_FORMAT = "| Chord | Chord | ... |"
+
+@ell.simple(model="gpt-4o", temperature=0.5)
+def write_a_chord_progression_for_song(genre: Optional[str], key : Optional[str]) :
+    return [
+        ell.system(f"You are a world class music theorist and composer. Your goal is to write chord progressions to songs given parameters. They should be fully featured and compositionally sound. Feel free to use advanced chords of your choosing. Only answer with the chord progression in {CHORD_FORMAT} format. Do not provide any additional text. Feel free to occaisonally use 13 chords and complex chords if necessary etc."),
+        ell.user(f"Write a chord progression for a song {'in ' + genre if genre else ''} {'in the key of ' + key if key else ''}.")
+
+    ]"""
+
+test2_v2 = """CHORD_FORMAT = "| Chord | Chord | ... |"
+
+@ell.simple(model="gpt-4o", temperature=0.7)
+def write_a_chord_progression_for_song(genre: Optional[str], key : Optional[str]) :
+    return [
+        ell.system(f"You are a world-renowned class music theorist and composer. Your goal is to write chord progressions for a song given a genre or key. They should be fully featured and compositionally sound. Feel free to use advanced chords of your choosing. Only answer with the chord progression in {CHORD_FORMAT} format. Do not provide any additional text. Feel free to occaisonally use 13 chords and complex chords if necessary etc."),
+        ell.user(f"Write a chord progression for a song {'in ' + genre if genre else ''} {'in the key of ' + key if key else ''}.")
+
+    ]"""
+
+from ell.util.differ import write_commit_message_for_diff
+
+# test 1
+(response, *args) = write_commit_message_for_diff(test1_v1, test1_v2)
+print(response)
+
+# test 2
+(response, *args) = write_commit_message_for_diff(test2_v1, test2_v2)
+print(response)
+
+### --BEFORE PROMPT CHANGES-- ###
+
+### CLAUDE 3 HAIKU ###
+# Test 1:
+# Reduced maximum script length in `get_random_length()` function:
+# * The maximum script length returned by `get_random_length()` was reduced from 300 to 200 words.
+# * The system prompt for the `joke()` function was updated to remove the word "standup comedy".
+#
+# <Reduced maximum script length in `get_random_length()` function, updated system prompt for `joke()` function>:
+# * Decreased the maximum script length returned by `get_random_length()` from 300 to 200 words.
+# * Removed the phrase "standup comedy" from the system prompt for the `joke()` function.
+
+# Test 2:
+# Increased temperature in @ell.simple decorator from 0.5 to 0.7:
+# * Changed the temperature parameter in the @ell.simple decorator from 0.5 to 0.7.
+# * Updated the system prompt to describe the music theorist and composer as "world-renowned" instead of "world class".
+#
+# <commit_message summarizing all changes with specificity>:
+# Increased temperature in @ell.simple decorator from 0.5 to 0.7, updated system prompt description.
+#
+# * Changed the temperature parameter in the @ell.simple decorator from 0.5 to 0.7.
+# * Updated the system prompt to describe the music theorist and composer as "world-renowned" instead of "world class".
+
+### GPT 4o MINI ###
+# Test 1:
+# Reduce script length to 200 words and update skit prompt:  
+# * Changed `get_random_length` to return 200 instead of 300.  
+# * Updated the system prompt in `joke` to specify "scripts for skits" instead of "scripts for a standup comedy skit."
+
+# Test 2:
+# Update model temperature and refine system prompt wording:
+# * Changed temperature from 0.5 to 0.7.
+# * Updated "world class" to "world-renowned class" in system prompt.
+# * Changed "to write chord progressions to songs given parameters" to "to write chord progressions for a song given a genre or key" in system prompt.
+
+
+### --AFTER PROMPT CHANGES-- ###
+
+### CLAUDE 3 HAIKU ###
+# Test 1:
+# Reduce maximum script length and simplify system prompt for joke function:
+# * Changed `get_random_length()` to return a maximum script length of 200 words, down from 300.
+# * Updated the system prompt for the `joke()` function to say "You are a funny comedian. You respond in scripts for skits." instead of "You are a funny comedian. You respond in scripts for a standup comedy skit."
+
+# Test 2:
+# Increase model temperature and refine system prompt wording:
+# * Changed temperature from 0.5 to 0.7.
+# * Updated "world class music theorist and composer" to "world-renowned class music theorist and composer" in system prompt.
+# * Changed "write chord progressions to songs given parameters" to "write chord progressions for a song given a genre or key" in system prompt.
+
+### GPT 4o MINI ###
+# Test 1:
+# Reduce script length and modify system prompt wording:
+# * Changed `get_random_length` to return 200 instead of 300.
+# * Updated system prompt from "scripts for a standup comedy skit" to "scripts for skits."
+
+# Test 2:
+# Update temperature and refine system prompt details:
+# * Changed temperature from 0.5 to 0.7.
+# * Updated "world class" to "world-renowned" in system prompt.
+# * Changed "write chord progressions to songs given parameters" to "write chord progressions for a song given a genre or key" in system prompt.