add a lower bound on the evaluations being run to save time

random_eval_matchmaker.py

@@ -13,7 +13,7 @@ class RandomEvalMatchmaker(BaseMatchmaker):
     def __init__(self, sigma_target=1, pretrained_agents: PretrainedAgents = None, non_latest_version_prob=[1, 0, 0, 0],
                  past_version_prob=1, full_team_trainings=0, full_team_evaluations=0, force_non_latest_orange=False,
                  showmatch=False,
-                 orange_agent_text_file=None):
+                 orange_agent_text_file=None, min_to_test=0):
         """
         :param sigma_target: The sigma value at which agents stop playing in eval matches frequently
         :param pretrained_agents: a configuration dict for how and how often to use pretrained agents in matchups.
@@ -23,6 +23,7 @@ def __init__(self, sigma_target=1, pretrained_agents: PretrainedAgents = None, n
         :param full_team_evaluations: The probability that a match uses all agents of the same type on a given team in evals.
         :param force_non_latest_orange: A boolean that, if true, ensures the first player in the list is latest agent (if one exists in the match).
         """
+        self.min_to_test = min_to_test
         self.orange_agents_text_file = orange_agent_text_file
         self.showmatch = showmatch
         self.sigma_target = sigma_target
@@ -86,6 +87,9 @@ def generate_matchup(self, redis, n_agents, evaluate):
             return [latest_version] * n_agents, [rating] * n_agents, False, n_agents // 2, n_agents // 2
 
         past_version_ratings = get_ratings(gamemode, redis)
+        past_version_ratings = {k: v for k, v in past_version_ratings.items()
+                                if int(k.split('-')[1].split('v')[1]) >= self.min_to_test}
+
         # This is the rating of the most recent eval model
         latest_rating = past_version_ratings[latest_key]
         past_version_ratings_keys, past_version_ratings_values = zip(

utils/redis_work.py

@@ -1,7 +1,8 @@
 from redis import Redis
 import os
 from trueskill import Rating
-from rocket_learn.rollout_generator.redis.utils import get_rating, _unserialize, _serialize, get_ratings, decode_buffers, PRETRAINED_QUALITIES
+from rocket_learn.rollout_generator.redis.utils import get_rating, _unserialize, _serialize, get_ratings, decode_buffers, PRETRAINED_QUALITIES,QUALITIES\
+        ,OPPONENT_MODELS
 
 
 host = "127.0.0.1"
@@ -15,6 +16,11 @@
 # quality_key = PRETRAINED_QUALITIES.format("3v3")
 # nexto = {k.decode("utf-8"): Rating(*_unserialize(v)) for k, v in r.hgetall(quality_key).items()}
 
+# rating = Rating()
+# for _gamemode in ['1v1', '2v2', '3v3']:
+#     for model in past_version_ratings.keys():
+#         r.hset(QUALITIES.format(_gamemode), model, _serialize(tuple(rating)))
+
 # new = Rating(mu=110, sigma=30)
 
 for gamemode in ['1v1', '2v2', '3v3']:

worker_gp.py

@@ -132,7 +132,7 @@
     local = True
     auto_minimize = True
     game_speed = 100
-    evaluation_prob = 1
+    evaluation_prob = 0
     past_version_prob = 0  # 0.5  # 0.1
     non_latest_version_prob = [0.85, 0.1, 0.05, 0.0]  # this includes past_version and pretrained
     deterministic_streamer = True
@@ -162,7 +162,8 @@
     #                         non_latest_version_prob=non_latest_version_prob)
 
     matchmaker = RandomEvalMatchmaker(sigma_target=0.5, past_version_prob=0, full_team_trainings=1, full_team_evaluations=1,
-                                      force_non_latest_orange=False, non_latest_version_prob=non_latest_version_prob)
+                                      force_non_latest_orange=False, non_latest_version_prob=non_latest_version_prob,
+                                      min_to_test=400)
 
 
     # terminals = [GoalScoredCondition(),