context recycling eventlet worker

app/clients/document_download.py

@@ -1,3 +1,6 @@
+import logging
+import os
+
 import requests
 from flask import current_app, request
 from flask.ctx import has_request_context
@@ -21,6 +24,7 @@ def __init__(self, app):
         self.api_host_internal = app.config["DOCUMENT_DOWNLOAD_API_HOST_INTERNAL"]
         self.auth_token = app.config["DOCUMENT_DOWNLOAD_API_KEY"]
         self.requests_session = requests.Session()
+        logging.getLogger("app").info("Constructing DocumentDownloadClient", extra={"process_": os.getpid()})
 
     def get_upload_url_for_simulated_email(self, service_id):
         """

gunicorn_config.py

@@ -1,5 +1,10 @@
+from collections import deque
+import contextvars
 import os
 
+import greenlet
+from gunicorn.workers.geventlet import EventletWorker
+
 from notifications_utils.gunicorn_defaults import set_gunicorn_defaults
 
 set_gunicorn_defaults(globals())
@@ -15,13 +20,32 @@ def child_exit(server, worker):
     multiprocess.mark_process_dead(worker.pid)
 
 
-workers = 4
-worker_class = "eventlet"
-worker_connections = 8  # limit runaway greenthread creation
+workers = 2
+# worker_class = "eventlet"
+worker_connections = 4  # limit runaway greenthread creation
 statsd_host = "{}:8125".format(os.getenv("STATSD_HOST"))
 keepalive = 0  # disable temporarily for diagnosing issues
 timeout = int(os.getenv("HTTP_SERVE_TIMEOUT_SECONDS", 30))  # though has little effect with eventlet worker_class
 
+
+class ContextRecyclingEventletWorker(EventletWorker):
+    def __init__(self, *args, **kwargs):
+        self.context_pool = deque()
+        super().__init__(*args, **kwargs)
+
+    def handle(self, *args, **kwargs):
+        g = greenlet.getcurrent()
+        if self.context_pool:
+            g.gr_context = self.context_pool.pop()
+
+        ret = super().handle(*args, **kwargs)
+
+        self.context_pool.append(g.gr_context)
+        g.gr_context = contextvars.Context()
+
+        return ret
+worker_class = "gunicorn_config.ContextRecyclingEventletWorker"
+
 debug_post_threshold = os.getenv("NOTIFY_GUNICORN_DEBUG_POST_REQUEST_LOG_THRESHOLD_SECONDS", None)
 if debug_post_threshold:
     debug_post_threshold_float = float(debug_post_threshold)