diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..52863a1 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +[*] +charset = utf-8 + +[*.{js,py,html,email}] +indent_size = 4 +indent_style = space +tab_width = 4 +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[*.md] +indent_size = 4 +indent_style = space +trim_trailing_whitespace = false diff --git a/CHANGELOG.md b/CHANGELOG.md index 49e6752..8d6ecb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,20 @@ # Changelog This file documents any relevant changes. +## [0.9.10] - 2024-05.06 +- fix: Improve and refactor entrypoint to make --workers and --threads argument working #7 + +## [0.9.9] - 2024-01.18 +- feat: added support timeout parameter +- refactor: Change pip version from exact to minimum conditions + +## [0.9.8] - 2023-11.22 +- feat: added support for custom environment variables + +## [0.9.7] - 2023-10.27 +- fix: readded subdomain support +- feat: added editorconf + ## [0.9.6] - 2023-09-15 - fix: updated grpcio to support python 3.11 - fix: gunicorn will be patched on start up diff --git a/setup.cfg b/setup.cfg index 948c517..df4d336 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,9 +6,9 @@ author_email = ak@mausbrand.de description = a lightweight web application launcher for gunicorn and static files. long_description = file: README.md long_description_content_type = text/markdown -url = https://github.com/XeoN-GHMB/app_server +url = https://github.com/viur-framework/viur-app_server project_urls = - Bug Tracker = https://github.com/XeoN-GHMB/app_server/issues + Bug Tracker = https://github.com/viur-framework/viur-app_server/issues classifiers = Programming Language :: Python :: 3 Operating System :: OS Independent @@ -19,11 +19,11 @@ package_dir = packages = find: python_requires = >=3.8 install_requires = - werkzeug==2.3.7 - pyyaml==6.0 - gunicorn==21.2.0 - fs==2.4.16 - grpcio==1.58.0 + werkzeug~=2.3.7 + pyyaml~=6.0 + gunicorn~=21.2.0 + fs~=2.4.16 + grpcio>=1.58.0 [options.packages.find] where = src diff --git a/src/app_server/__init__.py b/src/app_server/__init__.py index a32fe59..3b601a6 100644 --- a/src/app_server/__init__.py +++ b/src/app_server/__init__.py @@ -1,33 +1,45 @@ -import sys, os, re, subprocess, yaml, argparse, time, mimetypes, logging +import argparse +import logging +import mimetypes +import os +import re +import subprocess +import sys +import time import typing as t -from werkzeug.wrappers import Request, Response -from werkzeug.middleware.shared_data import SharedDataMiddleware -from werkzeug.middleware.http_proxy import ProxyMiddleware + +import yaml +from werkzeug._internal import _logger # noqa +from werkzeug.http import http_date, is_resource_modified from werkzeug.middleware.dispatcher import DispatcherMiddleware -from werkzeug.serving import run_simple, WSGIRequestHandler,_ansi_style,_log_add_style -from werkzeug.wsgi import get_path_info, wrap_file +from werkzeug.middleware.http_proxy import ProxyMiddleware +from werkzeug.middleware.shared_data import SharedDataMiddleware +from werkzeug.serving import run_simple, WSGIRequestHandler, _ansi_style, \ + _log_add_style +from werkzeug.urls import uri_to_iri from werkzeug.utils import get_content_type -from werkzeug.http import http_date, is_resource_modified -from werkzeug._internal import _logger -from werkzeug.urls import uri_to_iri, url_unquote - - +from werkzeug.wrappers import Request, Response +from werkzeug.wsgi import get_path_info, wrap_file -__version__ = "0.9.6" +__version__ = "0.9.10" subprocesses = [] -class myWSGIRequestHandler(WSGIRequestHandler): + +class MainWSGIRequestHandler(WSGIRequestHandler): def log_date_time_string(self): """Return the current time formatted for logging.""" now = time.time() year, month, day, hh, mm, ss, x, y, z = time.localtime(now) s = "%04d-%02d-%02d %02d:%02d:%02d" % ( - year , month,day , hh, mm, ss) + year, month, day, hh, mm, ss) return s - def log_request(self, code: t.Union[int, str] = "-", size: t.Union[int, str] = "-") -> None: - + def log_request( + self, + code: t.Union[int, str] = "-", + size: t.Union[int, str] = "-", + ) -> None: """coloring the status code""" try: path = uri_to_iri(self.path) @@ -39,7 +51,7 @@ def log_request(self, code: t.Union[int, str] = "-", size: t.Union[int, str] = " code = str(code) log_type = "info" - if code != "200": #possibility to filter 200 requests + if code != "200": # possibility to filter 200 requests log_type = "warning" if _log_add_style: @@ -68,10 +80,11 @@ def log(self, type: str, message: str, *args) -> None: _logger.setLevel(logging.INFO) _logger.addHandler(logging.StreamHandler()) - getattr(_logger, type)(f"[{self.log_date_time_string()}] {message % args}") + getattr(_logger, type)( + f"[{self.log_date_time_string()}] {message % args}") -class WrappingApp(object): +class WrappingApp: """simple wrapping app""" def __init__(self, config): @@ -79,13 +92,15 @@ def __init__(self, config): def wsgi_app(self, environ, start_response): request = Request(environ) - response = Response(f'Path not found or invalid: {request.path}', status=404) + response = Response(f'Path not found or invalid: {request.path}', + status=404) return response(environ, start_response) def __call__(self, environ, start_response): return self.wsgi_app(environ, start_response) -class myProxy(ProxyMiddleware): + +class Proxy(ProxyMiddleware): """this addition allows to redirect all routes to given targets""" def __init__(self, app, targets, chunk_size=2 << 13, timeout=10): @@ -102,7 +117,8 @@ def _set_defaults(opts): f"{k}": _set_defaults(v) for k, v in targets.items() } - def __call__(self, environ: "WSGIEnvironment", start_response: "StartResponse") -> t.Iterable[bytes]: + def __call__(self, environ: "WSGIEnvironment", + start_response: "StartResponse") -> t.Iterable[bytes]: path = get_path_info(environ, charset='utf-8', errors='replace') app = self.app for prefix, opts in self.targets.items(): @@ -112,7 +128,8 @@ def __call__(self, environ: "WSGIEnvironment", start_response: "StartResponse") return app(environ, start_response) -class myDispatcher(DispatcherMiddleware): + +class Dispatcher(DispatcherMiddleware): """use regex to find a matching route""" def __call__(self, environ, start_response): @@ -125,20 +142,21 @@ def __call__(self, environ, start_response): return app(environ, start_response) -class mySharedData(SharedDataMiddleware): +class SharedData(SharedDataMiddleware): """use regex to find a matching files""" def __init__( - self, - app, - exports, - disallow: None = None, - cache: bool = True, - cache_timeout: int = 60 * 60 * 12, - fallback_mimetype: str = "application/octet-stream", + self, + app, + exports, + disallow: None = None, + cache: bool = True, + cache_timeout: int = 60 * 60 * 12, + fallback_mimetype: str = "application/octet-stream", ) -> None: self.org_exports = exports.copy() - super().__init__(app, exports, disallow, cache, cache_timeout, fallback_mimetype) + super().__init__(app, exports, disallow, cache, cache_timeout, + fallback_mimetype) def __call__(self, environ, start_response): path = get_path_info(environ) @@ -147,8 +165,10 @@ def __call__(self, environ, start_response): for search_path, loader in self.exports: # lets check for regex, and inject real_path if re.match(search_path, path): - real_path = re.sub(search_path, self.org_exports[search_path], path, 1) - real_filename, file_loader = self.get_file_loader(real_path)(None) + real_path = re.sub(search_path, self.org_exports[search_path], + path, 1) + real_filename, file_loader = self.get_file_loader(real_path)( + None) if file_loader is not None: break @@ -168,11 +188,12 @@ def __call__(self, environ, start_response): if file_loader is not None: break - if file_loader is None or not self.is_allowed(real_filename): # type: ignore + if file_loader is None or not self.is_allowed(real_filename): # noqa return self.app(environ, start_response) guessed_type = mimetypes.guess_type(real_filename) # type: ignore - mime_type = get_content_type(guessed_type[0] or self.fallback_mimetype, "utf-8") + mime_type = get_content_type(guessed_type[0] or self.fallback_mimetype, + "utf-8") try: f, mtime, file_size = file_loader() @@ -183,7 +204,8 @@ def __call__(self, environ, start_response): if self.cache: timeout = self.cache_timeout - etag = self.generate_etag(mtime, file_size, real_filename) # type: ignore + etag = self.generate_etag(mtime, file_size, + real_filename) # type: ignore headers += [ ("Etag", f'"{etag}"'), ("Cache-Control", f"max-age={timeout}, public"), @@ -209,13 +231,21 @@ def __call__(self, environ, start_response): return wrap_file(environ, f) -def start_server(host, port, gunicorn_port, appFolder, appYaml, timeout, protocol="http"): +def start_server( + host: str, + port: int, + gunicorn_port: int, + app_folder: str, + app_yaml: dict, + timeout: int, + protocol: str = "http", +) -> None: """use the dispatcherMiddleware to connect SharedDataMiddleware and ProxyMiddleware with the wrapping app.""" app = WrappingApp({}) apps = {} # make shared middlewares for static files as configured in app.yaml - for route in appYaml["handlers"]: + for route in app_yaml["handlers"]: if path := route.get("static_dir"): pattern = route["url"] + "/.*" @@ -226,21 +256,32 @@ def start_server(host, port, gunicorn_port, appFolder, appYaml, timeout, protoco continue # skip # print(pattern, route["url"], path) - apps[pattern] = mySharedData(app.wsgi_app, {route["url"]: os.path.join(appFolder, path)}) + apps[pattern] = SharedData( + app.wsgi_app, {route["url"]: os.path.join(app_folder, path)} + ) - apps.update({"/": myProxy(app.wsgi_app, { + apps["/"] = Proxy(app.wsgi_app, { "/": { "target": f"{protocol}://{host}:{gunicorn_port}/", - "host": f"{host}:{port}" + "host": None } - },timeout=timeout)}) - app.wsgi_app = myDispatcher(app.wsgi_app, apps) + }, timeout=timeout) + app.wsgi_app = Dispatcher(app.wsgi_app, apps) + + run_simple(host, port, app, use_debugger=False, use_reloader=True, + threaded=True, request_handler=MainWSGIRequestHandler) - run_simple(host, port, app, use_debugger=False, use_reloader=True, threaded=True, request_handler=myWSGIRequestHandler) +def set_env_vars(application_id: str, args: argparse.Namespace, app_yaml: dict): + """set necessary environment variables""" + # First, merge the app.yaml into the environment so that the variables + # from the CLI can overwrite it. + if env_vars := app_yaml.get("env_variables"): + if not isinstance(env_vars, dict): + raise TypeError( + f"env_variables section in app.yaml must be a dict. Got {type(env_vars)}") + os.environ |= {k: str(v) for k, v in app_yaml["env_variables"].items()} -def envVars(application_id, args): - """set nessesary environment variables""" os.environ["GAE_ENV"] = "localdev" os.environ["CLOUDSDK_CORE_PROJECT"] = application_id os.environ["GOOGLE_CLOUD_PROJECT"] = application_id @@ -248,11 +289,17 @@ def envVars(application_id, args): os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "0" if args.storage: - os.environ["STORAGE_EMULATOR_HOST"] = f"http://{args.host}:{args.storage_port}" + os.environ["STORAGE_EMULATOR_HOST"] = \ + f"http://{args.host}:{args.storage_port}" if args.tasks: os.environ["TASKS_EMULATOR"] = f"{args.host}:{args.tasks_port}" + # Merge environment variables from CLI parameter + if args.env_var: + os.environ |= dict(v.split("=", 1) for v in args.env_var) + + def patch_gunicorn(): import gunicorn.workers.base with open(gunicorn.workers.base.__file__, 'r+') as file: @@ -268,26 +315,35 @@ def patch_gunicorn(): )) -def start_gunicorn(args, appYaml, appFolder, myFolder): +def start_gunicorn( + args: argparse.Namespace, + app_yaml: dict, + app_folder: str, +) -> None: # Gunicorn call command - entrypoint = appYaml.get("entrypoint", "gunicorn -b :$PORT -w $WORKER --threads $THREADS " - "--disable-redirect-access-to-syslog main:app") - for var, value in { - "PORT": args.gunicorn_port, - "WORKER": args.worker, - "THREADS": args.threads - }.items(): - entrypoint = entrypoint.replace(f"${var}", str(value)) + if not (entrypoint := args.entrypoint): + entrypoint = app_yaml.get( + "entrypoint", + "gunicorn -b :$PORT --disable-redirect-access-to-syslog main:app" + ) + entrypoint = entrypoint.replace(f"$PORT", str(args.gunicorn_port)) + # Remove -w / --workers / --threads arguments, + # we set them later with the values from our argparser + entrypoint = re.sub(r"\s+-(w|-workers|-threads)\s+\d+", " ", entrypoint) entrypoint = entrypoint.split() + entrypoint.extend(["--workers", str(args.workers)]) + entrypoint.extend(["--threads", str(args.threads)]) + if "--reload" not in entrypoint: entrypoint.insert(1, "--reload") if "--reuse-port" not in entrypoint: entrypoint.insert(1, "--reuse-port") - os.chdir(appFolder) - subprocesses.append(subprocess.Popen(entrypoint)) - os.chdir(myFolder) + entrypoint.extend(["--timeout", str(args.timeout)]) + + subprocesses.append(subprocess.Popen(entrypoint, cwd=app_folder)) + def main(): """main entrypoint @@ -298,66 +354,98 @@ def main(): start wrapping app """ ap = argparse.ArgumentParser( - description="alternative dev_appserver" + description="alternative dev_appserver", + epilog=f"Version: {__version__}" ) - ap.add_argument("config_paths", metavar='yaml_path', nargs='+', help='Path to app.yaml file') + ap.add_argument("config_paths", metavar='yaml_path', nargs='+', + help='Path to app.yaml file') ap.add_argument( - '-A', '--application', action='store', dest='app_id', required=True, help='Set the application id') - ap.add_argument('--host', default="localhost", help='host name to which application modules should bind') - ap.add_argument('--port', type=int, default=8080, help='port to which we bind the application') - ap.add_argument('--gunicorn_port', type=int, default=8090, help='internal gunicorn port') - ap.add_argument('--worker', type=int, default=1, help='amount of gunicorn workers') - ap.add_argument('--threads', type=int, default=5, help='amount of gunicorn threads') - ap.add_argument('--timeout', type=int, default=60, help='Time is seconds befor gunicorn abort a rquest') - ap.add_argument('-V', '--version', action='version', version='%(prog)s ' + __version__) - - ap.add_argument('--storage', default=False, action="store_true", dest="storage", help="also start Storage Emulator") - ap.add_argument('--storage_port', type=int, default=8092, help='internal Storage Emulator Port') + '-A', '--application', action='store', dest='app_id', required=True, + help='Set the application id') + ap.add_argument('--host', default="localhost", + help='host name to which application modules should bind') + ap.add_argument('--entrypoint', type=str, default=None, + help='The entrypoint is the basic gunicorn command. By default, it\'s taken from app.yaml. ' + 'This parameter can be used to set a different entrypoint. ' + 'To provide this parameter via ViUR-CLI, you have to double quote it: ' + ' --entrypoint "\'gunicorn -b :\$PORT --disable-redirect-access-to-syslog main:app\'"') + ap.add_argument('--port', type=int, default=8080, + help='port to which we bind the application') + ap.add_argument('--gunicorn_port', type=int, default=8090, + help='internal gunicorn port') + ap.add_argument('--workers', '--worker', type=int, default=1, + help='amount of gunicorn workers') + ap.add_argument('--threads', type=int, default=5, + help='amount of gunicorn threads') + ap.add_argument('--timeout', type=int, default=60, + help='Time is seconds before gunicorn abort a request') + ap.add_argument('-V', '--version', action='version', + version='%(prog)s ' + __version__) + + ap.add_argument('--storage', default=False, action="store_true", + dest="storage", help="also start Storage Emulator") + ap.add_argument('--storage_port', type=int, default=8092, + help='internal Storage Emulator Port') + + ap.add_argument('--tasks', default=False, action='store_true', dest="tasks", + help='also start Task-Queue Emulator') + ap.add_argument('--tasks_port', type=int, default=8091, + help='internal Task-Queue Emulator Port') + + ap.add_argument('--cron', default=False, action='store_true', dest="cron", + help='also start Cron Emulator') - ap.add_argument('--tasks', default=False, action='store_true', dest="tasks", help='also start Task-Queue Emulator') - ap.add_argument('--tasks_port', type=int, default=8091, help='internal Task-Queue Emulator Port') - - ap.add_argument('--cron', default=False, action='store_true', dest="cron", help='also start Cron Emulator') + ap.add_argument( + '--env_var', metavar="KEY=VALUE", nargs="*", + help="Set environment variable for the runtime. Each env_var is in " + "the format of KEY=VALUE, and you can define multiple " + "environment variables. You can also define them in app.yaml." + ) args = ap.parse_args() - envVars(args.app_id, args) - patch_gunicorn() - myFolder = os.getcwd() - appFolder = os.path.abspath(args.config_paths[0]) + + app_folder = os.path.abspath(args.config_paths[0]) # load & parse the app.yaml - with open(os.path.join(appFolder, "app.yaml"), "r") as f: - appYaml = yaml.load(f, Loader=yaml.Loader) + with open(os.path.join(app_folder, "app.yaml"), "r") as f: + app_yaml = yaml.load(f, Loader=yaml.Loader) + + set_env_vars(args.app_id, args, app_yaml) + patch_gunicorn() # Check for correct runtime - myRuntime = f"python{sys.version_info.major}{sys.version_info.minor}" - appRuntime = appYaml["runtime"] - assert appRuntime == myRuntime, f"app.yaml specifies {appRuntime} but you're on {myRuntime}, please correct this." + current_runtime = f"python{sys.version_info.major}{sys.version_info.minor}" + app_runtime = app_yaml["runtime"] + assert app_runtime == current_runtime, f"app.yaml specifies {app_runtime} but you're on {current_runtime}, please correct this." if "WERKZEUG_RUN_MAIN" in os.environ and os.environ["WERKZEUG_RUN_MAIN"]: - #only start subprocesses wenn reloader starts + # only start subprocesses wenn reloader starts if args.storage: - storage_subprocess =subprocess.Popen( - f"gcloud-storage-emulator start --port={args.storage_port} --default-bucket={args.app_id}.appspot.com".split()) + storage_subprocess = subprocess.Popen( + f"gcloud-storage-emulator start --port={args.storage_port}" + f" --default-bucket={args.app_id}.appspot.com".split()) subprocesses.append(storage_subprocess) - - if args.tasks and os.path.exists(os.path.join(appFolder, 'queue.yaml')): + if args.tasks and os.path.exists( + os.path.join(app_folder, 'queue.yaml')): cron = "" if args.cron: - cron = f"--cron-yaml={os.path.join(appFolder, 'cron.yaml')}" + cron = f"--cron-yaml={os.path.join(app_folder, 'cron.yaml')}" tasks_subprocess = subprocess.Popen( - f"gcloud-tasks-emulator start -p={args.tasks_port} -t={args.port} {cron} --queue-yaml={os.path.join(appFolder, 'queue.yaml')} --queue-yaml-project={args.app_id} --queue-yaml-location=local -r 50".split()) + f"gcloud-tasks-emulator start -p={args.tasks_port} -t={args.port} {cron}" + f" --queue-yaml={os.path.join(app_folder, 'queue.yaml')}" + f" --queue-yaml-project={args.app_id} --queue-yaml-location=local -r 50".split()) subprocesses.append(tasks_subprocess) - start_gunicorn(args, appYaml, appFolder, myFolder) + start_gunicorn(args, app_yaml, app_folder) - start_server(args.host, args.port, args.gunicorn_port, appFolder, appYaml, args.timeout) + start_server(args.host, args.port, args.gunicorn_port, app_folder, app_yaml, + args.timeout) try: for process in subprocesses: @@ -368,4 +456,3 @@ def main(): if __name__ == '__main__': main() -