diff --git a/.dockerignore b/.dockerignore index b457b49..e2ce317 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,6 +9,7 @@ env/ exports/ .editorconfig +.env .env.example .gitignore .markdownlint.json diff --git a/Makefile b/Makefile index 80574b9..59a5a1b 100644 --- a/Makefile +++ b/Makefile @@ -5,24 +5,21 @@ client: @cd client && npm start build: - @echo "Building the server..." - @docker-compose -f docker-compose.local.yml build + @echo "Building the server image..." + @docker build -t michaelfromyeg/bereal_wrapped -f docker/Dockerfile.server . + @docker push michaelfromyeg/bereal_wrapped up: @echo "Booting up the server..." - @docker-compose -f docker-compose.local.yml up -d + @docker stack deploy -c docker-stack.local.yml bereal-wrapped logs: @echo "Showing the server logs..." - @docker-compose -f docker-compose.local.yml logs -f + @docker service logs -f bereal-wrapped_web down: @echo "Shutting down the server..." - @docker-compose -f docker-compose.local.yml down - -kill: - @echo "Killing the server..." - @docker-compose -f docker-compose.local.yml kill + @docker stack rm bereal-wrapped start-redis: @echo "Booting up Redis..." diff --git a/bereal/send.py b/bereal/send.py index fa1f694..a256b74 100644 --- a/bereal/send.py +++ b/bereal/send.py @@ -16,14 +16,12 @@ def sms(phone: str, link: str) -> None: try: client = Client(TWILIO_ACCOUNT_SID, TWILIO_AUTH_TOKEN) - if FLASK_ENV == "development": - logger.info("Skipping SMS in development mode") - return - message_body = f"Here is the link to your BeReal Wrapped!\n\n{link}" - message = client.messages.create(body=message_body, from_=TWILIO_PHONE_NUMBER, to=phone) - - logger.info("Sent message to %s: %s", phone, message.sid) + if FLASK_ENV == "development": + logger.info("Skipping SMS in development mode; would send %s", message_body) + else: + message = client.messages.create(body=message_body, from_=TWILIO_PHONE_NUMBER, to=phone) + logger.info("Sent message %s to %s", message.sid, phone) except Exception as e: logger.error("Failed to send SMS: %s", e) pass diff --git a/bereal/server.py b/bereal/server.py index bc254f7..dfae9b5 100644 --- a/bereal/server.py +++ b/bereal/server.py @@ -151,7 +151,7 @@ def validate_otp() -> tuple[Response, int]: return jsonify({"error": "Bad Request", "message": "Invalid verification code"}), 400 # generate a custom app token; this we can safely save in our DB - bereal_token = secrets.token_urlsafe(20) + bereal_token = secrets.token_hex(10) insert_bereal_token(phone, bereal_token) @@ -317,8 +317,11 @@ def delete_expired_tokens() -> None: """ Delete all expired tokens from the database. """ - expiration_time = datetime.utcnow() - timedelta(hours=24) - BerealToken.query.filter(BerealToken.timestamp < expiration_time).delete() + expiration_time = datetime.utcnow() - timedelta(days=1) + + n = BerealToken.query.filter(BerealToken.timestamp < expiration_time).delete() + logger.info("Deleted %d expired tokens", n) + db.session.commit() return None @@ -328,7 +331,7 @@ def delete_old_videos() -> None: """ Delete videos that are more than a day old. """ - time_limit = datetime.now() - timedelta(days=1) + expiration_time = datetime.utcnow() - timedelta(days=1) for filename in os.listdir(EXPORTS_PATH): file_path = os.path.join(EXPORTS_PATH, filename) @@ -336,7 +339,7 @@ def delete_old_videos() -> None: if os.path.isfile(file_path): file_mod_time = datetime.fromtimestamp(os.path.getmtime(file_path)) - if file_mod_time < time_limit: + if file_mod_time < expiration_time: try: os.remove(file_path) logger.info("Deleted video file %s", file_path) diff --git a/bereal/utils.py b/bereal/utils.py index 41729bd..c3ab42a 100644 --- a/bereal/utils.py +++ b/bereal/utils.py @@ -7,22 +7,30 @@ from datetime import datetime from enum import StrEnum -from dotenv import load_dotenv - from .logger import logger + # Environment variables -load_dotenv() +def get_secret(secret_name: str) -> str | None: + """ + Get a Docker Swarm secret. + """ + try: + with open(f"/run/secrets/{secret_name}", "r") as secret_file: + return secret_file.read().strip() + except IOError: + return None + -SECRET_KEY = os.getenv("SECRET_KEY") or "SECRET_KEY" -FLASK_ENV = os.getenv("FLASK_ENV") or "production" +SECRET_KEY = get_secret("secret_key") or "SECRET_KEY" +FLASK_ENV = get_secret("flask_env") or "production" if SECRET_KEY == "SECRET_KEY": raise ValueError("SECRET_KEY environment variable not set or non-unique") -TWILIO_PHONE_NUMBER = os.getenv("TWILIO_PHONE_NUMBER") -TWILIO_AUTH_TOKEN = os.getenv("TWILIO_AUTH_TOKEN") -TWILIO_ACCOUNT_SID = os.getenv("TWILIO_ACCOUNT_SID") +TWILIO_PHONE_NUMBER = get_secret("twilio_phone_number") +TWILIO_AUTH_TOKEN = get_secret("twilio_auth_token") +TWILIO_ACCOUNT_SID = get_secret("twilio_account_sid") if TWILIO_PHONE_NUMBER is None or TWILIO_AUTH_TOKEN is None or TWILIO_ACCOUNT_SID is None: raise ValueError("TWILIO environment variables not set") @@ -90,14 +98,14 @@ def str2mode(s: str | None) -> Mode: config.read("config.ini") -HOST: str | None = os.getenv("HOST") or "localhost" -PORT: str | None = os.getenv("PORT") or "5000" +HOST: str | None = get_secret("host") or "localhost" +PORT: str | None = get_secret("port") or "5000" PORT = int(PORT) if PORT is not None else None TRUE_HOST = f"http://{HOST}:{PORT}" if FLASK_ENV == "development" else "https://api.bereal.michaeldemar.co" -REDIS_HOST: str | None = os.getenv("REDIS_HOST") or "redis" -REDIS_PORT: str | None = os.getenv("REDIS_PORT") or "6379" +REDIS_HOST: str | None = get_secret("redis_host") or "redis" +REDIS_PORT: str | None = get_secret("redis_port") or "6379" REDIS_PORT = int(REDIS_PORT) if REDIS_PORT is not None else None TIMEOUT = config.getint("bereal", "timeout", fallback=10) diff --git a/content/.gitkeep b/content/.gitkeep old mode 100644 new mode 100755 diff --git a/docker-compose.local.yml b/docker-compose.local.yml index e4e0833..be4d2c0 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -5,10 +5,12 @@ services: build: context: . dockerfile: docker/Dockerfile.server + image: bereal_web volumes: - ./exports:/app/exports - ./content:/app/content user: thekid + command: web ports: - "5000:5000" environment: @@ -17,14 +19,12 @@ services: - redis celery: - build: - context: . - dockerfile: docker/Dockerfile.celery + image: bereal_web volumes: - ./exports:/app/exports - ./content:/app/content user: thekid - command: celery -A bereal.celery worker --loglevel=INFO --logfile=celery.log -E + command: celery environment: - FLASK_APP=bereal.server depends_on: @@ -32,15 +32,13 @@ services: - redis flower: - build: - context: . - dockerfile: docker/Dockerfile.flower + image: bereal_web volumes: - ./exports:/app/exports user: thekid ports: - "5555:5555" - command: celery -A bereal.celery flower --address=0.0.0.0 --inspect --enable-events --loglevel=DEBUG --logfile=flower.log + command: flower environment: - FLASK_APP=bereal.server depends_on: diff --git a/docker-stack.local.yml b/docker-stack.local.yml new file mode 100644 index 0000000..c27089a --- /dev/null +++ b/docker-stack.local.yml @@ -0,0 +1,106 @@ +version: "3.8" + +services: + web: + image: michaelfromyeg/bereal_wrapped + volumes: + - exports:/app/exports + - content:/app/content + command: web + ports: + - "5000:5000" + environment: + - FLASK_APP=bereal.server + secrets: + - flask_env + - secret_key + - host + - port + - redis_host + - redis_port + - twilio_phone_number + - twilio_auth_token + - twilio_account_sid + deploy: + replicas: 1 + + celery: + image: michaelfromyeg/bereal_wrapped + volumes: + - exports:/app/exports + - content:/app/content + command: celery + environment: + - FLASK_APP=bereal.server + secrets: + - flask_env + - secret_key + - host + - port + - redis_host + - redis_port + - twilio_phone_number + - twilio_auth_token + - twilio_account_sid + deploy: + replicas: 1 + + flower: + image: michaelfromyeg/bereal_wrapped + volumes: + - exports:/app/exports + - content:/app/content + ports: + - "5555:5555" + command: flower + environment: + - FLASK_APP=bereal.server + secrets: + - flask_env + - secret_key + - host + - port + - redis_host + - redis_port + - twilio_phone_number + - twilio_auth_token + - twilio_account_sid + deploy: + replicas: 1 + + redis: + image: "redis:alpine" + configs: + - source: redis_conf + target: /usr/local/etc/redis/redis.conf + command: redis-server /usr/local/etc/redis/redis.conf + deploy: + replicas: 1 + +volumes: + exports: + content: + +configs: + redis_conf: + file: ./redis/redis.conf + +secrets: + flask_env: + external: true + secret_key: + external: true + host: + external: true + port: + external: true + redis_host: + external: true + redis_port: + external: true + twilio_phone_number: + external: true + twilio_auth_token: + external: true + twilio_account_sid: + external: true diff --git a/docker-stack.yml b/docker-stack.yml new file mode 100644 index 0000000..c250225 --- /dev/null +++ b/docker-stack.yml @@ -0,0 +1,100 @@ +version: "3.8" + +services: + web: + image: michaelfromyeg/bereal_wrapped:latest + volumes: + - /mnt/videos:/app/exports + - /mnt/content:/app/content + ports: + - "5000:5000" + environment: + - FLASK_APP=bereal.server + secrets: + - flask_env + - secret_key + - host + - port + - redis_host + - redis_port + - twilio_phone_number + - twilio_auth_token + - twilio_account_sid + deploy: + resources: + limits: + memory: 500M + replicas: 1 + + celery: + image: michaelfromyeg/bereal_wrapped:latest + volumes: + - /mnt/videos:/app/exports + - /mnt/content:/app/content + command: celery -A bereal.celery worker --loglevel=INFO --logfile=celery.log -E -c 1 + environment: + - FLASK_APP=bereal.server + secrets: + - flask_env + - secret_key + - host + - port + - redis_host + - redis_port + - twilio_phone_number + - twilio_auth_token + - twilio_account_sid + deploy: + resources: + limits: + memory: 3G + replicas: 1 + + redis: + image: "redis:alpine" + volumes: + - ./redis/redis.conf:/usr/local/etc/redis/redis.conf + command: redis-server /usr/local/etc/redis/redis.conf + deploy: + replicas: 1 + + nginx: + image: "nginx:alpine" + ports: + - "80:80" + - "443:443" + volumes: + - ./nginx:/etc/nginx/conf.d + - ./nginx/robots.txt:/usr/share/nginx/html/robots.txt + - /var/log/nginx:/var/log/nginx + - /etc/letsencrypt:/etc/letsencrypt + deploy: + replicas: 1 + +volumes: + exports: + content: + +configs: + redis_conf: + file: ./redis/redis.conf + +secrets: + flask_env: + external: true + secret_key: + external: true + host: + external: true + port: + external: true + redis_host: + external: true + redis_port: + external: true + twilio_phone_number: + external: true + twilio_auth_token: + external: true + twilio_account_sid: + external: true diff --git a/docker/Dockerfile.server b/docker/Dockerfile.server index d51fd5c..e8be343 100644 --- a/docker/Dockerfile.server +++ b/docker/Dockerfile.server @@ -5,11 +5,15 @@ WORKDIR /app COPY . . COPY --chmod=+x ./scripts/server-entrypoint.sh . +ENV PIP_NO_WARN_SCRIPT_LOCATION=0 + RUN apt-get update && apt-get install -y git && \ pip install --upgrade pip && \ - pip install --no-cache-dir -r requirements/prod.txt && \ - adduser --disabled-password --gecos '' thekid && \ - chown -R thekid:thekid /app + pip install --no-cache-dir -r requirements/prod.txt + +RUN addgroup --gid 1000 thekids && \ + adduser --disabled-password --gecos '' --uid 1000 --gid 1000 thekid && \ + chown -R thekid:thekids /app USER thekid diff --git a/exports/.gitkeep b/exports/.gitkeep old mode 100644 new mode 100755 diff --git a/logger.ini b/logger.ini index 280fccc..5b35368 100644 --- a/logger.ini +++ b/logger.ini @@ -2,33 +2,33 @@ keys=root,berealLogger [handlers] -keys=consoleHandler,fileHandler +keys=consoleHandler,timedRotatingFileHandler [formatters] -keys=sampleFormatter +keys=berealFormatter [logger_root] level=DEBUG -handlers=consoleHandler,fileHandler +handlers=consoleHandler,timedRotatingFileHandler [logger_berealLogger] level=DEBUG -handlers=consoleHandler,fileHandler +handlers=consoleHandler,timedRotatingFileHandler qualname=berealLogger propagate=0 [handler_consoleHandler] class=StreamHandler level=INFO -formatter=sampleFormatter +formatter=berealFormatter args=(sys.stdout,) -[handler_fileHandler] -class=FileHandler +[handler_timedRotatingFileHandler] +class=logging.handlers.TimedRotatingFileHandler level=DEBUG -formatter=sampleFormatter -args=('log.log', 'w') +formatter=berealFormatter +args=('log.log', 'midnight', 1, 3) -[formatter_sampleFormatter] +[formatter_berealFormatter] format=[%(asctime)s] (%(levelname)s) %(module)s:%(lineno)d %(message)s datefmt=%Y-%m-%d %H:%M:%S diff --git a/requirements/common.txt b/requirements/common.txt index cdbbf44..a291f64 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -1,7 +1,7 @@ celery[redis]==5.3.6 Flask==3.0.0 Flask-APScheduler==1.13.1 -flask-cors==4.0.0 +Flask-CORS==4.0.0 Flask-Limiter==3.5.0 Flask-Migrate==4.0.5 Flask-SQLAlchemy==3.1.1 @@ -10,10 +10,6 @@ gevent==23.9.1 gunicorn==21.2.0 librosa==0.10.1 moviepy==2.0.0.dev2 -numpy==1.26.2 Pillow==10.1.0 -pydub==0.25.1 -python-dotenv==1.0.0 Requests==2.31.0 -tqdm==4.66.1 twilio==8.11.0 diff --git a/scripts/server-entrypoint.sh b/scripts/server-entrypoint.sh index c7a8312..57e0d3c 100755 --- a/scripts/server-entrypoint.sh +++ b/scripts/server-entrypoint.sh @@ -1,15 +1,26 @@ #!/bin/sh -# Apply database migrations -echo "Running database migrations" -flask db upgrade +if [ "$1" = "web" ]; then + echo "Running database migrations" + flask db upgrade -# Check for success of migrations -if [ $? -ne 0 ]; then - echo "Failed to apply database migrations" + if [ $? -ne 0 ]; then + echo "Failed to apply database migrations" + exit 1 + fi + + echo "Starting the main application" + exec gunicorn -b :5000 -k gevent -w 4 -t 600 "bereal.server:app" + +elif [ "$1" = "celery" ]; then + echo "Starting Celery worker" + exec celery -A bereal.celery worker --loglevel=INFO --logfile=celery.log -E + +elif [ "$1" = "flower" ]; then + echo "Starting Celery Flower" + exec celery -A bereal.celery flower --address=0.0.0.0 --inspect --enable-events --loglevel=DEBUG --logfile=flower.log + +else + echo "Invalid argument. Please use 'web', 'celery', or 'flower'." exit 1 fi - -# Start the main process (gunicorn in your case) -echo "Starting the main application" -exec gunicorn -b :5000 -k gevent -w 4 -t 600 "bereal.server:app" diff --git a/scripts/volumes.sh b/scripts/volumes.sh new file mode 100644 index 0000000..9c88ac2 --- /dev/null +++ b/scripts/volumes.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +docker run --rm -v bereal-wrapped_content:/data ubuntu chown -R 1000:1000 /data +docker run --rm -v bereal-wrapped_exports:/data ubuntu chown -R 1000:1000 /data