diff --git a/.gitignore b/.gitignore index 7db91e63..e8de469f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,6 @@ *.*~ # project specifc -django_project/core/settings/secret.py xmlrunner # Byte-compiled / optimized / DLL files @@ -107,9 +106,6 @@ deployment/docker-compose.override.yml !deployment/sftp_backup/target/pg/.gitkeep !deployment/sftp_backup/backups/media/README.md !deployment/sftp_backup/backups/pg/README.md -django_project/realtime/tasks/realtime/celeryconfig.py -django_project/core/settings/custom_settings.py -django_project/core/settings/mapquest.py django_project/realtime/local_settings.py django_project/celerybeat.pid django_project/temp diff --git a/deployment/ansible/development/roles/docker_compose/templates/docker-compose.override.yml.j2 b/deployment/ansible/development/roles/docker_compose/templates/docker-compose.override.yml.j2 index af5696c4..de5e54ab 100644 --- a/deployment/ansible/development/roles/docker_compose/templates/docker-compose.override.yml.j2 +++ b/deployment/ansible/development/roles/docker_compose/templates/docker-compose.override.yml.j2 @@ -34,6 +34,10 @@ uwsgi: - VIRTUAL_HOST=users.inasafe.org - VIRTUAL_PORT=8080 - BROKER_URL=amqp://guest:guest@rabbitmq:5672/ + - INASAFE_REALTIME_BROKER_URL=amqp://guest:guest@{{ interpreters.inasafe_realtime.ipaddress }}:{{ docker_port_forward.inasafe_realtime.rabbitmq }}/ + # Secrets + - MAPQUEST_MAP_KEY={{ inasafe_django.mapquest_map_key }} + - SITE_DOMAIN_NAME={{ inasafe_django.site_domain_name }} links: - smtp:smtp - db:db diff --git a/deployment/ansible/development/roles/inasafe_django/tasks/main.yml b/deployment/ansible/development/roles/inasafe_django/tasks/main.yml deleted file mode 100644 index 3087eced..00000000 --- a/deployment/ansible/development/roles/inasafe_django/tasks/main.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -- name: customize django settings file and secret - template: - src: '{{ item }}.j2' - dest: '{{ project_path }}/django_project/core/settings/{{ item }}' - owner: '{{ remote_user }}' - group: '{{ remote_group }}' - mode: "u=rw,g=rw,o=r" - with_items: - - custom_settings.py - - mapquest.py - - secret.py - -- name: customize InaSAFE Realtime celery config - template: - src: 'celeryconfig.py.j2' - dest: '{{ project_path }}/django_project/realtime/tasks/realtime/celeryconfig.py' - owner: '{{ remote_user }}' - group: '{{ remote_group }}' - mode: "u=rw,g=rw,o=r" diff --git a/deployment/ansible/development/roles/inasafe_django/templates/celeryconfig.py.j2 b/deployment/ansible/development/roles/inasafe_django/templates/celeryconfig.py.j2 deleted file mode 100644 index 2dee45b6..00000000 --- a/deployment/ansible/development/roles/inasafe_django/templates/celeryconfig.py.j2 +++ /dev/null @@ -1,20 +0,0 @@ -# coding=utf-8 -"""Sample file configuration for celery_app worker - -This file is intended only for a sample. -Please copy it as celeryconfig.py so it can be read -""" - -__author__ = 'Rizky Maulana Nugraha ' -__date__ = '1/28/16' - - -BROKER_URL = 'amqp://guest:guest@{{ interpreters.inasafe_realtime.ipaddress }}:{{ docker_port_forward.inasafe_realtime.rabbitmq }}/' - -CELERY_RESULT_BACKEND = BROKER_URL - -CELERY_ROUTES = { - 'realtime.tasks.flood': { - 'queue': 'inasafe-realtime' - } -} diff --git a/deployment/ansible/development/roles/inasafe_django/templates/custom_settings.py.j2 b/deployment/ansible/development/roles/inasafe_django/templates/custom_settings.py.j2 deleted file mode 100644 index aa18ebf3..00000000 --- a/deployment/ansible/development/roles/inasafe_django/templates/custom_settings.py.j2 +++ /dev/null @@ -1,6 +0,0 @@ -# coding=utf-8 - -# We can use this files to override the settings -# This file will be imported at the end of the settings phase. - -SITE_DOMAIN_NAME = '{{ inasafe_django.site_domain_name }}' diff --git a/deployment/ansible/development/roles/inasafe_django/templates/mapquest.py.j2 b/deployment/ansible/development/roles/inasafe_django/templates/mapquest.py.j2 deleted file mode 100644 index 67cea788..00000000 --- a/deployment/ansible/development/roles/inasafe_django/templates/mapquest.py.j2 +++ /dev/null @@ -1,6 +0,0 @@ -# coding=utf-8 - -# This file is used to configure mapquest key -# Copy this file as "mapquest.py" - -MAPQUEST_MAP_KEY = '{{ inasafe_django.mapquest_map_key }}' diff --git a/deployment/ansible/development/roles/inasafe_django/templates/secret.py.j2 b/deployment/ansible/development/roles/inasafe_django/templates/secret.py.j2 deleted file mode 100644 index 72ec4d83..00000000 --- a/deployment/ansible/development/roles/inasafe_django/templates/secret.py.j2 +++ /dev/null @@ -1,5 +0,0 @@ -# coding=utf-8 - -# This secret is used for sentry - -SECRET_KEY = u'{{ inasafe_django.sentry_secret_key }}' diff --git a/deployment/ansible/development/roles/pycharm/templates/run-manager-environment.xml.j2 b/deployment/ansible/development/roles/pycharm/templates/run-manager-environment.xml.j2 index a7cdc9d1..f2044135 100644 --- a/deployment/ansible/development/roles/pycharm/templates/run-manager-environment.xml.j2 +++ b/deployment/ansible/development/roles/pycharm/templates/run-manager-environment.xml.j2 @@ -1,5 +1,12 @@ + + + + + + + diff --git a/deployment/ansible/development/site.yml b/deployment/ansible/development/site.yml index fe5ad020..29a95484 100644 --- a/deployment/ansible/development/site.yml +++ b/deployment/ansible/development/site.yml @@ -5,7 +5,6 @@ roles: - { role: pycharm, when: use_pycharm, tags: development } - docker_compose - - inasafe_django vars_prompt: - name: "ansible_sudo_pass" prompt: "Sudo password" diff --git a/deployment/production/docker/db/Dockerfile b/deployment/production/docker/db/Dockerfile new file mode 100644 index 00000000..ea971a14 --- /dev/null +++ b/deployment/production/docker/db/Dockerfile @@ -0,0 +1,4 @@ +FROM kartoza/postgis:9.3-2.1 + +ADD pg_hba.conf /etc/postgresql/9.3/main/pg_hba.conf +RUN chown postgres:postgres /etc/postgresql/9.3/main/pg_hba.conf diff --git a/deployment/production/docker/db/build.sh b/deployment/production/docker/db/build.sh new file mode 100755 index 00000000..e3a74bec --- /dev/null +++ b/deployment/production/docker/db/build.sh @@ -0,0 +1,5 @@ +#!/bin/sh +IMAGE_NAME=inasafe-django_db +TAG_NAME=v3.4 +docker build -t kartoza/${IMAGE_NAME} . +docker tag kartoza/${IMAGE_NAME}:latest kartoza/${IMAGE_NAME}:${TAG_NAME} diff --git a/deployment/production/docker/db/pg_hba.conf b/deployment/production/docker/db/pg_hba.conf new file mode 100644 index 00000000..d9d97e8b --- /dev/null +++ b/deployment/production/docker/db/pg_hba.conf @@ -0,0 +1,102 @@ +# PostgreSQL Client Authentication Configuration File +# =================================================== +# +# Refer to the "Client Authentication" section in the PostgreSQL +# documentation for a complete description of this file. A short +# synopsis follows. +# +# This file controls: which hosts are allowed to connect, how clients +# are authenticated, which PostgreSQL user names they can use, which +# databases they can access. Records take one of these forms: +# +# local DATABASE USER METHOD [OPTIONS] +# host DATABASE USER ADDRESS METHOD [OPTIONS] +# hostssl DATABASE USER ADDRESS METHOD [OPTIONS] +# hostnossl DATABASE USER ADDRESS METHOD [OPTIONS] +# +# (The uppercase items must be replaced by actual values.) +# +# The first field is the connection type: "local" is a Unix-domain +# socket, "host" is either a plain or SSL-encrypted TCP/IP socket, +# "hostssl" is an SSL-encrypted TCP/IP socket, and "hostnossl" is a +# plain TCP/IP socket. +# +# DATABASE can be "all", "sameuser", "samerole", "replication", a +# database name, or a comma-separated list thereof. The "all" +# keyword does not match "replication". Access to replication +# must be enabled in a separate record (see example below). +# +# USER can be "all", a user name, a group name prefixed with "+", or a +# comma-separated list thereof. In both the DATABASE and USER fields +# you can also write a file name prefixed with "@" to include names +# from a separate file. +# +# ADDRESS specifies the set of hosts the record matches. It can be a +# host name, or it is made up of an IP address and a CIDR mask that is +# an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that +# specifies the number of significant bits in the mask. A host name +# that starts with a dot (.) matches a suffix of the actual host name. +# Alternatively, you can write an IP address and netmask in separate +# columns to specify the set of hosts. Instead of a CIDR-address, you +# can write "samehost" to match any of the server's own IP addresses, +# or "samenet" to match any address in any subnet that the server is +# directly connected to. +# +# METHOD can be "trust", "reject", "md5", "password", "gss", "sspi", +# "krb5", "ident", "peer", "pam", "ldap", "radius" or "cert". Note that +# "password" sends passwords in clear text; "md5" is preferred since +# it sends encrypted passwords. +# +# OPTIONS are a set of options for the authentication in the format +# NAME=VALUE. The available options depend on the different +# authentication methods -- refer to the "Client Authentication" +# section in the documentation for a list of which options are +# available for which authentication methods. +# +# Database and user names containing spaces, commas, quotes and other +# special characters must be quoted. Quoting one of the keywords +# "all", "sameuser", "samerole" or "replication" makes the name lose +# its special character, and just match a database or username with +# that name. +# +# This file is read on server startup and when the postmaster receives +# a SIGHUP signal. If you edit the file on a running system, you have +# to SIGHUP the postmaster for the changes to take effect. You can +# use "pg_ctl reload" to do that. + +# Put your actual configuration here +# ---------------------------------- +# +# If you want to allow non-local connections, you need to add more +# "host" records. In that case you will also need to make PostgreSQL +# listen on a non-local interface via the listen_addresses +# configuration parameter, or via the -i or -h command line switches. + + + + +# DO NOT DISABLE! +# If you change this first entry you will need to make sure that the +# database superuser can access the database using some other method. +# Noninteractive access to all databases is required during automatic +# maintenance (custom daily cronjobs, replication, and similar tasks). +# +# Database administrative login by Unix domain socket +local all postgres peer + +# TYPE DATABASE USER ADDRESS METHOD + +# "local" is for Unix domain socket connections only +local all all peer +# IPv4 local connections: +host all all 127.0.0.1/32 md5 +# IPv6 local connections: +host all all ::1/128 md5 +# Allow replication connections from localhost, by a user with the +# replication privilege. +#local replication postgres peer +#host replication postgres 127.0.0.1/32 md5 +#host replication postgres ::1/128 md5 + +# Allow connection from arbitrary docker containers in the docker network +host all all 0.0.0.0/0 md5 diff --git a/deployment/production/docker/docker-compose.yml b/deployment/production/docker/docker-compose.yml index f592c29b..e4c1e3c1 100644 --- a/deployment/production/docker/docker-compose.yml +++ b/deployment/production/docker/docker-compose.yml @@ -1,12 +1,33 @@ version: '2' volumes: django-postgis-data: + driver: rancher-nfs + driver_opts: + onRemove: retain django-media-data: + driver: rancher-nfs + driver_opts: + onRemove: retain django-static-data: + driver: rancher-nfs + driver_opts: + onRemove: retain django-postgis-dbbackup-data: + driver: rancher-nfs + driver_opts: + onRemove: retain sftpbackup-postgis-target-data: + driver: rancher-nfs + driver_opts: + onRemove: retain sftpbackup-postgis-history-data: + driver: rancher-nfs + driver_opts: + onRemove: retain sftpbackup-media-history-data: + driver: rancher-nfs + driver_opts: + onRemove: retain services: smtp: image: catatnight/postfix @@ -16,7 +37,7 @@ services: - smtp_user=noreply:docker db: - image: kartoza/postgis:9.3-2.1 + image: kartoza/inasafe-django_db:v3.4 restart: on-failure volumes: - django-postgis-data:/var/lib/postgresql @@ -36,7 +57,7 @@ services: - db:db dbbackup: &dbbackup - image: inasafe-django_dbbackup + image: kartoza/inasafe-django_dbbackup:v3.4 volumes: - django-postgis-dbbackup-data:/backups # this folder is used as target folder for sftppgbackup as the @@ -76,7 +97,6 @@ services: - SFTP_USER=user - SFTP_PASSWORD=password - SFTP_DIR=/ - - TARGET_FOLDER=/pg_backup sftpmediabackup: &sftpmediabackup @@ -101,10 +121,9 @@ services: - SFTP_USER=user - SFTP_PASSWORD=password - SFTP_DIR=/ - - TARGET_FOLDER=/pg_backup uwsgi: &uwsgi - image: inasafe-django_uwsgi + image: kartoza/inasafe-django_uwsgi:v3.4 restart: on-failure environment: - DATABASE_NAME=gis @@ -115,6 +134,10 @@ services: - VIRTUAL_HOST=realtime.inasafe.org - VIRTUAL_PORT=8080 - BROKER_URL=amqp://guest:guest@rabbitmq:5672/ + # Secrets + - SECRET_KEY= + - MAPQUEST_MAP_KEY= + - SITE_DOMAIN_NAME= # InAWARE Credentials - INAWARE_HOST= - INAWARE_USER= @@ -155,7 +178,7 @@ services: command: celery -A core.celery_app worker --loglevel=info -Q inasafe-django-indicator -B -n indicator.%h web: &web - image: inasafe-django_nginx + image: kartoza/inasafe-django_nginx:v3.4 volumes: - django-static-data:/home/web/static:ro - django-media-data:/home/web/media:ro diff --git a/deployment/production/docker/uwsgi/Dockerfile b/deployment/production/docker/uwsgi/Dockerfile index 674b4a18..1c0d0064 100644 --- a/deployment/production/docker/uwsgi/Dockerfile +++ b/deployment/production/docker/uwsgi/Dockerfile @@ -16,16 +16,16 @@ RUN pip install uwsgi RUN apt-get update -y; apt-get install -y gettext # Clone source code - -ADD entry-point.sh /entry-point.sh - RUN mkdir -p /home/src/ RUN mkdir -p /home/web/ -RUN cd /home/src +WORKDIR /home/src RUN git clone --branch v3.4 --depth 1 https://github.com/inasafe/inasafe-django.git # Copy only django_project folder -RUN cp -r /home/src/django_project /home/web/ +RUN mkdir -p /home/web +RUN cp -r inasafe-django/django_project /home/web/ WORKDIR /home/web/django_project + +ADD entry-point.sh /entry-point.sh CMD ["/entry-point.sh"] diff --git a/deployment/production/docker/uwsgi/build.sh b/deployment/production/docker/uwsgi/build.sh index 524b5a81..e4cfb8c5 100755 --- a/deployment/production/docker/uwsgi/build.sh +++ b/deployment/production/docker/uwsgi/build.sh @@ -1,2 +1,5 @@ #!/bin/sh - docker build -t inasafe/inasafe-django_uwsgi . +IMAGE_NAME=inasafe-django_uwsgi +TAG_NAME=v3.4 +docker build -t kartoza/${IMAGE_NAME} . +docker tag kartoza/${IMAGE_NAME}:latest kartoza/${IMAGE_NAME}:${TAG_NAME} diff --git a/deployment/production/docker/uwsgi/entry-point.sh b/deployment/production/docker/uwsgi/entry-point.sh index ee74b8c9..e5cd26af 100755 --- a/deployment/production/docker/uwsgi/entry-point.sh +++ b/deployment/production/docker/uwsgi/entry-point.sh @@ -1,10 +1,12 @@ #!/usr/bin/env bash # Run database migrations +echo "Run database migrations" ./manage.py migrate # Run collectstatic -./manage.py collectstatic +echo "Run collectstatic" +./manage.py collectstatic --noinput # Run uwsgi -uwsgi --ini uwsgi.conf +uwsgi --ini /uwsgi.conf diff --git a/django_project/core/settings/base.py b/django_project/core/settings/base.py index 14b898e9..6c23a60f 100644 --- a/django_project/core/settings/base.py +++ b/django_project/core/settings/base.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Django settings for inasafe project. +import os import sys from .utils import ABS_PATH @@ -53,7 +54,7 @@ ) # import SECRET_KEY into current namespace -from .secret import SECRET_KEY # noqa +from .secret import * # noqa # default middleware classes MIDDLEWARE_CLASSES = ( @@ -105,4 +106,4 @@ DJANGO_ROOT = ABS_PATH('.') -SITE_DOMAIN_NAME = 'example.com' +SITE_DOMAIN_NAME = os.environ.get('SITE_DOMAIN_NAME', 'example.com') diff --git a/django_project/core/settings/contrib.py b/django_project/core/settings/contrib.py index 572a9eba..916bbb3a 100644 --- a/django_project/core/settings/contrib.py +++ b/django_project/core/settings/contrib.py @@ -69,8 +69,4 @@ 'file_browser_callback': 'mce_filebrowser' } -try: - # This settings will be used for mapquest tiles for realtime and user_map - from .mapquest import MAPQUEST_MAP_KEY # noqa -except: - pass +from .secret import MAPQUEST_MAP_KEY # noqa diff --git a/django_project/core/settings/custom_settings.sample.py b/django_project/core/settings/custom_settings.sample.py deleted file mode 100644 index f2b2fe49..00000000 --- a/django_project/core/settings/custom_settings.sample.py +++ /dev/null @@ -1,13 +0,0 @@ -# coding=utf-8 - -__author__ = 'Rizky Maulana Nugraha ' -__date__ = '7/19/16' - - -# We can use this files to override the settings -# This file will be imported at the end of the settings phase. - -# for example - - -SITE_DOMAIN_NAME = 'http://192.168.99.100:61102' diff --git a/django_project/core/settings/dev_docker.py b/django_project/core/settings/dev_docker.py index 3169187b..9a8d7223 100644 --- a/django_project/core/settings/dev_docker.py +++ b/django_project/core/settings/dev_docker.py @@ -67,12 +67,3 @@ 'level': 'WARNING' } } - - -SITE_DOMAIN_NAME = 'http://192.168.99.100:61102' - -try: - # Used to load final custom settings override - from .custom_settings import * # noqa -except: - pass diff --git a/django_project/core/settings/mapquest.sample.py b/django_project/core/settings/mapquest.sample.py deleted file mode 100644 index c78d4865..00000000 --- a/django_project/core/settings/mapquest.sample.py +++ /dev/null @@ -1,10 +0,0 @@ -# coding=utf-8 - -__author__ = 'Rizky Maulana Nugraha ' -__date__ = '7/18/16' - - -# This file is used to configure mapquest key -# Copy this file as "mapquest.py" - -MAPQUEST_MAP_KEY = 'PIvAcge99avs4740iOKzAy11hMTfXzCR' diff --git a/django_project/core/settings/prod_docker.py b/django_project/core/settings/prod_docker.py index c622f056..564bcbc5 100644 --- a/django_project/core/settings/prod_docker.py +++ b/django_project/core/settings/prod_docker.py @@ -45,10 +45,5 @@ DEFAULT_FROM_EMAIL = 'noreply@inasafe.org' -SITE_DOMAIN_NAME = 'http://realtime.inasafe.org' - -try: - # Used to load final custom settings override - from .custom_settings import * # noqa -except: - pass +SITE_DOMAIN_NAME = os.environ.get( + 'SITE_DOMAIN_NAME', 'http://realtime.inasafe.org') diff --git a/django_project/core/settings/secret.py b/django_project/core/settings/secret.py new file mode 100644 index 00000000..bac9e900 --- /dev/null +++ b/django_project/core/settings/secret.py @@ -0,0 +1,16 @@ +# coding=utf-8 +import os + +# This is django secret + +if 'SECRET_KEY' in os.environ: + SECRET_KEY = os.environ.get('SECRET_KEY') +else: + # Use default secret key for development use + # Generate new one and don't share it for production + SECRET_KEY = "&+&ly2fd$$tj&9)polyg!qol@@=f!jr@k3-pb)5q8vqj)9=2l#" + + +# This secret is used by mapquest + +MAPQUEST_MAP_KEY = os.environ.get('MAPQUEST_MAP_KEY') diff --git a/django_project/realtime/tasks/realtime/celeryconfig_sample.py b/django_project/realtime/tasks/realtime/celeryconfig.py similarity index 84% rename from django_project/realtime/tasks/realtime/celeryconfig_sample.py rename to django_project/realtime/tasks/realtime/celeryconfig.py index a6ff0e67..8c04d2ab 100644 --- a/django_project/realtime/tasks/realtime/celeryconfig_sample.py +++ b/django_project/realtime/tasks/realtime/celeryconfig.py @@ -5,11 +5,13 @@ Please copy it as celeryconfig.py so it can be read """ +import os + __author__ = 'Rizky Maulana Nugraha ' __date__ = '1/28/16' -BROKER_URL = 'amqp://guest:guest@192.168.99.100:8100/' +BROKER_URL = os.environ.get('INASAFE_REALTIME_BROKER_URL') CELERY_RESULT_BACKEND = BROKER_URL