diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..921d86b0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +data/ +postgres/ +redis/ +venv/ diff --git a/.env b/.env new file mode 100644 index 00000000..da25955a --- /dev/null +++ b/.env @@ -0,0 +1,5 @@ +POSTGRES_PASSWORD=33SebQd4Ugkqx4KjqanZDsWRXTR54M6k +DJANGO_POSTGRES_PASSWORD=ZV35A5x89pbHuiYSrvqroPHKDnng7dRF +DS_DATA_DIR=./data +DJANGO_MODE=development +INVOKE_DJANGO_DEBUG=1 diff --git a/.gitignore b/.gitignore index 68ab3b27..5ee26e76 100644 --- a/.gitignore +++ b/.gitignore @@ -3,18 +3,14 @@ datascope.iml .DS_store *.pyc -*.db -*.csv -db-dump.*.json -*.sql *.pkl +celerybeat-schedule.db -datagrowth/docs/_build/ +venv/ -datascope/settings.py -datascope/bootstrap.py -datascope/secrets.py -datascope/logs/ -datascope/statics/ +src/datascope/logs/ +src/datascope/statics/ -data/ +/data +/models +secrets.bk.py diff --git a/.travis.yml b/.travis.yml index 38195e5d..b1ac4bf2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: python python: - - "3.5" - "3.6" # command to install dependencies before_install: @@ -8,8 +7,9 @@ before_install: - sudo sed -i -e '/local.*peer/s/postgres/all/' -e 's/peer\|md5/trust/g' /etc/postgresql/*/main/pg_hba.conf - sudo service postgresql restart - sleep 1 -install: "pip install -r datascope/requirements/production.txt" +install: "pip install -r src/datascope/requirements/production.txt" # command to run tests +before_script: cd src && python manage.py collectstatic --noinput script: python manage.py test --settings=datascope.settings_test services: - postgresql @@ -22,3 +22,6 @@ addons: env: global: - PGPORT=5432 + - DJANGO_CONTEXT=host + - INVOKE_POSTGRES_HOST=127.0.0.1 + - INVOKE_DJANGO_DATABASE_USER=postgres diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..3b408e36 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,51 @@ +FROM python:3.6 + +RUN apt-get update && \ + apt-get install -y vim binutils libproj-dev gdal-bin gettext default-jdk + +# Setup directories, 3rd party models and users +RUN mkdir -p /usr/etc/datascope +COPY deploy/environments /usr/etc/datascope +RUN mkdir -p /usr/etc/models +COPY models /usr/etc/models +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app +RUN useradd -ms /bin/bash app + +# Install Python dependencies and copy app +RUN pip install --upgrade pip +COPY src/datascope/requirements /usr/src/app/datascope/requirements +RUN pip install --no-cache-dir -r datascope/requirements/production.txt +COPY --chown=app:app src /usr/src/app + +# We setup spaCy models outside of the pip flow to prevent repetious downloads +RUN python -m spacy link /usr/etc/models/spacy/en_core_web_lg-2.0.0/en_core_web_lg en_core_web_lg +RUN python -m spacy link /usr/etc/models/spacy/nl_core_news_sm-2.0.0/nl_core_news_sm nl_core_news_sm + +# We're serving static files through Whitenoise +# See: http://whitenoise.evans.io/en/stable/index.html# +# If you doubt this decision then read the "infrequently asked question" section for details +# Here we gather static files that get served through uWSGI +# NB: runs with production settings +RUN python manage.py collectstatic --noinput + +# We're switching user to a non-priviliged user +# The Python packages directory and datagrowth package needs to belong to that user +# for dynamic packaging (see entrypoint) +RUN chown app:app /usr/local/lib/python3.6/site-packages +RUN chown -R app:app /usr/local/lib/python3.6/site-packages/datagrowth* +USER app:app + +# Compiling translations +# NB: runs with production settings +# NB: not enabled +# RUN python manage.py compilemessages + +# Entrypoint handles some edge cases before running +ENTRYPOINT ["/usr/src/app/entrypoint.sh"] + +# The default command is to start a uWSGI server +CMD ["uwsgi", "--ini", "/usr/src/app/uwsgi.ini"] + +# EXPOSE port 8000 to allow communication to/from server +EXPOSE 8000 diff --git a/Makefile b/Makefile index 6acba150..dc69165b 100644 --- a/Makefile +++ b/Makefile @@ -1,19 +1,7 @@ -now = $(shell date +"%Y-%m-%d") - clean: find . -type f -name "*.pyc" -delete; find . -type d -name "__pycache__" -delete; -deploy: clean - sudo service uwsgi restart - sudo service celeryd restart - -backup-db: - pg_dump -h localhost -U postgres datascope > data/datascope.postgres.sql - -import-db: - cat $(backup) | psql -h localhost -U postgres datascope - backup-data: # Syncing local data to a harddrive # -z means use compression @@ -24,10 +12,10 @@ backup-data: rsync -zrthv --progress data /Volumes/Leo65/data/datascope start-celery: - celery -A datascope worker --loglevel=info -B + cd src && celery -A datascope worker --loglevel=info -B start-postgres: psql -h localhost -U postgres -d postgres test: - ./manage.py test --settings=datascope.settings_test $(filter) + cd src && ./manage.py test --settings=datascope.settings_test $(filter) diff --git a/activate.sh b/activate.sh new file mode 100644 index 00000000..29b5dc79 --- /dev/null +++ b/activate.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source venv/bin/activate +export DJANGO_CONTEXT=host +export $(cat .env | xargs) diff --git a/build.py b/build.py new file mode 100644 index 00000000..a0ae56e9 --- /dev/null +++ b/build.py @@ -0,0 +1,34 @@ +from invoke import task +from src.datascope.version import get_project_version + + +REPOSITORY = "eu.gcr.io/datascope-266618" + + +@task() +def container(ctx, version=None): + + if not version: + version = get_project_version("src/package.json") + + print(f"Building: {version}") + tag = f"{REPOSITORY}/datascope:{version}" + rsl = ctx.run( + f"docker build . -t {tag}", + echo=True + ) + + if rsl.failed: + print(rsl.stderr) + raise RuntimeError("Failed to build docker image") + + print("Pushing") + rsl = ctx.run( + "docker push {}".format(tag), + echo=True, + pty=True + ) + + if rsl.failed: + print(rsl.stderr) + raise RuntimeError("Failed to push image tagged {}".format(tag)) diff --git a/datascope/configuration.py b/datascope/configuration.py deleted file mode 100644 index b852395e..00000000 --- a/datascope/configuration.py +++ /dev/null @@ -1,11 +0,0 @@ -from django.conf import settings - -from datagrowth.configuration import DEFAULT_CONFIGURATION - - -PROCESS_CHOICE_LIST = [ - ("HttpResourceProcessor.fetch", "Fetch content from HTTP resource"), - ("HttpResourceProcessor.fetch_mass", "Fetch content from multiple HTTP resources"), - ("ExtractProcessor.extract_from_resource", "Extract content from one or more resources"), - ("ExtractProcessor.pass_resource_through", "Take content 'as is' from one or more resources"), -] diff --git a/datascope/environments/aws_bootstrap.py b/datascope/environments/aws_bootstrap.py deleted file mode 100644 index 129cc6d3..00000000 --- a/datascope/environments/aws_bootstrap.py +++ /dev/null @@ -1,11 +0,0 @@ -# DATASCOPE_VERSION gets prepended during deploy -####################################################### -# Digital Ocean -####################################################### - -# Required variables needed to complete setup -PATH_TO_PROJECT = '/srv/artefacts/datascope/{}/src/'.format(DATASCOPE_VERSION) -URL_TO_PROJECT = '/' - -# Optional variables that control features of an installation -USE_WEBSOCKETS = False diff --git a/datascope/environments/aws_requirements.txt b/datascope/environments/aws_requirements.txt deleted file mode 100644 index f23dc392..00000000 --- a/datascope/environments/aws_requirements.txt +++ /dev/null @@ -1 +0,0 @@ --r ../requirements/production.txt diff --git a/datascope/environments/aws_settings.py b/datascope/environments/aws_settings.py deleted file mode 100644 index 8a6dda8b..00000000 --- a/datascope/environments/aws_settings.py +++ /dev/null @@ -1,27 +0,0 @@ -import sentry_sdk -from sentry_sdk.integrations.django import DjangoIntegration - -from datascope.settings_base import * - -DATABASES["default"]["NAME"] = 'datascope' -DATABASES["default"]["USER"] = 'django' -DATABASES["default"]["PASSWORD"] = DATABASE_PASSWORD - -STATIC_IP = "34.251.167.142" - -# Sentry error reporting -sentry_sdk.init( - dsn="https://8402764df7cc487b93b15cf2163c456f@sentry.io/1277824", - integrations=[DjangoIntegration()], - release=DATASCOPE_VERSION, - server_name='data-scope.com' -) - -# This disables sessions for other than admin -# CSRF problems with rest_framework need to be solved -# Before we can allow sessions on /data -SESSION_COOKIE_PATH = "/admin" - -DATAGROWTH_DATA_DIR = os.path.join(os.sep, "srv", "data") -DATAGROWTH_MEDIA_ROOT = MEDIA_ROOT = os.path.join(DATAGROWTH_DATA_DIR, "media") -DATAGROWTH_BIN_DIR = os.path.join(os.sep, "srv", "bin") diff --git a/datascope/environments/digital-ocean_bootstrap.py b/datascope/environments/digital-ocean_bootstrap.py deleted file mode 100644 index 129cc6d3..00000000 --- a/datascope/environments/digital-ocean_bootstrap.py +++ /dev/null @@ -1,11 +0,0 @@ -# DATASCOPE_VERSION gets prepended during deploy -####################################################### -# Digital Ocean -####################################################### - -# Required variables needed to complete setup -PATH_TO_PROJECT = '/srv/artefacts/datascope/{}/src/'.format(DATASCOPE_VERSION) -URL_TO_PROJECT = '/' - -# Optional variables that control features of an installation -USE_WEBSOCKETS = False diff --git a/datascope/environments/digital-ocean_requirements.txt b/datascope/environments/digital-ocean_requirements.txt deleted file mode 100644 index f23dc392..00000000 --- a/datascope/environments/digital-ocean_requirements.txt +++ /dev/null @@ -1 +0,0 @@ --r ../requirements/production.txt diff --git a/datascope/environments/digital-ocean_settings.py b/datascope/environments/digital-ocean_settings.py deleted file mode 100644 index eef50965..00000000 --- a/datascope/environments/digital-ocean_settings.py +++ /dev/null @@ -1,31 +0,0 @@ -from celery.schedules import crontab -import sentry_sdk -from sentry_sdk.integrations.django import DjangoIntegration - -from datascope.settings_base import * - - -STATIC_IP = "37.139.10.19" - -CELERY_BEAT_SCHEDULE = { - "grow_locafora": { - "task": "nautilus.grow_locafora", - "schedule": crontab(minute=0, hour=23, day_of_week=4), - }, - "clear_locafora": { - "task": "nautilus.clear_locafora", - "schedule": crontab(minute=0, hour=23, day_of_week=6), - } -} - -# Sentry error reporting -sentry_sdk.init( - dsn="https://407d0ac6dc4542c9a60fb299e32e464d@sentry.io/241870", - integrations=[DjangoIntegration()], - release=DATASCOPE_VERSION, - server_name='data-scope.com' -) - -DATAGROWTH_DATA_DIR = os.path.join(os.sep, "srv", "data") -DATAGROWTH_MEDIA_ROOT = MEDIA_ROOT = os.path.join(DATAGROWTH_DATA_DIR, "media") -DATAGROWTH_BIN_DIR = os.path.join(os.sep, "srv", "bin") diff --git a/datascope/environments/local_bootstrap.py b/datascope/environments/local_bootstrap.py deleted file mode 100644 index fd0a5ffd..00000000 --- a/datascope/environments/local_bootstrap.py +++ /dev/null @@ -1,12 +0,0 @@ -DATASCOPE_VERSION = "0.0.0" # adjust manually to version in package.json -####################################################### -# Local -####################################################### - -# Required variables needed to complete setup -PROJECT_NAME = 'datascope' -PATH_TO_PROJECT = '/path/to/project/' -URL_TO_PROJECT = '/' - -# Optional variables that control features of an installation -USE_WEBSOCKETS = False diff --git a/datascope/environments/local_requirements.txt b/datascope/environments/local_requirements.txt deleted file mode 100644 index 083ee7ed..00000000 --- a/datascope/environments/local_requirements.txt +++ /dev/null @@ -1 +0,0 @@ --r ../requirements/development.txt diff --git a/datascope/environments/local_settings.py b/datascope/environments/local_settings.py deleted file mode 100644 index cb120e12..00000000 --- a/datascope/environments/local_settings.py +++ /dev/null @@ -1,38 +0,0 @@ -from .settings_base import * - -DEBUG = True -USE_MOCKS = True - -INSTALLED_APPS += ( - 'debug_toolbar', - #'django_extensions', -) - -DEBUG_TOOLBAR = True -INTERNAL_IPS = [ - "127.0.0.1" -] -DEBUG_TOOLBAR_CONFIG = { - 'SHOW_COLLAPSED': True -} -MIDDLEWARE_CLASSES += ( - 'debug_toolbar.middleware.DebugToolbarMiddleware', -) - -#DATAGROWTH_REQUESTS_PROXIES = { -# "http": "localhost:8888" -#} -#DATAGROWTH_REQUESTS_VERIFY = False - -STATIC_IP = "127.0.0.1" -ALLOWED_HOSTS = ["*"] - -ADMINS = ( - ('Administrator', 'administrator@example.com'), -) - -MANAGERS = ( - ('Manager', 'manager@example.com'), -) - -SERVER_EMAIL = 'server@example.com' diff --git a/datascope/environments/secrets_example.py b/datascope/environments/secrets_example.py deleted file mode 100644 index 685f8b0d..00000000 --- a/datascope/environments/secrets_example.py +++ /dev/null @@ -1,38 +0,0 @@ -####################################################### -# REQUIRED -####################################################### -# Following secrets have to be set always - -# Django SECRET_KEY -# Documentation: https://docs.djangoproject.com/en/1.11/ref/settings/#std:setting-SECRET_KEY -SECRET_KEY = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' - -####################################################### -# DATABASE -####################################################### -# Set these secrets when using a database - -DATABASE_USER = '' # use 'django' for development -DATABASE_PASSWORD = '' # use 'ubjJYxrEzLmDDKmqVNsFbJSr8wjMU5BV' for development - -####################################################### -# OPTIONAL -####################################################### -# Following secrets are completely optional - -# Various API keys for 3rd party API's -GOOGLE_API_KEY = '' -INDICO_API_KEY = '' -WIZENOZE_API_KEY = '' - -# Django email settings -# Documentation: https://docs.djangoproject.com/en/1.11/ref/settings/#std:setting-EMAIL_HOST -EMAIL_HOST_USER = '' -EMAIL_HOST_PASSWORD = '' - -# Locafora secrets -LOCAFORA_PASSWORD = '' - -# Wiki feed secrets -WIKI_USER = '' -WIKI_PASSWORD = '' diff --git a/datascope/environments/wikipedia_bootstrap.py b/datascope/environments/wikipedia_bootstrap.py deleted file mode 100644 index d72d5268..00000000 --- a/datascope/environments/wikipedia_bootstrap.py +++ /dev/null @@ -1,9 +0,0 @@ -# DATASCOPE_VERSION gets prepended during deploy -####################################################### -# Wikipedia -####################################################### - -PATH_TO_PROJECT = '/data/project/algo-news/artefacts/datascope/{}/src/'.format(DATASCOPE_VERSION) -URL_TO_PROJECT = '/algo-news/' - -USE_WEBSOCKETS = False diff --git a/datascope/environments/wikipedia_requirements.txt b/datascope/environments/wikipedia_requirements.txt deleted file mode 100644 index cfc511c9..00000000 --- a/datascope/environments/wikipedia_requirements.txt +++ /dev/null @@ -1 +0,0 @@ --r ../requirements/minimal.txt diff --git a/datascope/environments/wikipedia_settings.py b/datascope/environments/wikipedia_settings.py deleted file mode 100644 index 6ee3da35..00000000 --- a/datascope/environments/wikipedia_settings.py +++ /dev/null @@ -1,25 +0,0 @@ -from celery.schedules import crontab - -from datascope.settings_base import * - -DATABASES["default"]["ENGINE"] = "django.db.backends.mysql" -DATABASES["default"]["NAME"] = "s52573__datascope" -DATABASES["default"]["USER"] = "s52573" -DATABASES["default"]["PASSWORD"] = DATABASE_PASSWORD -DATABASES["default"]["HOST"] = "tools-db" - -SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") -STATIC_URL = "//tools-static.wmflabs.org/algo-news/static/" - -CELERY_DEFAULT_QUEUE = "datascope" -CELERY_BROKER_URL = "redis://tools-redis:6379/0" -# CELERY_BEAT_SCHEDULE = { -# "update_wiki_feed": { -# "task": "wiki_feed.update_wiki_feed", -# "schedule": crontab(minute=0, hour=6), -# } -# } - -MAX_BATCH_SIZE = 100 - -STATIC_IP = '127.0.0.1' diff --git a/deploy/.gitignore b/deploy/.gitignore new file mode 100644 index 00000000..94b3316a --- /dev/null +++ b/deploy/.gitignore @@ -0,0 +1,5 @@ +**/secrets/invoke.* +**/secrets/gcloud.json +environments/production/secrets/.env +/.env +/docker-compose.yml diff --git a/deploy/docker-compose.web.yml b/deploy/docker-compose.web.yml new file mode 100644 index 00000000..c69af6c8 --- /dev/null +++ b/deploy/docker-compose.web.yml @@ -0,0 +1,128 @@ +version: '3.3' + +services: + + datascope: + image: eu.gcr.io/datascope-266618/datascope:${RELEASE_VERSION} + deploy: + replicas: 1 + resources: + limits: + cpus: "1" + memory: 0.5G + restart_policy: + condition: any + depends_on: + - "postgres" + - "redis" + network_mode: host + healthcheck: + test: curl -sS http://localhost:8000/health/?docker=1 || exit 1 + interval: 5s + timeout: 10s + retries: 3 + environment: + - PYTHONUNBUFFERED=1 + - DJANGO_MODE=${DJANGO_MODE} + - INVOKE_DJANGO_DEBUG=${INVOKE_DJANGO_DEBUG} + - INVOKE_POSTGRES_CREDENTIALS=${INVOKE_POSTGRES_CREDENTIALS} + ports: + - "8000:8000" + + tasks: + image: eu.gcr.io/datascope-266618/datascope:${RELEASE_VERSION} + deploy: + replicas: 1 + resources: + limits: + cpus: "1" + memory: 0.5G + restart_policy: + condition: any + command: celery -A datascope worker --loglevel=info + depends_on: + - "postgres" + - "redis" + network_mode: host + labels: + nl.surfpol.tasks: "tasks" + environment: + - PYTHONUNBUFFERED=1 + - DJANGO_MODE=${DJANGO_MODE} + - INVOKE_DJANGO_DEBUG=${INVOKE_DJANGO_DEBUG} + - INVOKE_POSTGRES_CREDENTIALS=${INVOKE_POSTGRES_CREDENTIALS} + volumes: + - /srv/data:/usr/src/data + + postgres: + image: postgres:10.10 + deploy: + replicas: 1 + resources: + limits: + cpus: "1" + memory: 1G + restart_policy: + condition: any + command: postgres -c 'config_file=/etc/postgresql/postgresql.conf' + environment: + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_USER: postgres + POSTGRES_DB: datascope + PGDATA: /var/lib/postgresql/data/pgdata + DJANGO_POSTGRES_PASSWORD: ${DJANGO_POSTGRES_PASSWORD} + volumes: + - ./postgres/conf/postgresql.conf:/etc/postgresql/postgresql.conf + - postgres-data:/var/lib/postgresql/data/pgdata + - ./postgres/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d + ports: + - 5432:5432 + network_mode: host + labels: + nl.surfpol.db: "postgres" + + redis: + image: redis + deploy: + replicas: 1 + resources: + limits: + cpus: "1" + memory: 0.25G + restart_policy: + condition: any + command: redis-server /usr/local/etc/redis/redis.conf + volumes: + - ./redis/redis.conf:/usr/local/etc/redis/redis.conf + ports: + - "6379:6379" + network_mode: host + + elastic: + image: docker.elastic.co/elasticsearch/elasticsearch:6.8.3 + deploy: + replicas: 1 + resources: + limits: + cpus: "1" + memory: 1.75G + restart_policy: + condition: any + environment: + - node.name=elastic + - discovery.type=single-node + - cluster.name=elastic + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - network.host=0.0.0.0 + network_mode: host + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - elastic-data:/usr/share/elasticsearch/data + +volumes: + postgres-data: + elastic-data: diff --git a/deploy/docker-compose.worker.yml b/deploy/docker-compose.worker.yml new file mode 100644 index 00000000..981dc317 --- /dev/null +++ b/deploy/docker-compose.worker.yml @@ -0,0 +1,59 @@ +version: '3' + +services: + + datascope: + image: eu.gcr.io/datascope-266618/datascope:${RELEASE_VERSION} + deploy: + replicas: 1 + resources: + limits: + cpus: "1" + memory: 0.5G + restart_policy: + condition: any + depends_on: + - "postgres" + network_mode: host + healthcheck: + test: curl -sS http://localhost:8000/health/?docker=1 || exit 1 + interval: 5s + timeout: 10s + retries: 3 + environment: + - PYTHONUNBUFFERED=1 + - DJANGO_MODE=${DJANGO_MODE} + - INVOKE_DJANGO_DEBUG=${INVOKE_DJANGO_DEBUG} + - INVOKE_POSTGRES_CREDENTIALS=${INVOKE_POSTGRES_CREDENTIALS} + ports: + - "8000:8000" + + postgres: + image: postgres:10.10 + deploy: + replicas: 1 + resources: + limits: + cpus: "1" + memory: 1G + restart_policy: + condition: any + command: postgres -c 'config_file=/etc/postgresql/postgresql.conf' + environment: + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_USER: postgres + POSTGRES_DB: datascope + PGDATA: /var/lib/postgresql/data/pgdata + DJANGO_POSTGRES_PASSWORD: ${DJANGO_POSTGRES_PASSWORD} + volumes: + - ./postgres/conf/postgresql.conf:/etc/postgresql/postgresql.conf + - postgres-data:/var/lib/postgresql/data/pgdata + - ./postgres/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d + ports: + - 5432:5432 + network_mode: host + labels: + nl.surfpol.db: "postgres" + +volumes: + postgres-data: diff --git a/deploy/environments/development/invoke.yml b/deploy/environments/development/invoke.yml new file mode 100644 index 00000000..4a68e9a8 --- /dev/null +++ b/deploy/environments/development/invoke.yml @@ -0,0 +1,22 @@ +django: + debug: 1 + database_name: datascope + database_user: django + secret_key: "secret-key" + database_password: "secret" + administrator: + name: "Anonymous" + email: "email" + manager: + name: "Anonymous" + email: "email" +datascope: + google_cx: "004613812033868156538:5pcwbuudj1m" + data_dir: ./data + google_api_key: "secret" + locafora_password: "secret" + wiki_user: "Wiki Feed Bot" + wiki_password: "secret" +postgres: + credentials: + host: "127.0.0.1" diff --git a/deploy/environments/development/secrets/.env b/deploy/environments/development/secrets/.env new file mode 100644 index 00000000..d94cd7dc --- /dev/null +++ b/deploy/environments/development/secrets/.env @@ -0,0 +1,5 @@ +DJANGO_MODE=development +INVOKE_POSTGRES_CREDENTIALS= +INVOKE_DJANGO_DEBUG=1 +POSTGRES_PASSWORD=33SebQd4Ugkqx4KjqanZDsWRXTR54M6k +DJANGO_POSTGRES_PASSWORD=ZV35A5x89pbHuiYSrvqroPHKDnng7dRF diff --git a/datascope/logs/.gitkeep b/deploy/environments/development/secrets/.gitkeep similarity index 100% rename from datascope/logs/.gitkeep rename to deploy/environments/development/secrets/.gitkeep diff --git a/deploy/environments/production/invoke.yml b/deploy/environments/production/invoke.yml new file mode 100644 index 00000000..2def7fad --- /dev/null +++ b/deploy/environments/production/invoke.yml @@ -0,0 +1,22 @@ +django: + debug: 0 + database_name: datascope + database_user: django + secret_key: "secret-key" + database_password: "secret" + administrator: + name: "Anonymous" + email: "email" + manager: + name: "Anonymous" + email: "email" +datascope: + google_cx: "004613812033868156538:5pcwbuudj1m" + data_dir: ./data + google_api_key: "secret" + locafora_password: "secret" + wiki_user: "Wiki Feed Bot" + wiki_password: "secret" +postgres: + credentials: + host: "127.0.0.1" diff --git a/core/migrations/__init__.py b/deploy/environments/production/secrets/.gitkeep similarity index 100% rename from core/migrations/__init__.py rename to deploy/environments/production/secrets/.gitkeep diff --git a/datascope/postgres/conf/postgresql.conf b/deploy/postgres/conf/postgresql.conf similarity index 100% rename from datascope/postgres/conf/postgresql.conf rename to deploy/postgres/conf/postgresql.conf diff --git a/deploy/postgres/docker-entrypoint-initdb.d/initdb.sh b/deploy/postgres/docker-entrypoint-initdb.d/initdb.sh new file mode 100644 index 00000000..183df111 --- /dev/null +++ b/deploy/postgres/docker-entrypoint-initdb.d/initdb.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -e + + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + CREATE USER django WITH ENCRYPTED PASSWORD '${DJANGO_POSTGRES_PASSWORD}'; + ALTER USER django CREATEDB; + GRANT ALL PRIVILEGES ON DATABASE datascope TO django; +EOSQL diff --git a/datascope/redis/redis.conf b/deploy/redis/redis.conf similarity index 100% rename from datascope/redis/redis.conf rename to deploy/redis/redis.conf diff --git a/deploy/reset.sql b/deploy/reset.sql new file mode 100644 index 00000000..4aa9e4f8 --- /dev/null +++ b/deploy/reset.sql @@ -0,0 +1,16 @@ +SELECT 'SELECT SETVAL(' || + quote_literal(quote_ident(PGT.schemaname) || '.' || quote_ident(S.relname)) || + ', COALESCE(MAX(' ||quote_ident(C.attname)|| '), 1) ) FROM ' || + quote_ident(PGT.schemaname)|| '.'||quote_ident(T.relname)|| ';' +FROM pg_class AS S, + pg_depend AS D, + pg_class AS T, + pg_attribute AS C, + pg_tables AS PGT +WHERE S.relkind = 'S' + AND S.oid = D.objid + AND D.refobjid = T.oid + AND D.refobjid = C.attrelid + AND D.refobjsubid = C.attnum + AND T.relname = PGT.tablename +ORDER BY S.relname; diff --git a/deploy/tasks.py b/deploy/tasks.py new file mode 100644 index 00000000..b3e7287b --- /dev/null +++ b/deploy/tasks.py @@ -0,0 +1,96 @@ +""" +A file with tasks that can be executed on nodes +This file needs to be compatible with invoke 0.11.0 +""" +from getpass import getpass +from datetime import datetime +try: + from invoke import ctask as task +except ImportError: + from invoke import task + + +REPOSITORY = "eu.gcr.io/datascope-266618" + + +@task() +def init(ctx, mode, role): + assert role in ["worker", "web", f"Only 'web' and 'worker' are valid roles not {role}"] + ctx.run(f"cp environments/{mode}/secrets/.env .env") + ctx.run(f"cp docker-compose.{role}.yml docker-compose.yml") + + +@task() +def pull(ctx, version): + ctx.run( + f"docker pull {REPOSITORY}/datascope:{version}", + echo=True, + pty=True + ) + + +@task() +def deploy(ctx, version, debug=False): + ctx.run( + ("export INVOKE_DJANGO_DEBUG=1 && " if debug else "") + + f"export RELEASE_VERSION={version} && " + "docker-compose -f docker-compose.yml down && " + "docker-compose -f docker-compose.yml up -d", + # "docker-compose -f docker-compose.yml config 2>/dev/null | " # compiles configuration + # "docker stack deploy -c - --with-registry-auth --prune datascope", # deploys latest "datascope" service + echo=True, + ) + + +@task() +def migrate(ctx, version): + postgres_password = getpass("Postgres password:") + print("Running migration with root database user through docker-compose run ...") + ctx.run( + f"export RELEASE_VERSION={version} && " + f"export INVOKE_POSTGRES_CREDENTIALS=postgres:{postgres_password} && " + "docker-compose -f docker-compose.yml run --rm datascope python manage.py migrate" + ) + + +@task() +def run(ctx, version): + print("Please run the following directly to access a container directly") + print( + f"export RELEASE_VERSION={version} && " + "docker-compose -f docker-compose.yml run --rm datascope bash" + ) + + +@task() +def publish_scripts(ctx): + ctx.run("rm -f deploy/.env") + ctx.run("gsutil rsync -rd -J deploy gs://ds-deploy/") + + +@task() +def update_scripts(ctx): + ctx.run("gsutil rsync -r -J gs://ds-deploy/ .") + + +@task() +def prune(ctx): + ctx.run("docker system prune --all") + + +@task() +def db_dump(ctx): + now = datetime.now() + ctx.run(f"pg_dump -h localhost -U postgres datascope > datascope.postgres.{now:%Y-%m-%d}.sql") + + +@task() +def db_load(ctx, dump_file): + # First we pipe a dumpfile into psql + ctx.run(f"cat {dump_file} | psql -h localhost -U postgres datascope") + # Now we need to reset sequences to make sure that autoid fields act normally + # For this we store the output of reset.sql into a tmp file + ctx.run("psql -h localhost -U postgres -Atq -f deploy/reset.sql -o tmp.sql datascope") + # And then we execute that output + ctx.run("psql -h localhost -U postgres -f tmp.sql datascope") + ctx.run("rm tmp.sql") diff --git a/docker-compose.yml b/docker-compose.yml index ff0b550b..5e72be35 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,26 @@ version: '3' services: + + datascope: + build: . + image: datascope + command: uwsgi --ini /usr/src/app/uwsgi.ini --py-autoreload=3 --honour-stdin + volumes: + - ./src:/usr/src/app + - ./deploy/environments:/usr/etc/datascope + - ${DS_DATA_DIR}:/usr/src/data + - ../datagrowth:/usr/src/datagrowth + ports: + - "8000:8000" + depends_on: + - "postgres" + environment: + - PYTHONUNBUFFERED=1 + - INVOKE_POSTGRES_HOST=postgres + - DJANGO_MODE=${DJANGO_MODE} + - INVOKE_DJANGO_DEBUG=${INVOKE_DJANGO_DEBUG} + postgres: image: postgres:10.10 command: postgres -c 'config_file=/etc/postgresql/postgresql.conf' @@ -9,19 +29,47 @@ services: POSTGRES_USER: postgres POSTGRES_DB: datascope PGDATA: /var/lib/postgresql/data/pgdata + DJANGO_POSTGRES_PASSWORD: ZV35A5x89pbHuiYSrvqroPHKDnng7dRF volumes: - - ./datascope/postgres/conf/postgresql.conf:/etc/postgresql/postgresql.conf + - ./deploy/postgres/conf/postgresql.conf:/etc/postgresql/postgresql.conf - postgres-data:/var/lib/postgresql/data/pgdata - - ./datascope/postgres/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d + - ./deploy/postgres/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d ports: - 5432:5432 + redis: image: redis command: redis-server /usr/local/etc/redis/redis.conf volumes: - - ./datascope/redis/redis.conf:/usr/local/etc/redis/redis.conf + - ./deploy/redis/redis.conf:/usr/local/etc/redis/redis.conf ports: - "6379:6379" + + elastic: + image: docker.elastic.co/elasticsearch/elasticsearch:6.8.3 + environment: + - node.name=elastic + - discovery.type=single-node + - cluster.name=elastic + #- xpack.security.enabled=true + #- xpack.security.transport.ssl.enabled=true + #- xpack.security.audit.enabled=true + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - network.host=0.0.0.0 +# - http.cors.enabled=true +# - http.cors.allow-origin=https://surfpol.nl +# - http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization +# - http.cors.allow-credentials=true + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - elastic-data:/usr/share/elasticsearch/data + #ports: + # - 127.0.0.1:9200:9200 + adminer: image: adminer ports: @@ -29,3 +77,4 @@ services: volumes: postgres-data: + elastic-data: diff --git a/core/__init__.py b/src/core/__init__.py similarity index 100% rename from core/__init__.py rename to src/core/__init__.py diff --git a/core/admin/__init__.py b/src/core/admin/__init__.py similarity index 100% rename from core/admin/__init__.py rename to src/core/admin/__init__.py diff --git a/core/admin/organisms.py b/src/core/admin/organisms.py similarity index 100% rename from core/admin/organisms.py rename to src/core/admin/organisms.py diff --git a/core/admin/resources.py b/src/core/admin/resources.py similarity index 100% rename from core/admin/resources.py rename to src/core/admin/resources.py diff --git a/core/apps.py b/src/core/apps.py similarity index 100% rename from core/apps.py rename to src/core/apps.py diff --git a/core/exceptions.py b/src/core/exceptions.py similarity index 100% rename from core/exceptions.py rename to src/core/exceptions.py diff --git a/core/migrations/0001_squashed_0026_auto_20190218_1423.py b/src/core/migrations/0001_squashed_0026_auto_20190218_1423.py similarity index 100% rename from core/migrations/0001_squashed_0026_auto_20190218_1423.py rename to src/core/migrations/0001_squashed_0026_auto_20190218_1423.py diff --git a/core/migrations/0002_empty_schema_default.py b/src/core/migrations/0002_empty_schema_default.py similarity index 100% rename from core/migrations/0002_empty_schema_default.py rename to src/core/migrations/0002_empty_schema_default.py diff --git a/core/models/organisms/managers/__init__.py b/src/core/migrations/__init__.py similarity index 100% rename from core/models/organisms/managers/__init__.py rename to src/core/migrations/__init__.py diff --git a/core/models/__init__.py b/src/core/models/__init__.py similarity index 100% rename from core/models/__init__.py rename to src/core/models/__init__.py diff --git a/core/models/organisms/__init__.py b/src/core/models/organisms/__init__.py similarity index 100% rename from core/models/organisms/__init__.py rename to src/core/models/organisms/__init__.py diff --git a/core/models/organisms/collective.py b/src/core/models/organisms/collective.py similarity index 100% rename from core/models/organisms/collective.py rename to src/core/models/organisms/collective.py diff --git a/core/models/organisms/community.py b/src/core/models/organisms/community.py similarity index 100% rename from core/models/organisms/community.py rename to src/core/models/organisms/community.py diff --git a/core/models/organisms/growth.py b/src/core/models/organisms/growth.py similarity index 96% rename from core/models/organisms/growth.py rename to src/core/models/organisms/growth.py index 51d5f969..85f8711e 100644 --- a/core/models/organisms/growth.py +++ b/src/core/models/organisms/growth.py @@ -8,7 +8,6 @@ from datagrowth.datatypes import DocumentBase, CollectionBase from datagrowth.exceptions import DGNoContent -from datascope.configuration import PROCESS_CHOICE_LIST, DEFAULT_CONFIGURATION from core.processors.base import ArgumentsTypes from core.processors.mixins import ProcessorMixin from core.utils.configuration import ConfigurationField @@ -44,6 +43,14 @@ class ContributeType(object): ] +PROCESS_CHOICE_LIST = [ + ("HttpResourceProcessor.fetch", "Fetch content from HTTP resource"), + ("HttpResourceProcessor.fetch_mass", "Fetch content from multiple HTTP resources"), + ("ExtractProcessor.extract_from_resource", "Extract content from one or more resources"), + ("ExtractProcessor.pass_resource_through", "Take content 'as is' from one or more resources"), +] + + class Growth(models.Model, ProcessorMixin): community = GenericForeignKey(ct_field="community_type", fk_field="community_id") @@ -52,7 +59,6 @@ class Growth(models.Model, ProcessorMixin): type = models.CharField(max_length=255) config = ConfigurationField( - config_defaults=DEFAULT_CONFIGURATION, namespace="growth", private=["args", "kwargs", "async"] ) diff --git a/core/models/organisms/individual.py b/src/core/models/organisms/individual.py similarity index 100% rename from core/models/organisms/individual.py rename to src/core/models/organisms/individual.py diff --git a/core/models/organisms/managers/tests/__init__.py b/src/core/models/organisms/managers/__init__.py similarity index 100% rename from core/models/organisms/managers/tests/__init__.py rename to src/core/models/organisms/managers/__init__.py diff --git a/core/models/organisms/managers/community.py b/src/core/models/organisms/managers/community.py similarity index 100% rename from core/models/organisms/managers/community.py rename to src/core/models/organisms/managers/community.py diff --git a/core/models/organisms/tests/__init__.py b/src/core/models/organisms/managers/tests/__init__.py similarity index 100% rename from core/models/organisms/tests/__init__.py rename to src/core/models/organisms/managers/tests/__init__.py diff --git a/core/models/organisms/managers/tests/community.py b/src/core/models/organisms/managers/tests/community.py similarity index 100% rename from core/models/organisms/managers/tests/community.py rename to src/core/models/organisms/managers/tests/community.py diff --git a/core/models/organisms/states.py b/src/core/models/organisms/states.py similarity index 100% rename from core/models/organisms/states.py rename to src/core/models/organisms/states.py diff --git a/core/models/resources/tests/__init__.py b/src/core/models/organisms/tests/__init__.py similarity index 100% rename from core/models/resources/tests/__init__.py rename to src/core/models/organisms/tests/__init__.py diff --git a/core/models/organisms/tests/collective.py b/src/core/models/organisms/tests/collective.py similarity index 100% rename from core/models/organisms/tests/collective.py rename to src/core/models/organisms/tests/collective.py diff --git a/core/models/organisms/tests/community.py b/src/core/models/organisms/tests/community.py similarity index 100% rename from core/models/organisms/tests/community.py rename to src/core/models/organisms/tests/community.py diff --git a/core/models/organisms/tests/growth.py b/src/core/models/organisms/tests/growth.py similarity index 100% rename from core/models/organisms/tests/growth.py rename to src/core/models/organisms/tests/growth.py diff --git a/core/models/organisms/tests/individual.py b/src/core/models/organisms/tests/individual.py similarity index 100% rename from core/models/organisms/tests/individual.py rename to src/core/models/organisms/tests/individual.py diff --git a/core/models/resources/__init__.py b/src/core/models/resources/__init__.py similarity index 100% rename from core/models/resources/__init__.py rename to src/core/models/resources/__init__.py diff --git a/core/models/resources/http.py b/src/core/models/resources/http.py similarity index 100% rename from core/models/resources/http.py rename to src/core/models/resources/http.py diff --git a/core/models/resources/manifestation.py b/src/core/models/resources/manifestation.py similarity index 100% rename from core/models/resources/manifestation.py rename to src/core/models/resources/manifestation.py diff --git a/core/models/resources/resource.py b/src/core/models/resources/resource.py similarity index 100% rename from core/models/resources/resource.py rename to src/core/models/resources/resource.py diff --git a/core/models/resources/shell.py b/src/core/models/resources/shell.py similarity index 100% rename from core/models/resources/shell.py rename to src/core/models/resources/shell.py diff --git a/core/processors/tests/__init__.py b/src/core/models/resources/tests/__init__.py similarity index 100% rename from core/processors/tests/__init__.py rename to src/core/models/resources/tests/__init__.py diff --git a/core/models/resources/tests/manifestation.py b/src/core/models/resources/tests/manifestation.py similarity index 100% rename from core/models/resources/tests/manifestation.py rename to src/core/models/resources/tests/manifestation.py diff --git a/core/processors/__init__.py b/src/core/processors/__init__.py similarity index 100% rename from core/processors/__init__.py rename to src/core/processors/__init__.py diff --git a/core/processors/authentication.py b/src/core/processors/authentication.py similarity index 100% rename from core/processors/authentication.py rename to src/core/processors/authentication.py diff --git a/core/processors/base.py b/src/core/processors/base.py similarity index 100% rename from core/processors/base.py rename to src/core/processors/base.py diff --git a/core/processors/compare.py b/src/core/processors/compare.py similarity index 100% rename from core/processors/compare.py rename to src/core/processors/compare.py diff --git a/core/processors/expansion.py b/src/core/processors/expansion.py similarity index 100% rename from core/processors/expansion.py rename to src/core/processors/expansion.py diff --git a/core/processors/filter.py b/src/core/processors/filter.py similarity index 100% rename from core/processors/filter.py rename to src/core/processors/filter.py diff --git a/core/processors/mixins.py b/src/core/processors/mixins.py similarity index 100% rename from core/processors/mixins.py rename to src/core/processors/mixins.py diff --git a/core/processors/rank.py b/src/core/processors/rank.py similarity index 97% rename from core/processors/rank.py rename to src/core/processors/rank.py index 9ec1a1f8..9a979219 100644 --- a/core/processors/rank.py +++ b/src/core/processors/rank.py @@ -7,11 +7,9 @@ import six import warnings -from collections import OrderedDict -from itertools import islice from copy import deepcopy -from datascope.configuration import DEFAULT_CONFIGURATION +from datagrowth.configuration import DEFAULT_CONFIGURATION from core.utils.helpers import merge_iter from core.processors.base import Processor from core.utils.configuration import ConfigurationProperty @@ -106,7 +104,7 @@ def flush_batch(batch, result_size): from itertools import islice -from datascope.configuration import DEFAULT_CONFIGURATION +from datagrowth.configuration import DEFAULT_CONFIGURATION from core.processors.base import QuerySetProcessor from core.utils.data import NumericFeaturesFrame, TextFeaturesFrame from core.utils.configuration import ConfigurationProperty diff --git a/core/processors/resources/__init__.py b/src/core/processors/resources/__init__.py similarity index 100% rename from core/processors/resources/__init__.py rename to src/core/processors/resources/__init__.py diff --git a/core/processors/resources/base.py b/src/core/processors/resources/base.py similarity index 100% rename from core/processors/resources/base.py rename to src/core/processors/resources/base.py diff --git a/core/processors/resources/http.py b/src/core/processors/resources/http.py similarity index 96% rename from core/processors/resources/http.py rename to src/core/processors/resources/http.py index 91fb9f00..43fb7fc0 100644 --- a/core/processors/resources/http.py +++ b/src/core/processors/resources/http.py @@ -1,6 +1,6 @@ import requests -from datascope.configuration import DEFAULT_CONFIGURATION +from datagrowth.configuration import DEFAULT_CONFIGURATION from core.processors.resources.base import ResourceProcessor from core.tasks.http import send, send_mass from core.utils.configuration import ConfigurationProperty diff --git a/core/processors/resources/manifest.py b/src/core/processors/resources/manifest.py similarity index 95% rename from core/processors/resources/manifest.py rename to src/core/processors/resources/manifest.py index f1e7d0ba..45e4ac45 100644 --- a/core/processors/resources/manifest.py +++ b/src/core/processors/resources/manifest.py @@ -1,6 +1,6 @@ import logging -from datascope.configuration import DEFAULT_CONFIGURATION +from datagrowth.configuration import DEFAULT_CONFIGURATION from core.models.organisms.individual import Individual from core.tasks import manifest, manifest_serie from core.processors.resources.base import ResourceProcessor diff --git a/core/processors/resources/shell.py b/src/core/processors/resources/shell.py similarity index 92% rename from core/processors/resources/shell.py rename to src/core/processors/resources/shell.py index 86785a84..899f7081 100644 --- a/core/processors/resources/shell.py +++ b/src/core/processors/resources/shell.py @@ -1,6 +1,4 @@ -import requests - -from datascope.configuration import DEFAULT_CONFIGURATION +from datagrowth.configuration import DEFAULT_CONFIGURATION from core.processors.resources.base import ResourceProcessor from core.tasks.shell import run, run_serie from core.utils.configuration import ConfigurationProperty diff --git a/core/tasks/tests/__init__.py b/src/core/processors/tests/__init__.py similarity index 100% rename from core/tasks/tests/__init__.py rename to src/core/processors/tests/__init__.py diff --git a/core/processors/tests/authentication.py b/src/core/processors/tests/authentication.py similarity index 100% rename from core/processors/tests/authentication.py rename to src/core/processors/tests/authentication.py diff --git a/core/processors/tests/compare.py b/src/core/processors/tests/compare.py similarity index 100% rename from core/processors/tests/compare.py rename to src/core/processors/tests/compare.py diff --git a/core/processors/tests/expansion.py b/src/core/processors/tests/expansion.py similarity index 100% rename from core/processors/tests/expansion.py rename to src/core/processors/tests/expansion.py diff --git a/core/processors/tests/manifest.py b/src/core/processors/tests/manifest.py similarity index 100% rename from core/processors/tests/manifest.py rename to src/core/processors/tests/manifest.py diff --git a/core/processors/tests/mixins.py b/src/core/processors/tests/mixins.py similarity index 100% rename from core/processors/tests/mixins.py rename to src/core/processors/tests/mixins.py diff --git a/core/processors/tests/rank.py b/src/core/processors/tests/rank.py similarity index 100% rename from core/processors/tests/rank.py rename to src/core/processors/tests/rank.py diff --git a/core/processors/tests/resources.py b/src/core/processors/tests/resources.py similarity index 100% rename from core/processors/tests/resources.py rename to src/core/processors/tests/resources.py diff --git a/core/static/core/css/bootstrap-theme.css b/src/core/static/core/css/bootstrap-theme.css similarity index 100% rename from core/static/core/css/bootstrap-theme.css rename to src/core/static/core/css/bootstrap-theme.css diff --git a/core/static/core/css/bootstrap-theme.css.map b/src/core/static/core/css/bootstrap-theme.css.map similarity index 100% rename from core/static/core/css/bootstrap-theme.css.map rename to src/core/static/core/css/bootstrap-theme.css.map diff --git a/core/static/core/css/bootstrap-theme.min.css b/src/core/static/core/css/bootstrap-theme.min.css similarity index 100% rename from core/static/core/css/bootstrap-theme.min.css rename to src/core/static/core/css/bootstrap-theme.min.css diff --git a/core/static/core/css/bootstrap.css b/src/core/static/core/css/bootstrap.css similarity index 100% rename from core/static/core/css/bootstrap.css rename to src/core/static/core/css/bootstrap.css diff --git a/core/static/core/css/bootstrap.css.map b/src/core/static/core/css/bootstrap.css.map similarity index 100% rename from core/static/core/css/bootstrap.css.map rename to src/core/static/core/css/bootstrap.css.map diff --git a/core/static/core/css/bootstrap.min.css b/src/core/static/core/css/bootstrap.min.css similarity index 100% rename from core/static/core/css/bootstrap.min.css rename to src/core/static/core/css/bootstrap.min.css diff --git a/core/static/core/css/style.css b/src/core/static/core/css/style.css similarity index 100% rename from core/static/core/css/style.css rename to src/core/static/core/css/style.css diff --git a/core/static/core/fonts/glyphicons-halflings-regular.eot b/src/core/static/core/fonts/glyphicons-halflings-regular.eot similarity index 100% rename from core/static/core/fonts/glyphicons-halflings-regular.eot rename to src/core/static/core/fonts/glyphicons-halflings-regular.eot diff --git a/core/static/core/fonts/glyphicons-halflings-regular.svg b/src/core/static/core/fonts/glyphicons-halflings-regular.svg similarity index 100% rename from core/static/core/fonts/glyphicons-halflings-regular.svg rename to src/core/static/core/fonts/glyphicons-halflings-regular.svg diff --git a/core/static/core/fonts/glyphicons-halflings-regular.ttf b/src/core/static/core/fonts/glyphicons-halflings-regular.ttf similarity index 100% rename from core/static/core/fonts/glyphicons-halflings-regular.ttf rename to src/core/static/core/fonts/glyphicons-halflings-regular.ttf diff --git a/core/static/core/fonts/glyphicons-halflings-regular.woff b/src/core/static/core/fonts/glyphicons-halflings-regular.woff similarity index 100% rename from core/static/core/fonts/glyphicons-halflings-regular.woff rename to src/core/static/core/fonts/glyphicons-halflings-regular.woff diff --git a/core/static/core/fonts/glyphicons-halflings-regular.woff2 b/src/core/static/core/fonts/glyphicons-halflings-regular.woff2 similarity index 100% rename from core/static/core/fonts/glyphicons-halflings-regular.woff2 rename to src/core/static/core/fonts/glyphicons-halflings-regular.woff2 diff --git a/core/static/core/js/jquery-1.11.1.min.js b/src/core/static/core/js/jquery-1.11.1.min.js similarity index 100% rename from core/static/core/js/jquery-1.11.1.min.js rename to src/core/static/core/js/jquery-1.11.1.min.js diff --git a/core/static/core/js/uri.js b/src/core/static/core/js/uri.js similarity index 100% rename from core/static/core/js/uri.js rename to src/core/static/core/js/uri.js diff --git a/core/static/core/js/wikidata.js b/src/core/static/core/js/wikidata.js similarity index 100% rename from core/static/core/js/wikidata.js rename to src/core/static/core/js/wikidata.js diff --git a/core/tasks/__init__.py b/src/core/tasks/__init__.py similarity index 100% rename from core/tasks/__init__.py rename to src/core/tasks/__init__.py diff --git a/core/tasks/http.py b/src/core/tasks/http.py similarity index 100% rename from core/tasks/http.py rename to src/core/tasks/http.py diff --git a/core/tasks/manifestation.py b/src/core/tasks/manifestation.py similarity index 100% rename from core/tasks/manifestation.py rename to src/core/tasks/manifestation.py diff --git a/core/tasks/shell.py b/src/core/tasks/shell.py similarity index 100% rename from core/tasks/shell.py rename to src/core/tasks/shell.py diff --git a/core/templatetags/__init__.py b/src/core/tasks/tests/__init__.py similarity index 100% rename from core/templatetags/__init__.py rename to src/core/tasks/tests/__init__.py diff --git a/core/tasks/tests/http.py b/src/core/tasks/tests/http.py similarity index 100% rename from core/tasks/tests/http.py rename to src/core/tasks/tests/http.py diff --git a/core/tasks/tests/manifestation.py b/src/core/tasks/tests/manifestation.py similarity index 100% rename from core/tasks/tests/manifestation.py rename to src/core/tasks/tests/manifestation.py diff --git a/core/templates/core/base.html b/src/core/templates/core/base.html similarity index 100% rename from core/templates/core/base.html rename to src/core/templates/core/base.html diff --git a/core/templates/core/components/search.html b/src/core/templates/core/components/search.html similarity index 100% rename from core/templates/core/components/search.html rename to src/core/templates/core/components/search.html diff --git a/core/templates/mock/accepted.html b/src/core/templates/mock/accepted.html similarity index 100% rename from core/templates/mock/accepted.html rename to src/core/templates/mock/accepted.html diff --git a/core/templates/mock/bad-request.html b/src/core/templates/mock/bad-request.html similarity index 100% rename from core/templates/mock/bad-request.html rename to src/core/templates/mock/bad-request.html diff --git a/core/templates/mock/index.html b/src/core/templates/mock/index.html similarity index 100% rename from core/templates/mock/index.html rename to src/core/templates/mock/index.html diff --git a/core/templates/mock/no-content.html b/src/core/templates/mock/no-content.html similarity index 100% rename from core/templates/mock/no-content.html rename to src/core/templates/mock/no-content.html diff --git a/core/templates/mock/ok.html b/src/core/templates/mock/ok.html similarity index 100% rename from core/templates/mock/ok.html rename to src/core/templates/mock/ok.html diff --git a/core/tests/__init__.py b/src/core/templatetags/__init__.py similarity index 100% rename from core/tests/__init__.py rename to src/core/templatetags/__init__.py diff --git a/core/templatetags/bootstrap_tags.py b/src/core/templatetags/bootstrap_tags.py similarity index 100% rename from core/templatetags/bootstrap_tags.py rename to src/core/templatetags/bootstrap_tags.py diff --git a/core/templatetags/template_context.py b/src/core/templatetags/template_context.py similarity index 100% rename from core/templatetags/template_context.py rename to src/core/templatetags/template_context.py diff --git a/core/tests/mocks/__init__.py b/src/core/tests/__init__.py similarity index 100% rename from core/tests/mocks/__init__.py rename to src/core/tests/__init__.py diff --git a/core/tests/fixtures/test-community.json b/src/core/tests/fixtures/test-community.json similarity index 100% rename from core/tests/fixtures/test-community.json rename to src/core/tests/fixtures/test-community.json diff --git a/core/tests/fixtures/test-growth.json b/src/core/tests/fixtures/test-growth.json similarity index 100% rename from core/tests/fixtures/test-growth.json rename to src/core/tests/fixtures/test-growth.json diff --git a/core/tests/fixtures/test-http-resource-mock.json b/src/core/tests/fixtures/test-http-resource-mock.json similarity index 100% rename from core/tests/fixtures/test-http-resource-mock.json rename to src/core/tests/fixtures/test-http-resource-mock.json diff --git a/core/tests/fixtures/test-manifestation.json b/src/core/tests/fixtures/test-manifestation.json similarity index 100% rename from core/tests/fixtures/test-manifestation.json rename to src/core/tests/fixtures/test-manifestation.json diff --git a/core/tests/fixtures/test-organisms.json b/src/core/tests/fixtures/test-organisms.json similarity index 100% rename from core/tests/fixtures/test-organisms.json rename to src/core/tests/fixtures/test-organisms.json diff --git a/core/utils/__init__.py b/src/core/tests/mocks/__init__.py similarity index 100% rename from core/utils/__init__.py rename to src/core/tests/mocks/__init__.py diff --git a/core/tests/mocks/celery.py b/src/core/tests/mocks/celery.py similarity index 100% rename from core/tests/mocks/celery.py rename to src/core/tests/mocks/celery.py diff --git a/core/tests/mocks/community.py b/src/core/tests/mocks/community.py similarity index 100% rename from core/tests/mocks/community.py rename to src/core/tests/mocks/community.py diff --git a/core/tests/mocks/data.py b/src/core/tests/mocks/data.py similarity index 100% rename from core/tests/mocks/data.py rename to src/core/tests/mocks/data.py diff --git a/core/tests/mocks/http.py b/src/core/tests/mocks/http.py similarity index 100% rename from core/tests/mocks/http.py rename to src/core/tests/mocks/http.py diff --git a/core/tests/mocks/processor.py b/src/core/tests/mocks/processor.py similarity index 100% rename from core/tests/mocks/processor.py rename to src/core/tests/mocks/processor.py diff --git a/core/tests/mocks/requests.py b/src/core/tests/mocks/requests.py similarity index 100% rename from core/tests/mocks/requests.py rename to src/core/tests/mocks/requests.py diff --git a/core/tests/mocks/tasks.py b/src/core/tests/mocks/tasks.py similarity index 100% rename from core/tests/mocks/tasks.py rename to src/core/tests/mocks/tasks.py diff --git a/core/tests/mocks/urls.py b/src/core/tests/mocks/urls.py similarity index 100% rename from core/tests/mocks/urls.py rename to src/core/tests/mocks/urls.py diff --git a/core/tests/runner.py b/src/core/tests/runner.py similarity index 100% rename from core/tests/runner.py rename to src/core/tests/runner.py diff --git a/core/tests/tests.py b/src/core/tests/tests.py similarity index 100% rename from core/tests/tests.py rename to src/core/tests/tests.py diff --git a/core/utils/tests/__init__.py b/src/core/utils/__init__.py similarity index 100% rename from core/utils/tests/__init__.py rename to src/core/utils/__init__.py diff --git a/core/utils/configuration.py b/src/core/utils/configuration.py similarity index 100% rename from core/utils/configuration.py rename to src/core/utils/configuration.py diff --git a/core/utils/data/__init__.py b/src/core/utils/data/__init__.py similarity index 100% rename from core/utils/data/__init__.py rename to src/core/utils/data/__init__.py diff --git a/core/utils/data/deprecated.py b/src/core/utils/data/deprecated.py similarity index 100% rename from core/utils/data/deprecated.py rename to src/core/utils/data/deprecated.py diff --git a/core/utils/data/key_path.py b/src/core/utils/data/key_path.py similarity index 100% rename from core/utils/data/key_path.py rename to src/core/utils/data/key_path.py diff --git a/core/utils/data/numeric_features.py b/src/core/utils/data/numeric_features.py similarity index 100% rename from core/utils/data/numeric_features.py rename to src/core/utils/data/numeric_features.py diff --git a/core/utils/data/text_features.py b/src/core/utils/data/text_features.py similarity index 100% rename from core/utils/data/text_features.py rename to src/core/utils/data/text_features.py diff --git a/core/utils/files.py b/src/core/utils/files.py similarity index 100% rename from core/utils/files.py rename to src/core/utils/files.py diff --git a/core/utils/helpers.py b/src/core/utils/helpers.py similarity index 100% rename from core/utils/helpers.py rename to src/core/utils/helpers.py diff --git a/core/utils/image.py b/src/core/utils/image.py similarity index 100% rename from core/utils/image.py rename to src/core/utils/image.py diff --git a/core/views/tests/__init__.py b/src/core/utils/tests/__init__.py similarity index 100% rename from core/views/tests/__init__.py rename to src/core/utils/tests/__init__.py diff --git a/core/utils/tests/data.py b/src/core/utils/tests/data.py similarity index 100% rename from core/utils/tests/data.py rename to src/core/utils/tests/data.py diff --git a/core/utils/tests/files.py b/src/core/utils/tests/files.py similarity index 100% rename from core/utils/tests/files.py rename to src/core/utils/tests/files.py diff --git a/core/utils/tests/helpers.py b/src/core/utils/tests/helpers.py similarity index 100% rename from core/utils/tests/helpers.py rename to src/core/utils/tests/helpers.py diff --git a/core/utils/tests/image.py b/src/core/utils/tests/image.py similarity index 100% rename from core/utils/tests/image.py rename to src/core/utils/tests/image.py diff --git a/core/views/__init__.py b/src/core/views/__init__.py similarity index 100% rename from core/views/__init__.py rename to src/core/views/__init__.py diff --git a/core/views/collective.py b/src/core/views/collective.py similarity index 100% rename from core/views/collective.py rename to src/core/views/collective.py diff --git a/core/views/community.py b/src/core/views/community.py similarity index 100% rename from core/views/community.py rename to src/core/views/community.py diff --git a/core/views/content.py b/src/core/views/content.py similarity index 100% rename from core/views/content.py rename to src/core/views/content.py diff --git a/core/views/individual.py b/src/core/views/individual.py similarity index 100% rename from core/views/individual.py rename to src/core/views/individual.py diff --git a/future_fashion/management/__init__.py b/src/core/views/tests/__init__.py similarity index 100% rename from future_fashion/management/__init__.py rename to src/core/views/tests/__init__.py diff --git a/core/views/tests/collective.py b/src/core/views/tests/collective.py similarity index 100% rename from core/views/tests/collective.py rename to src/core/views/tests/collective.py diff --git a/core/views/tests/community.py b/src/core/views/tests/community.py similarity index 100% rename from core/views/tests/community.py rename to src/core/views/tests/community.py diff --git a/core/views/tests/individual.py b/src/core/views/tests/individual.py similarity index 100% rename from core/views/tests/individual.py rename to src/core/views/tests/individual.py diff --git a/datascope/__init__.py b/src/datascope/__init__.py similarity index 100% rename from datascope/__init__.py rename to src/datascope/__init__.py diff --git a/datascope/celery.py b/src/datascope/celery.py similarity index 100% rename from datascope/celery.py rename to src/datascope/celery.py diff --git a/src/datascope/configuration.py b/src/datascope/configuration.py new file mode 100644 index 00000000..8f345185 --- /dev/null +++ b/src/datascope/configuration.py @@ -0,0 +1,50 @@ +""" +This module exposes the environment which is an invoke Config that holds environment specific configuration. +The idea is that all configuration is managed by just two environment variables: + * DJANGO_MODE + * DJANGO_CONTEXT +The first specifies a mode like "production", "acceptance" or "development". +The latter specifies how the configuration files are found. Inside a Docker container or outside of them. +Read more about invoke Config here: http://docs.pyinvoke.org/en/stable/concepts/configuration.html#config-hierarchy + +Since the config is created outside of invoke there are a few special adjustments +The system invoke files are the environment configuration files. +While the user invoke files are the secret configuration files. +For the rest the project and shell environment variables get loaded as normal and my override environments and secrets. +""" +import os +from invoke.config import Config +from datascope.version import get_project_version + + +MODE = os.environ.get("DJANGO_MODE", "production") +CONTEXT = os.environ.get("DJANGO_CONTEXT", "container") + +BASE_ENVIRONMENT = "/usr/etc/datascope" if CONTEXT == "container" else "../deploy/environments" +MODE_ENVIRONMENT = os.path.join(BASE_ENVIRONMENT, MODE) +SECRET_ENVIRONMENT = os.path.join(MODE_ENVIRONMENT, "secrets") + + +environment = Config( + overrides={ + "project_version": get_project_version("package.json") + }, + system_prefix=MODE_ENVIRONMENT + os.path.sep, + user_prefix=SECRET_ENVIRONMENT + os.path.sep +) +environment.load_system() +environment.load_user() +environment.load_project() +environment.load_shell_env() + +# Load computed overrides (we post process to prevent setting some variables everywhere) +database_credentials = environment.postgres.credentials +if database_credentials: + user, password = database_credentials.split(":") + environment.load_overrides({ + "project_version": get_project_version("package.json"), + "django": { + "database_user": user, + "database_password": password + } + }) diff --git a/future_fashion/management/commands/__init__.py b/src/datascope/logs/.gitkeep similarity index 100% rename from future_fashion/management/commands/__init__.py rename to src/datascope/logs/.gitkeep diff --git a/datascope/requirements/analysis.txt b/src/datascope/requirements/analysis.txt similarity index 100% rename from datascope/requirements/analysis.txt rename to src/datascope/requirements/analysis.txt diff --git a/datascope/requirements/development.txt b/src/datascope/requirements/development.txt similarity index 94% rename from datascope/requirements/development.txt rename to src/datascope/requirements/development.txt index 1cdf2207..02f2cb3f 100644 --- a/datascope/requirements/development.txt +++ b/src/datascope/requirements/development.txt @@ -7,11 +7,11 @@ spacy-arguing-lexicon==0.0.3 IndicoIo==0.15.4 # Debugging and development -django-debug-toolbar==1.9.1 django-extensions==1.9.0 Sphinx==1.3.1 sphinx-rtd-theme==0.1.8 pipdeptree==0.10.1 +invoke==1.4.1 # SpaCy language models https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-2.0.0/en_core_web_lg-2.0.0.tar.gz diff --git a/datascope/requirements/minimal.txt b/src/datascope/requirements/minimal.txt similarity index 79% rename from datascope/requirements/minimal.txt rename to src/datascope/requirements/minimal.txt index 14df9d2a..e7236fc6 100644 --- a/datascope/requirements/minimal.txt +++ b/src/datascope/requirements/minimal.txt @@ -1,8 +1,8 @@ -Django==1.11.23 -mysqlclient==1.3.7 +Django==1.11.29 +whitenoise==5.0.1 sqlparse==0.2.0 # raw sql queries URLObject==2.4.0 -celery==4.3.0 +celery==4.4.0 git+https://github.com/fako/django-json-field.git djangorestframework==3.9.1 django-cors-headers==2.4.0 @@ -21,14 +21,18 @@ tqdm==4.19.8 django-celery-results==1.0.1 anyjson==0.3.3 redis==3.3.8 +vine==1.3.0 # Advanced features selenium==2.47.1 -Pillow==6.2.0 +Pillow==7.1.0 colorz==1.0.3 +elasticsearch==6.0.0 # Debug and test +invoke==1.4.0 sentry-sdk==0.11.1 ipdb==0.8 ipython==6.1.0 mock==1.3.0 +django-debug-toolbar==1.9.1 diff --git a/datascope/requirements/production.txt b/src/datascope/requirements/production.txt similarity index 61% rename from datascope/requirements/production.txt rename to src/datascope/requirements/production.txt index 15b032d2..2f43d07d 100644 --- a/datascope/requirements/production.txt +++ b/src/datascope/requirements/production.txt @@ -1,9 +1,10 @@ -r minimal.txt +https://projects.unbit.it/downloads/uwsgi-lts.tar.gz psycopg2-binary==2.8.3 spacy==2.0.18 spacy-arguing-lexicon==0.0.3 IndicoIo==0.15.4 # Datagrowth -datagrowth==0.15.2 +datagrowth==0.15.5 diff --git a/datascope/requirements/websockets.txt b/src/datascope/requirements/websockets.txt similarity index 100% rename from datascope/requirements/websockets.txt rename to src/datascope/requirements/websockets.txt diff --git a/src/datascope/settings.py b/src/datascope/settings.py new file mode 100644 index 00000000..4cef58de --- /dev/null +++ b/src/datascope/settings.py @@ -0,0 +1 @@ +from .settings_base import * diff --git a/datascope/settings_base.py b/src/datascope/settings_base.py similarity index 68% rename from datascope/settings_base.py rename to src/datascope/settings_base.py index c6b67319..d2ac115b 100644 --- a/datascope/settings_base.py +++ b/src/datascope/settings_base.py @@ -1,64 +1,74 @@ import os import logging +import sentry_sdk +from sentry_sdk.integrations.django import DjangoIntegration -log = logging.getLogger(__name__) +from datascope.configuration import environment +from datascope.version import get_project_version -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +log = logging.getLogger(__name__) ####################################################### # DEFAULT BOOTSTRAP ####################################################### -PATH_TO_PROJECT = '' -URL_TO_PROJECT = '/' +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# The version of the entire project (frontend and backend) +DATASCOPE_VERSION = get_project_version(os.path.join(BASE_DIR, "package.json")) + +URL_TO_PROJECT = '/' # Wikipedia specific probably USE_WEBSOCKETS = False -SECRET_KEY = 'default' -DATABASE_TYPE = 'postgres' -DATABASE_USER = 'postgres' -DATABASE_PASSWORD = '' +STATIC_IP = "" USE_MOCKS = False +# Wikipedia specific hack to fix prefixes of paths +# Probably couldn't use SCRIPT_NAME properly in that environment +# But can't remember exactly ... +SEGMENTS_BEFORE_PROJECT_ROOT = len([segment for segment in URL_TO_PROJECT.split('/') if segment]) +SEGMENTS_TO_SERVICE = SEGMENTS_BEFORE_PROJECT_ROOT + 3 # /data/v1// + +# Makes Pillow work more reliable with images from the web from PIL import ImageFile ImageFile.LOAD_TRUNCATED_IMAGES = True ####################################################### -# DATAGROWTH BOOTSTRAP +# INTEGRATION BOOTSTRAP ####################################################### -GOOGLE_CX = os.environ.get("GOOGLE_CX", "004613812033868156538:5pcwbuudj1m") +GOOGLE_CX = environment.datascope.google_cx +GOOGLE_API_KEY = environment.datascope.google_api_key + +WIKI_USER = environment.datascope.wiki_user +WIKI_PASSWORD = environment.datascope.wiki_password ####################################################### -# LOAD ENVIRONMENT +# DATAGROWTH BOOTSTRAP ####################################################### -try: - from .bootstrap import * -except ImportError: - log.warning("Could not import bootstrap settings. Are they created?") -try: - from .secrets import * -except ImportError: - log.error("Could not import secret settings. Are they created? Do not run in production!") - -DATABASE_USER = os.environ.get('DJANGO_DATABASE_USER', DATABASE_USER) -DATABASE_PASSWORD = os.environ.get('DJANGO_DATABASE_PASSWORD', DATABASE_PASSWORD) +DATAGROWTH_DATA_DIR = os.environ.get('DATAGROWTH_DATA_DIR', os.path.join(BASE_DIR, "..", "data")) +DATAGROWTH_BIN_DIR = os.path.join(DATAGROWTH_DATA_DIR, "bin") ####################################################### # DJANGO SETTINGS ####################################################### -DEBUG = False -DEBUG_TOOLBAR = False +# Django SECRET_KEY +# Documentation: https://docs.djangoproject.com/en/1.11/ref/settings/#std:setting-SECRET_KEY +SECRET_KEY = environment.django.secret_key + +DEBUG = environment.django.debug +DEBUG_TOOLBAR = DEBUG MAX_BATCH_SIZE = 1000 -PATH_TO_LOGS = PATH_TO_PROJECT + "datascope/logs/" +PATH_TO_LOGS = os.path.join(BASE_DIR, "datascope", "logs") INSTALLED_APPS = ( 'django.contrib.auth', @@ -97,10 +107,10 @@ 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', 'NAME': 'datascope', - 'USER': DATABASE_USER, - 'PASSWORD': DATABASE_PASSWORD, - 'HOST': '127.0.0.1', - 'PORT': os.environ.get('PGPORT', '5432') + 'USER': environment.django.database_user, + 'PASSWORD': environment.django.database_password, + 'HOST': environment.postgres.host, + #'PORT': os.environ.get('PGPORT', '5432') } } @@ -136,7 +146,8 @@ 'www.2ndhandstylist.com', ] - +# Do not redirect when slashes at the end are missing +# Can probably be enabled again if we rewrite paths when upgrading to Django 2.2 APPEND_SLASH = False # Local time zone for this installation. Choices can be found here: @@ -163,56 +174,21 @@ # If you set this to False, Django will not use timezone-aware datetimes. USE_TZ = False -# Available languages for all projects -ugettext = lambda s: s # a dummy ugettext to prevent circular import -LANGUAGES = ( - ('en', ugettext('English')), - ('pt', ugettext('Portuguese')), - ('nl', ugettext('Dutch')), - ('de', ugettext('German')), - ('es', ugettext('Spanish')), - ('fr', ugettext('French')), -) - -LOCALE_PATHS = ( - PATH_TO_PROJECT + 'src/locale/', -) - -SEGMENTS_BEFORE_PROJECT_ROOT = len([segment for segment in URL_TO_PROJECT.split('/') if segment]) -SEGMENTS_TO_SERVICE = SEGMENTS_BEFORE_PROJECT_ROOT + 3 # /data/v1// -# Absolute filesystem path to the directory that will hold user-uploaded files. -# Example: "/home/media/media.lawrence.com/media/" -MEDIA_ROOT = os.path.join(BASE_DIR, 'data', 'media', '') +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/2.2/howto/static-files/ -# URL that handles the media served from MEDIA_ROOT. Make sure to use a -# trailing slash. -# Examples: "http://media.lawrence.com/media/", "http://example.com/media/" -MEDIA_URL = URL_TO_PROJECT + 'media/' - -# Absolute path to the directory static files should be collected to. -# Don't put anything in this directory yourself; store your static files -# in apps' "static/" subdirectories and in STATICFILES_DIRS. -# Example: "/home/media/media.lawrence.com/static/" -STATIC_ROOT = PATH_TO_PROJECT + 'datascope/statics/' - -# URL prefix for static files. -# Example: "http://media.lawrence.com/static/" STATIC_URL = URL_TO_PROJECT + 'static/' +STATIC_ROOT = os.path.join(BASE_DIR, 'datascope', 'statics') +STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage' +WHITENOISE_INDEX_FILE = 'index.html' +MEDIA_URL = URL_TO_PROJECT + 'media/' +MEDIA_ROOT = os.path.join(DATAGROWTH_DATA_DIR, "media") -# Additional locations of static files -STATICFILES_DIRS = ( - # Put strings here, like "/home/html/static" or "C:/www/django/static". - # Always use forward slashes, even on Windows. - # Don't forget to use absolute paths, not relative paths. -) +if DEBUG: + WHITENOISE_USE_FINDERS = True + WHITENOISE_AUTOREFRESH = True -# List of finder classes that know how to find static files in -# various locations. -STATICFILES_FINDERS = ( - 'django.contrib.staticfiles.finders.FileSystemFinder', - 'django.contrib.staticfiles.finders.AppDirectoriesFinder', -) TEMPLATES = [ { @@ -239,8 +215,10 @@ }, ] -MIDDLEWARE_CLASSES = ( - 'corsheaders.middleware.CorsMiddleware', +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'corsheaders.middleware.CorsMiddleware', # Added for CORS policy (Cross Origin Resource Security) + 'whitenoise.middleware.WhiteNoiseMiddleware', # Added for serving static files 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.locale.LocaleMiddleware', 'django.middleware.common.CommonMiddleware', @@ -248,7 +226,7 @@ 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', -) +] ROOT_URLCONF = 'datascope.urls' @@ -256,7 +234,7 @@ WSGI_APPLICATION = 'datascope.wsgi.application' FIXTURE_DIRS = ( - PATH_TO_PROJECT + 'core/tests/fixtures/', + os.path.join(BASE_DIR, 'core', 'tests', 'fixtures'), ) LOGGING = { @@ -280,7 +258,7 @@ 'file': { 'level': 'DEBUG', 'class': 'logging.handlers.TimedRotatingFileHandler', - 'filename': PATH_TO_LOGS + 'datascope.log', + 'filename': os.path.join(PATH_TO_LOGS, 'datascope.log'), 'when': 'midnight', 'backupCount': 10, }, @@ -319,7 +297,7 @@ } # Celery settings -CELERY_BROKER_URL = 'redis://localhost:6379/0' +CELERY_BROKER_URL = 'redis://127.0.0.1:6379/0' CELERY_RESULT_BACKEND = "django-db" CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' @@ -331,6 +309,12 @@ EMAIL_PORT = 587 EMAIL_SUBJECT_PREFIX = "[datascope] " SERVER_EMAIL = "no-reply@fakoberkers.nl" +ADMINS = ( + (environment.django.administrator.name, environment.django.administrator.email), +) +MANAGERS = ( + (environment.django.manager.name, environment.django.manager.email), +) ####################################################### # PLUGIN SETTINGS @@ -346,3 +330,44 @@ WSGI_APPLICATION = 'ws4redis.django_runserver.application' WS4REDIS_EXPIRE = 0 WS4REDIS_HEARTBEAT = '--heartbeat--' + + +# Debug toolbar +# https://django-debug-toolbar.readthedocs.io/en/latest/index.html + +if DEBUG_TOOLBAR: + # Activation + INSTALLED_APPS += ( + 'debug_toolbar', + ) + MIDDLEWARE = MIDDLEWARE[0:3] + ['debug_toolbar.middleware.DebugToolbarMiddleware'] + MIDDLEWARE[3:] + + # Configuration + INTERNAL_IPS = [ + '127.0.0.1', + 'localhost:8000', + ] + DEBUG_TOOLBAR_CONFIG = { + "SHOW_TOOLBAR_CALLBACK": lambda request: DEBUG + } + + +# Elastic Search +# https://elasticsearch-py.readthedocs.io/en/master/ + +ELASTIC_SEARCH_ANALYSERS = { + 'en': 'english', + 'nl': 'dutch' +} + + +# Sentry error reporting +# https://sentry.io + +if not DEBUG: + sentry_sdk.init( + dsn="https://407d0ac6dc4542c9a60fb299e32e464d@sentry.io/241870", + integrations=[DjangoIntegration()], + release=DATASCOPE_VERSION, + server_name='data-scope.com' + ) diff --git a/src/datascope/settings_experimental.py b/src/datascope/settings_experimental.py new file mode 100644 index 00000000..2f5fc0aa --- /dev/null +++ b/src/datascope/settings_experimental.py @@ -0,0 +1,5 @@ +from .settings_base import * + + +USE_WEBSOCKETS = True +STATIC_IP = "192.168.43.107" diff --git a/datascope/settings_test.py b/src/datascope/settings_test.py similarity index 98% rename from datascope/settings_test.py rename to src/datascope/settings_test.py index 9b853fb6..c42ee9ed 100644 --- a/datascope/settings_test.py +++ b/src/datascope/settings_test.py @@ -5,6 +5,7 @@ MAX_BATCH_SIZE = None # better for sqlite to let Django determine batch size STATIC_IP = "127.0.0.1" USE_MOCKS = True +DEBUG_TOOLBAR = True LOGGING["loggers"] = {} diff --git a/datascope/urls.py b/src/datascope/urls.py similarity index 96% rename from datascope/urls.py rename to src/datascope/urls.py index 6bc1e014..0069c0b1 100644 --- a/datascope/urls.py +++ b/src/datascope/urls.py @@ -55,7 +55,8 @@ url(r'^data/v1/', include(legacy_patterns, namespace="v1")), url(r'^api/v1/auth/token/?$', rest_views.obtain_auth_token), url(r'^api/v1/', include(datagrowth_patterns, namespace="api-v1")), - url(r'^admin/', include(admin.site.urls)) + url(r'^admin/', include(admin.site.urls)), + url(r'^health/?$', views.health_check), ] diff --git a/src/datascope/version.py b/src/datascope/version.py new file mode 100644 index 00000000..c27b08fb --- /dev/null +++ b/src/datascope/version.py @@ -0,0 +1,7 @@ +import json + + +def get_project_version(package_path): + with open(package_path) as package_file: + package = json.load(package_file) + return package["version"] diff --git a/datascope/views.py b/src/datascope/views.py similarity index 64% rename from datascope/views.py rename to src/datascope/views.py index ec30041e..e18a20c1 100644 --- a/datascope/views.py +++ b/src/datascope/views.py @@ -5,6 +5,10 @@ from django.shortcuts import HttpResponse from django.views.decorators import csrf, http from django.core.mail import mail_managers +from rest_framework.response import Response +from rest_framework import status +from rest_framework.decorators import api_view, schema, permission_classes +from rest_framework.permissions import AllowAny def index(request): @@ -20,3 +24,10 @@ def question(request): text = pformat(data, indent=4) mail_managers(data['question'], text) return HttpResponse('send') + + +@api_view() +@permission_classes([AllowAny]) +@schema(None) +def health_check(request): + return Response({"healthy" : True}, status.HTTP_200_OK) diff --git a/datascope/wsgi.py b/src/datascope/wsgi.py similarity index 100% rename from datascope/wsgi.py rename to src/datascope/wsgi.py diff --git a/src/entrypoint.sh b/src/entrypoint.sh new file mode 100755 index 00000000..05ce8a43 --- /dev/null +++ b/src/entrypoint.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + + +# Exit immediately on error +set -e + + +if [ "$INVOKE_DJANGO_DEBUG" == "1" ] && [ -d "/usr/src/datagrowth" ] && \ + [ $(pip show datagrowth | grep "Location:" | awk -F "/" '{print $NF}') == "site-packages" ] +then + echo "Replacing datagrowth PyPi installation with editable version" + pip uninstall -y datagrowth + pip install -e /usr/src/datagrowth +fi + + +# Executing the normal commands +exec "$@" diff --git a/future_fashion/README.md b/src/future_fashion/README.md similarity index 100% rename from future_fashion/README.md rename to src/future_fashion/README.md diff --git a/future_fashion/__init__.py b/src/future_fashion/__init__.py similarity index 100% rename from future_fashion/__init__.py rename to src/future_fashion/__init__.py diff --git a/future_fashion/admin.py b/src/future_fashion/admin.py similarity index 100% rename from future_fashion/admin.py rename to src/future_fashion/admin.py diff --git a/future_fashion/apps.py b/src/future_fashion/apps.py similarity index 100% rename from future_fashion/apps.py rename to src/future_fashion/apps.py diff --git a/future_fashion/colors.py b/src/future_fashion/colors.py similarity index 100% rename from future_fashion/colors.py rename to src/future_fashion/colors.py diff --git a/future_fashion/docs/stylist-research.md b/src/future_fashion/docs/stylist-research.md similarity index 100% rename from future_fashion/docs/stylist-research.md rename to src/future_fashion/docs/stylist-research.md diff --git a/future_fashion/frames.py b/src/future_fashion/frames.py similarity index 100% rename from future_fashion/frames.py rename to src/future_fashion/frames.py diff --git a/future_fashion/migrations/__init__.py b/src/future_fashion/management/__init__.py similarity index 100% rename from future_fashion/migrations/__init__.py rename to src/future_fashion/management/__init__.py diff --git a/future_fashion/models/sources/__init__.py b/src/future_fashion/management/commands/__init__.py similarity index 100% rename from future_fashion/models/sources/__init__.py rename to src/future_fashion/management/commands/__init__.py diff --git a/future_fashion/management/commands/cluster_future_fashion.py b/src/future_fashion/management/commands/cluster_future_fashion.py similarity index 100% rename from future_fashion/management/commands/cluster_future_fashion.py rename to src/future_fashion/management/commands/cluster_future_fashion.py diff --git a/future_fashion/management/commands/match_face_colors.py b/src/future_fashion/management/commands/match_face_colors.py similarity index 100% rename from future_fashion/management/commands/match_face_colors.py rename to src/future_fashion/management/commands/match_face_colors.py diff --git a/future_fashion/management/commands/match_image_colors.py b/src/future_fashion/management/commands/match_image_colors.py similarity index 100% rename from future_fashion/management/commands/match_image_colors.py rename to src/future_fashion/management/commands/match_image_colors.py diff --git a/future_fashion/management/commands/match_palette_colors.py b/src/future_fashion/management/commands/match_palette_colors.py similarity index 100% rename from future_fashion/management/commands/match_palette_colors.py rename to src/future_fashion/management/commands/match_palette_colors.py diff --git a/future_fashion/management/commands/store_clothing_frames.py b/src/future_fashion/management/commands/store_clothing_frames.py similarity index 100% rename from future_fashion/management/commands/store_clothing_frames.py rename to src/future_fashion/management/commands/store_clothing_frames.py diff --git a/future_fashion/migrations/0001_squashed_0014_annotation.py b/src/future_fashion/migrations/0001_squashed_0014_annotation.py similarity index 100% rename from future_fashion/migrations/0001_squashed_0014_annotation.py rename to src/future_fashion/migrations/0001_squashed_0014_annotation.py diff --git a/future_fashion/migrations/0002_empty_schema_default.py b/src/future_fashion/migrations/0002_empty_schema_default.py similarity index 100% rename from future_fashion/migrations/0002_empty_schema_default.py rename to src/future_fashion/migrations/0002_empty_schema_default.py diff --git a/nautilus/management/__init__.py b/src/future_fashion/migrations/__init__.py similarity index 100% rename from nautilus/management/__init__.py rename to src/future_fashion/migrations/__init__.py diff --git a/future_fashion/models/__init__.py b/src/future_fashion/models/__init__.py similarity index 100% rename from future_fashion/models/__init__.py rename to src/future_fashion/models/__init__.py diff --git a/future_fashion/models/annotation.py b/src/future_fashion/models/annotation.py similarity index 100% rename from future_fashion/models/annotation.py rename to src/future_fashion/models/annotation.py diff --git a/future_fashion/models/clothing_sets.py b/src/future_fashion/models/clothing_sets.py similarity index 100% rename from future_fashion/models/clothing_sets.py rename to src/future_fashion/models/clothing_sets.py diff --git a/future_fashion/models/data.py b/src/future_fashion/models/data.py similarity index 100% rename from future_fashion/models/data.py rename to src/future_fashion/models/data.py diff --git a/future_fashion/models/inventory.py b/src/future_fashion/models/inventory.py similarity index 100% rename from future_fashion/models/inventory.py rename to src/future_fashion/models/inventory.py diff --git a/nautilus/management/commands/__init__.py b/src/future_fashion/models/sources/__init__.py similarity index 100% rename from nautilus/management/commands/__init__.py rename to src/future_fashion/models/sources/__init__.py diff --git a/future_fashion/models/sources/files.py b/src/future_fashion/models/sources/files.py similarity index 100% rename from future_fashion/models/sources/files.py rename to src/future_fashion/models/sources/files.py diff --git a/future_fashion/models/sources/indico.py b/src/future_fashion/models/sources/indico.py similarity index 100% rename from future_fashion/models/sources/indico.py rename to src/future_fashion/models/sources/indico.py diff --git a/future_fashion/models/sources/recognition.py b/src/future_fashion/models/sources/recognition.py similarity index 100% rename from future_fashion/models/sources/recognition.py rename to src/future_fashion/models/sources/recognition.py diff --git a/future_fashion/models/storage.py b/src/future_fashion/models/storage.py similarity index 100% rename from future_fashion/models/storage.py rename to src/future_fashion/models/storage.py diff --git a/future_fashion/processors/__init__.py b/src/future_fashion/processors/__init__.py similarity index 100% rename from future_fashion/processors/__init__.py rename to src/future_fashion/processors/__init__.py diff --git a/future_fashion/processors/clothing_matchers.py b/src/future_fashion/processors/clothing_matchers.py similarity index 100% rename from future_fashion/processors/clothing_matchers.py rename to src/future_fashion/processors/clothing_matchers.py diff --git a/future_fashion/results/little-french-dress.json b/src/future_fashion/results/little-french-dress.json similarity index 100% rename from future_fashion/results/little-french-dress.json rename to src/future_fashion/results/little-french-dress.json diff --git a/future_fashion/templates/future_fashion/accepted.html b/src/future_fashion/templates/future_fashion/accepted.html similarity index 100% rename from future_fashion/templates/future_fashion/accepted.html rename to src/future_fashion/templates/future_fashion/accepted.html diff --git a/future_fashion/templates/future_fashion/bad-request.html b/src/future_fashion/templates/future_fashion/bad-request.html similarity index 100% rename from future_fashion/templates/future_fashion/bad-request.html rename to src/future_fashion/templates/future_fashion/bad-request.html diff --git a/future_fashion/templates/future_fashion/index.html b/src/future_fashion/templates/future_fashion/index.html similarity index 100% rename from future_fashion/templates/future_fashion/index.html rename to src/future_fashion/templates/future_fashion/index.html diff --git a/future_fashion/templates/future_fashion/no-content.html b/src/future_fashion/templates/future_fashion/no-content.html similarity index 100% rename from future_fashion/templates/future_fashion/no-content.html rename to src/future_fashion/templates/future_fashion/no-content.html diff --git a/future_fashion/templates/future_fashion/ok.html b/src/future_fashion/templates/future_fashion/ok.html similarity index 100% rename from future_fashion/templates/future_fashion/ok.html rename to src/future_fashion/templates/future_fashion/ok.html diff --git a/future_fashion/tests.py b/src/future_fashion/tests.py similarity index 100% rename from future_fashion/tests.py rename to src/future_fashion/tests.py diff --git a/future_fashion/urls.py b/src/future_fashion/urls.py similarity index 100% rename from future_fashion/urls.py rename to src/future_fashion/urls.py diff --git a/future_fashion/views/__init__.py b/src/future_fashion/views/__init__.py similarity index 100% rename from future_fashion/views/__init__.py rename to src/future_fashion/views/__init__.py diff --git a/future_fashion/views/annotation.py b/src/future_fashion/views/annotation.py similarity index 100% rename from future_fashion/views/annotation.py rename to src/future_fashion/views/annotation.py diff --git a/future_fashion/views/clothing_sets.py b/src/future_fashion/views/clothing_sets.py similarity index 100% rename from future_fashion/views/clothing_sets.py rename to src/future_fashion/views/clothing_sets.py diff --git a/future_fashion/views/data.py b/src/future_fashion/views/data.py similarity index 100% rename from future_fashion/views/data.py rename to src/future_fashion/views/data.py diff --git a/future_fashion/views/inventory.py b/src/future_fashion/views/inventory.py similarity index 100% rename from future_fashion/views/inventory.py rename to src/future_fashion/views/inventory.py diff --git a/future_fashion/views/swipe.py b/src/future_fashion/views/swipe.py similarity index 100% rename from future_fashion/views/swipe.py rename to src/future_fashion/views/swipe.py diff --git a/manage.py b/src/manage.py similarity index 100% rename from manage.py rename to src/manage.py diff --git a/nautilus/__init__.py b/src/nautilus/__init__.py similarity index 100% rename from nautilus/__init__.py rename to src/nautilus/__init__.py diff --git a/nautilus/admin.py b/src/nautilus/admin.py similarity index 100% rename from nautilus/admin.py rename to src/nautilus/admin.py diff --git a/nautilus/apps.py b/src/nautilus/apps.py similarity index 100% rename from nautilus/apps.py rename to src/nautilus/apps.py diff --git a/nautilus/locafora.py b/src/nautilus/locafora.py similarity index 100% rename from nautilus/locafora.py rename to src/nautilus/locafora.py diff --git a/nautilus/migrations/__init__.py b/src/nautilus/management/__init__.py similarity index 100% rename from nautilus/migrations/__init__.py rename to src/nautilus/management/__init__.py diff --git a/online_discourse/discourse/configurations/en/__init__.py b/src/nautilus/management/commands/__init__.py similarity index 100% rename from online_discourse/discourse/configurations/en/__init__.py rename to src/nautilus/management/commands/__init__.py diff --git a/nautilus/management/commands/grow_locafora.py b/src/nautilus/management/commands/grow_locafora.py similarity index 100% rename from nautilus/management/commands/grow_locafora.py rename to src/nautilus/management/commands/grow_locafora.py diff --git a/nautilus/migrations/0001_initial.py b/src/nautilus/migrations/0001_initial.py similarity index 100% rename from nautilus/migrations/0001_initial.py rename to src/nautilus/migrations/0001_initial.py diff --git a/nautilus/migrations/0002_auto_20171119_1315.py b/src/nautilus/migrations/0002_auto_20171119_1315.py similarity index 100% rename from nautilus/migrations/0002_auto_20171119_1315.py rename to src/nautilus/migrations/0002_auto_20171119_1315.py diff --git a/nautilus/migrations/0003_auto_20181202_1424.py b/src/nautilus/migrations/0003_auto_20181202_1424.py similarity index 100% rename from nautilus/migrations/0003_auto_20181202_1424.py rename to src/nautilus/migrations/0003_auto_20181202_1424.py diff --git a/nautilus/migrations/0004_auto_20190402_0928.py b/src/nautilus/migrations/0004_auto_20190402_0928.py similarity index 100% rename from nautilus/migrations/0004_auto_20190402_0928.py rename to src/nautilus/migrations/0004_auto_20190402_0928.py diff --git a/online_discourse/discourse/configurations/nl/__init__.py b/src/nautilus/migrations/__init__.py similarity index 100% rename from online_discourse/discourse/configurations/nl/__init__.py rename to src/nautilus/migrations/__init__.py diff --git a/nautilus/models/__init__.py b/src/nautilus/models/__init__.py similarity index 100% rename from nautilus/models/__init__.py rename to src/nautilus/models/__init__.py diff --git a/nautilus/models/community.py b/src/nautilus/models/community.py similarity index 100% rename from nautilus/models/community.py rename to src/nautilus/models/community.py diff --git a/nautilus/models/sources.py b/src/nautilus/models/sources.py similarity index 100% rename from nautilus/models/sources.py rename to src/nautilus/models/sources.py diff --git a/nautilus/tasks.py b/src/nautilus/tasks.py similarity index 100% rename from nautilus/tasks.py rename to src/nautilus/tasks.py diff --git a/nautilus/tests.py b/src/nautilus/tests.py similarity index 100% rename from nautilus/tests.py rename to src/nautilus/tests.py diff --git a/nautilus/urls.py b/src/nautilus/urls.py similarity index 100% rename from nautilus/urls.py rename to src/nautilus/urls.py diff --git a/nautilus/views.py b/src/nautilus/views.py similarity index 100% rename from nautilus/views.py rename to src/nautilus/views.py diff --git a/online_discourse/DESCRIPTION.md b/src/online_discourse/DESCRIPTION.md similarity index 100% rename from online_discourse/DESCRIPTION.md rename to src/online_discourse/DESCRIPTION.md diff --git a/online_discourse/__init__.py b/src/online_discourse/__init__.py similarity index 100% rename from online_discourse/__init__.py rename to src/online_discourse/__init__.py diff --git a/online_discourse/admin.py b/src/online_discourse/admin.py similarity index 77% rename from online_discourse/admin.py rename to src/online_discourse/admin.py index 827350e6..c02a0950 100644 --- a/online_discourse/admin.py +++ b/src/online_discourse/admin.py @@ -3,13 +3,17 @@ from datagrowth.admin import HttpResourceAdmin, ShellResourceAdmin, DocumentAdmin, DataStorageAdmin from core.admin.organisms import CommunityAdmin from online_discourse.models import (DiscourseSearchCommunity, DiscourseOrder, ComplexityAnalysis, WebTextResource, - WebContentDownload, WebTextTikaResource, Collection, Document) + WebContentDownload, WebTextTikaResource, Collection, Document, ElasticIndex) class DiscourseOrderAdmin(admin.ModelAdmin): list_display = ("__str__", "name", "email", "topic", "created_at") +class ElasticIndexAdmin(admin.ModelAdmin): + list_display = ("signature", "remote_name", "remote_exists", "error_count", "language", "created_at", "modified_at",) + + admin.site.register(DiscourseSearchCommunity, CommunityAdmin) admin.site.register(DiscourseOrder, DiscourseOrderAdmin) admin.site.register(ComplexityAnalysis, HttpResourceAdmin) @@ -18,3 +22,4 @@ class DiscourseOrderAdmin(admin.ModelAdmin): admin.site.register(WebTextTikaResource, ShellResourceAdmin) admin.site.register(Collection, DataStorageAdmin) admin.site.register(Document, DocumentAdmin) +admin.site.register(ElasticIndex, ElasticIndexAdmin) diff --git a/online_discourse/apps.py b/src/online_discourse/apps.py similarity index 100% rename from online_discourse/apps.py rename to src/online_discourse/apps.py diff --git a/online_discourse/discourse/__init__.py b/src/online_discourse/discourse/__init__.py similarity index 100% rename from online_discourse/discourse/__init__.py rename to src/online_discourse/discourse/__init__.py diff --git a/online_discourse/discourse/configurations/__init__.py b/src/online_discourse/discourse/configurations/__init__.py similarity index 79% rename from online_discourse/discourse/configurations/__init__.py rename to src/online_discourse/discourse/configurations/__init__.py index 777aae2c..5e4f6ef5 100644 --- a/online_discourse/discourse/configurations/__init__.py +++ b/src/online_discourse/discourse/configurations/__init__.py @@ -8,3 +8,5 @@ from .nl.biomassa import biomassa from .nl.koopvaardij_bescherming import koopvaardij_bescherming from .nl.diepzee_mijnbouw import diepzee_mijnbouw +from .nl.europees_leger import europees_leger +from .nl.drugs_legalisatie import drugs_legalisatie diff --git a/online_discourse/migrations/__init__.py b/src/online_discourse/discourse/configurations/en/__init__.py similarity index 100% rename from online_discourse/migrations/__init__.py rename to src/online_discourse/discourse/configurations/en/__init__.py diff --git a/online_discourse/discourse/configurations/en/death_penalty.py b/src/online_discourse/discourse/configurations/en/death_penalty.py similarity index 100% rename from online_discourse/discourse/configurations/en/death_penalty.py rename to src/online_discourse/discourse/configurations/en/death_penalty.py diff --git a/online_discourse/discourse/configurations/en/family_planning.py b/src/online_discourse/discourse/configurations/en/family_planning.py similarity index 100% rename from online_discourse/discourse/configurations/en/family_planning.py rename to src/online_discourse/discourse/configurations/en/family_planning.py diff --git a/online_discourse/discourse/configurations/en/gay_rights.py b/src/online_discourse/discourse/configurations/en/gay_rights.py similarity index 100% rename from online_discourse/discourse/configurations/en/gay_rights.py rename to src/online_discourse/discourse/configurations/en/gay_rights.py diff --git a/open_data/management/__init__.py b/src/online_discourse/discourse/configurations/nl/__init__.py similarity index 100% rename from open_data/management/__init__.py rename to src/online_discourse/discourse/configurations/nl/__init__.py diff --git a/online_discourse/discourse/configurations/nl/biomassa.py b/src/online_discourse/discourse/configurations/nl/biomassa.py similarity index 100% rename from online_discourse/discourse/configurations/nl/biomassa.py rename to src/online_discourse/discourse/configurations/nl/biomassa.py diff --git a/online_discourse/discourse/configurations/nl/diepzee_mijnbouw.py b/src/online_discourse/discourse/configurations/nl/diepzee_mijnbouw.py similarity index 100% rename from online_discourse/discourse/configurations/nl/diepzee_mijnbouw.py rename to src/online_discourse/discourse/configurations/nl/diepzee_mijnbouw.py diff --git a/online_discourse/discourse/configurations/nl/doodstraf.py b/src/online_discourse/discourse/configurations/nl/doodstraf.py similarity index 100% rename from online_discourse/discourse/configurations/nl/doodstraf.py rename to src/online_discourse/discourse/configurations/nl/doodstraf.py diff --git a/src/online_discourse/discourse/configurations/nl/drugs_legalisatie.py b/src/online_discourse/discourse/configurations/nl/drugs_legalisatie.py new file mode 100644 index 00000000..f413cb82 --- /dev/null +++ b/src/online_discourse/discourse/configurations/nl/drugs_legalisatie.py @@ -0,0 +1,76 @@ +from online_discourse.discourse import DiscourseConfiguration + + +SINGULAR_SUBJECTS = [ + "Europese Unie", + "Jellinek", + "Festival", + "politie", + "GGZ", + "Leger des Heils", + "D66", + "VVD", + "Groen Links", + "SGP", + "CDA", + "PvdD", + "PvdA", + "Religie", + "Junk", + "Gebruiker", + "Crimineel", + "Drugsbaron", + "Cartel", +] + +PLURAL_SUBJECTS = [ + "Coffeeshops", + "Lidstaten", + "Criminele organisaties", + "Ziekenhuizen", + "Artsen", + "Scholen", + "BN-ers", + "bekende Nederlanders", + "Ouders", + "Influencers", +] + +DESCRIPTIVE_ADJECTIVES = [ + "afname criminaliteit", + "vrije keuze", + "Medicijn", + "economie", + "zwartgeldcircuit", + "Kwaliteitsgarantie", + "gevangenen", + "oorlog tegen drugs", + "goedkoper", + "medicinaal", + "Volksgezondheid", + "Verslaving", + "Medische kosten", + "jeugdbescherming", + "Criminaliteit verschuift", + "gateway drug", + "diplomatie", + "Milieuschade", + "Stepping stone", + "harde criminaliteit", + "Zonde", + "haram", + "handel", + "Verbod", +] + +drugs_legalisatie = DiscourseConfiguration( + title="Drugs legalisatie", + description="Nederland kent al heel lang een gedoog beleid waarbij de verkoop van softdrugs is toegestaan, " + "maar de inkoop niet. Men spreekt al een tijd over het geheel legaliseren van softdrugs, " + "maar tot nu toe zonder resultaat.", + language="nl", + topics=["drugs", "softdrugs"], + singular_subjects=SINGULAR_SUBJECTS, + plural_subjects=PLURAL_SUBJECTS, + descriptive_adjectives=DESCRIPTIVE_ADJECTIVES +) diff --git a/src/online_discourse/discourse/configurations/nl/europees_leger.py b/src/online_discourse/discourse/configurations/nl/europees_leger.py new file mode 100644 index 00000000..2c74cb78 --- /dev/null +++ b/src/online_discourse/discourse/configurations/nl/europees_leger.py @@ -0,0 +1,81 @@ +from online_discourse.discourse import DiscourseConfiguration + + +SINGULAR_SUBJECTS = [ + "Wapenhandelaar", + "Wapenindustrie", + "Vredesactivist", + "Regering", + "Media", + "Parlement", + "Generaal", + "Diplomaat", + "Soldaat", + "Wapenlobby", + "NAVO", + "Europese Unie", + "Verenigde Naties", + "Poetin", + "China", + "Marine", + "Defensie", + "Luchtmacht", + "OVSE", + "Schiphol", + "Stork", + "Damen Shipyards", + "TNO", + "Universiteit", + "Airbus", + "Instituut Clingendael", + "Europese Commissie", + "Raad van Europa", + "Europees Parlement", + "Europese Raad", +] + +PLURAL_SUBJECTS = [ + "Vredesactivisten", + "Diplomaten", + "Universiteiten", + "Soldaten", +] + +DESCRIPTIVE_ADJECTIVES = [ + "Wapenwedloop", + "Minder soevereiniteit", + "Band met Afrika", + "Economische schade", + "Onenigheid lidstaten", + "NAVO is sterker", + "bodybags", + "Veiligheid", + "Macht", + "Binding lidstaten", + "Samenwerking lidstaten", + "budget", + "Europe first", + "ontwikkeling van de EU", + "3D-diplomacy", + "Sterker tegenover Poetin", + "Sterker tegenover Erdogan", + "Conflict Oekraïne is oplosbaar", + "Supranationale besluitvorming", + "Vrede", + "Banen", + "Export", + "Industrieel militair complex", + "cohesie", +] + +europees_leger = DiscourseConfiguration( + title="Europees leger", + description="Er is veel kritiek op de Europese Unie. " + "Toch zijn er ook geluiden, die aangeven dat we meer zouden moeten samenwerken op Europees niveau. " + "Bijvoorbeeld op militair gebied.", + language="nl", + topics=["leger"], + singular_subjects=SINGULAR_SUBJECTS, + plural_subjects=PLURAL_SUBJECTS, + descriptive_adjectives=DESCRIPTIVE_ADJECTIVES +) diff --git a/online_discourse/discourse/configurations/nl/immigranten.py b/src/online_discourse/discourse/configurations/nl/immigranten.py similarity index 100% rename from online_discourse/discourse/configurations/nl/immigranten.py rename to src/online_discourse/discourse/configurations/nl/immigranten.py diff --git a/online_discourse/discourse/configurations/nl/koopvaardij_bescherming.py b/src/online_discourse/discourse/configurations/nl/koopvaardij_bescherming.py similarity index 100% rename from online_discourse/discourse/configurations/nl/koopvaardij_bescherming.py rename to src/online_discourse/discourse/configurations/nl/koopvaardij_bescherming.py diff --git a/online_discourse/discourse/configurations/nl/wiv.py b/src/online_discourse/discourse/configurations/nl/wiv.py similarity index 100% rename from online_discourse/discourse/configurations/nl/wiv.py rename to src/online_discourse/discourse/configurations/nl/wiv.py diff --git a/src/online_discourse/elastic.py b/src/online_discourse/elastic.py new file mode 100644 index 00000000..91fbe76b --- /dev/null +++ b/src/online_discourse/elastic.py @@ -0,0 +1,58 @@ +from elasticsearch import Elasticsearch + +from django.conf import settings + + +def get_es_client(silent=False): + """ + Returns the elasticsearch client which uses the configuration file + """ + es_client = Elasticsearch(["elastic"], + scheme='http', + port=9200, + http_compress=True) + # test if it works + if not silent and not es_client.cat.health(request_timeout=30): + raise ValueError('Credentials do not work for Elastic search') + return es_client + + +def get_index_config(lang): + """ + Returns the elasticsearch index configuration. + Configures the analysers based on the language passed in. + """ + return { + "settings": { + "index": { + "number_of_shards": 1, + "number_of_replicas": 0 + } + }, + 'mappings': { + '_doc': { + 'properties': { + 'title': { + 'type': 'text', + 'analyzer': settings.ELASTIC_SEARCH_ANALYSERS[lang] + }, + 'content': { + 'type': 'text', + 'analyzer': settings.ELASTIC_SEARCH_ANALYSERS[lang] + }, + 'url': {'type': 'text'}, + 'title_plain': {'type': 'text'}, + 'content_plain': {'type': 'text'}, + 'author': { + 'type': 'keyword' + }, + 'source': { + 'type': 'keyword' + }, + 'argument_score': { + 'type': 'float' + } + } + } + } + } diff --git a/online_discourse/management/commands/analyse_wizenoze.py b/src/online_discourse/management/commands/analyse_wizenoze.py similarity index 100% rename from online_discourse/management/commands/analyse_wizenoze.py rename to src/online_discourse/management/commands/analyse_wizenoze.py diff --git a/src/online_discourse/management/commands/create_elastic_index.py b/src/online_discourse/management/commands/create_elastic_index.py new file mode 100644 index 00000000..20b980e8 --- /dev/null +++ b/src/online_discourse/management/commands/create_elastic_index.py @@ -0,0 +1,51 @@ +import logging + +from django.conf import settings + +from datagrowth.management.base import DatasetCommand + +from online_discourse.discourse import configurations +from online_discourse.models import ElasticIndex +from online_discourse.elastic import get_index_config + + +log = logging.getLogger("datagrowth") + + +class Command(DatasetCommand): + + def add_arguments(self, parser): + super().add_arguments(parser) + parser.add_argument('-r', '--recreate', action="store_true") + parser.add_argument('-p', '--promote', action="store_true") + + def handle_dataset(self, dataset, *args, **options): + + configuration_name = next(arg for arg in dataset.signature.split("&") if "=" not in arg) + configuration = getattr(configurations, configuration_name) + if configuration.language not in settings.ELASTIC_SEARCH_ANALYSERS: + log.error(f"Can't create index for {configuration.title} because {configuration.language} is not supported") + return + + recreate = options["recreate"] + promote = options["promote"] + print(f"Creating index for { configuration.title }; " + f"language:{configuration.language}, recreate:{recreate} and documents:{dataset.documents.count()}") + + elastic_configuration = get_index_config(configuration.language) + index, created = ElasticIndex.objects.get_or_create( + signature=dataset.signature, + language=configuration.language, + defaults={ + "dataset": dataset, + "configuration": elastic_configuration + } + ) + index.clean() + index.push((doc.to_search() for doc in dataset.kernel.document_set.all()), recreate=recreate) + index.save() + if promote: + print(f"Promoting index { index.remote_name } to latest") + index.promote_to_latest() + + log.info(f'{configuration.title} errors:{index.error_count}') diff --git a/online_discourse/management/commands/grow_discourse.py b/src/online_discourse/management/commands/grow_discourse.py similarity index 77% rename from online_discourse/management/commands/grow_discourse.py rename to src/online_discourse/management/commands/grow_discourse.py index 842a8e49..2baf59b8 100644 --- a/online_discourse/management/commands/grow_discourse.py +++ b/src/online_discourse/management/commands/grow_discourse.py @@ -1,6 +1,6 @@ import logging -from core.management.commands.grow_community import Command as GrowCommand +from datagrowth.management.commands.grow_dataset import Command as GrowCommand from core.utils.configuration import DecodeConfigAction from online_discourse.discourse import configurations @@ -11,11 +11,11 @@ class Command(GrowCommand): - community_model = "DiscourseSearchCommunity" + dataset_model = "online_discourse.DiscourseSearchCommunity" def add_arguments(self, parser): parser.add_argument('topic', type=str) - parser.add_argument('--community', type=str, nargs="?", default=self.community_model) + parser.add_argument('--dataset', type=str, nargs="?", default=self.dataset_model) parser.add_argument('-c', '--config', type=str, action=DecodeConfigAction, nargs="?", default={}) def handle(self, *args, **kwargs): diff --git a/online_discourse/management/commands/store_online_discourse_frames.py b/src/online_discourse/management/commands/store_online_discourse_frames.py similarity index 100% rename from online_discourse/management/commands/store_online_discourse_frames.py rename to src/online_discourse/management/commands/store_online_discourse_frames.py diff --git a/online_discourse/migrations/0001_squashed_0010_collection_document.py b/src/online_discourse/migrations/0001_squashed_0010_collection_document.py similarity index 100% rename from online_discourse/migrations/0001_squashed_0010_collection_document.py rename to src/online_discourse/migrations/0001_squashed_0010_collection_document.py diff --git a/online_discourse/migrations/0002_empty_schema_default.py b/src/online_discourse/migrations/0002_empty_schema_default.py similarity index 100% rename from online_discourse/migrations/0002_empty_schema_default.py rename to src/online_discourse/migrations/0002_empty_schema_default.py diff --git a/src/online_discourse/migrations/0003_elastic_search.py b/src/online_discourse/migrations/0003_elastic_search.py new file mode 100644 index 00000000..68ec53b5 --- /dev/null +++ b/src/online_discourse/migrations/0003_elastic_search.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.23 on 2020-01-27 17:41 +from __future__ import unicode_literals + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('online_discourse', '0002_empty_schema_default'), + ] + + operations = [ + migrations.CreateModel( + name='ElasticIndex', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('signature', models.CharField(max_length=255)), + ('language', models.CharField(choices=[('en', 'english'), ('nl', 'dutch')], max_length=5)), + ('configuration', django.contrib.postgres.fields.jsonb.JSONField(blank=True)), + ('error_count', models.IntegerField(default=0)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('modified_at', models.DateTimeField(auto_now=True)), + ('dataset', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='indices', to='online_discourse.DiscourseSearchCommunity')), + ], + options={ + 'verbose_name': 'elastic index', + 'verbose_name_plural': 'elastic indices', + }, + ), + ] diff --git a/open_data/management/commands/__init__.py b/src/online_discourse/migrations/__init__.py similarity index 100% rename from open_data/management/commands/__init__.py rename to src/online_discourse/migrations/__init__.py diff --git a/online_discourse/models/__init__.py b/src/online_discourse/models/__init__.py similarity index 86% rename from online_discourse/models/__init__.py rename to src/online_discourse/models/__init__.py index 2a3475a0..338b5585 100644 --- a/online_discourse/models/__init__.py +++ b/src/online_discourse/models/__init__.py @@ -2,3 +2,4 @@ from .sources import WebTextResource, ComplexityAnalysis, WebContentDownload, WebTextTikaResource from .storage import Collection, Document from .search import DiscourseSearchCommunity +from .elastic import ElasticIndex diff --git a/src/online_discourse/models/elastic.py b/src/online_discourse/models/elastic.py new file mode 100644 index 00000000..eb0ff838 --- /dev/null +++ b/src/online_discourse/models/elastic.py @@ -0,0 +1,90 @@ +from elasticsearch.helpers import streaming_bulk + +from django.conf import settings +from django.db import models +from django.utils.text import slugify +from django.contrib.postgres.fields import JSONField +from rest_framework import serializers + +from online_discourse.models import DiscourseSearchCommunity +from online_discourse.elastic import get_index_config, get_es_client + + +elastic_client = get_es_client(silent=True) + + +class ElasticIndex(models.Model): + + signature = models.CharField(max_length=255) + language = models.CharField(max_length=5, choices=settings.ELASTIC_SEARCH_ANALYSERS.items()) + dataset = models.ForeignKey(DiscourseSearchCommunity, related_name="indices") + configuration = JSONField(blank=True) + error_count = models.IntegerField(default=0) + + created_at = models.DateTimeField(auto_now_add=True) + modified_at = models.DateTimeField(auto_now=True) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.client = elastic_client + + @property + def remote_name(self): + if not self.id: + raise ValueError("Can't get the remote name for an unsaved object") + return slugify("{}-{}-{}".format(self.signature, self.language, self.id)) + + @property + def remote_exists(self): + if not self.id: + raise ValueError("Can't check for existence with an unsaved object") + return self.client.indices.exists(self.remote_name) + + def push(self, elastic_documents, recreate=True, request_timeout=300): # why is the elastic cluster usually slow? + if not self.id: + raise ValueError("Can't push index with unsaved object") + + remote_name = self.remote_name + if self.remote_exists and recreate: + self.client.indices.delete(remote_name) + + self.client.indices.create( + index=remote_name, + body=self.configuration, + request_timeout=request_timeout + ) + if recreate: + self.error_count = 0 + for is_ok, result in streaming_bulk(self.client, elastic_documents, index=remote_name, doc_type="_doc", + chunk_size=100, yield_ok=False, raise_on_error=False, + request_timeout=request_timeout): + if not is_ok: + self.error_count += 1 + print(f'Error in sending bulk:{result}') + self.save() + + def promote_to_latest(self): + latest_alias = "latest-" + self.language + if self.client.indices.exists_alias(name=latest_alias): + self.client.indices.delete_alias(index="_all", name=latest_alias) + self.client.indices.put_alias(index=self.remote_name, name=latest_alias) + + def clean(self): + if not self.signature: + self.signature = self.dataset.signature + if self.language and not self.configuration: + self.configuration = get_index_config(self.language) + + def __str__(self): + return self.signature + + class Meta: + verbose_name = "elastic index" + verbose_name_plural = "elastic indices" + + +class ElasticIndexSerializer(serializers.ModelSerializer): + + class Meta: + model = ElasticIndex + fields = ("id", "signature", "language", "remote_name",) diff --git a/online_discourse/models/orders.py b/src/online_discourse/models/orders.py similarity index 100% rename from online_discourse/models/orders.py rename to src/online_discourse/models/orders.py diff --git a/online_discourse/models/search.py b/src/online_discourse/models/search.py similarity index 97% rename from online_discourse/models/search.py rename to src/online_discourse/models/search.py index 4489c544..d99cf506 100644 --- a/online_discourse/models/search.py +++ b/src/online_discourse/models/search.py @@ -30,6 +30,8 @@ class DiscourseSearchCommunity(CommunityCollectionDocumentMixin, Community): + USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36" + aggregates = json_field.JSONField(default={}, blank=True) COMMUNITY_SPIRIT = OrderedDict([ @@ -66,7 +68,8 @@ class DiscourseSearchCommunity(CommunityCollectionDocumentMixin, Community): "_update_key": "url", "$language": "en", # TODO: think of a way to include language as a parameter without using config # Keys below are used by WebContentDownload to format its output - "resource_key": "resourcePath" + "resource_key": "resourcePath", + "user_agent": USER_AGENT }, "schema": {}, "errors": {}, @@ -83,7 +86,8 @@ class DiscourseSearchCommunity(CommunityCollectionDocumentMixin, Community): "_update_key": "home", # Keys below are used by WebContentDownload to format its output "url_key": "home", - "resource_key": "home_resource" + "resource_key": "home_resource", + "user_agent": USER_AGENT }, "schema": {}, "errors": {}, @@ -298,6 +302,8 @@ def begin_content(self, out): for entry in tqdm(out.documents.iterator(), total=total): entry.clean() entry.save() + assert total != out.documents.filter(identity__isnull=True).count(), \ + "At begin_content there are no documents with any content. Aborting" deletes = out.documents.filter(identity__isnull=True).delete() log.info("Deletes: {}".format(deletes)) diff --git a/online_discourse/models/sources/__init__.py b/src/online_discourse/models/sources/__init__.py similarity index 100% rename from online_discourse/models/sources/__init__.py rename to src/online_discourse/models/sources/__init__.py diff --git a/online_discourse/models/sources/downloads.py b/src/online_discourse/models/sources/downloads.py similarity index 98% rename from online_discourse/models/sources/downloads.py rename to src/online_discourse/models/sources/downloads.py index bdf91fe3..deebf265 100644 --- a/online_discourse/models/sources/downloads.py +++ b/src/online_discourse/models/sources/downloads.py @@ -22,7 +22,7 @@ def content(self): return content_type, file variables = self.variables() return content_type, { - self.config.url_key: variables["url"], + self.config.url_key: variables["url"][0], self.config.resource_key: os.path.join(default_storage.location, self.body) } diff --git a/online_discourse/models/sources/wizenoze.py b/src/online_discourse/models/sources/wizenoze.py similarity index 100% rename from online_discourse/models/sources/wizenoze.py rename to src/online_discourse/models/sources/wizenoze.py diff --git a/online_discourse/models/storage.py b/src/online_discourse/models/storage.py similarity index 63% rename from online_discourse/models/storage.py rename to src/online_discourse/models/storage.py index 498222f6..d9eba915 100644 --- a/online_discourse/models/storage.py +++ b/src/online_discourse/models/storage.py @@ -25,3 +25,17 @@ class Document(DocumentPostgres, DocumentBase): community = GenericForeignKey(ct_field="community_type", fk_field="community_id") community_type = models.ForeignKey(ContentType, related_name="+") community_id = models.PositiveIntegerField() + + def to_search(self): + content = " ".join(partial for partial in self.properties["content"] if partial) + return { + "_id": self.id, + "title": self.properties["title"], + "title_plain": self.properties["title"], + "argument_score": float(self.properties.get("argument_score", 0.0001)), + "url": self.properties["url"], + "author": self.properties["author"], + "source": self.properties["source"], + "content": content, + "content_plain": content + } diff --git a/online_discourse/processors/__init__.py b/src/online_discourse/processors/__init__.py similarity index 100% rename from online_discourse/processors/__init__.py rename to src/online_discourse/processors/__init__.py diff --git a/online_discourse/processors/entities.py b/src/online_discourse/processors/entities.py similarity index 100% rename from online_discourse/processors/entities.py rename to src/online_discourse/processors/entities.py diff --git a/online_discourse/processors/rank.py b/src/online_discourse/processors/rank.py similarity index 100% rename from online_discourse/processors/rank.py rename to src/online_discourse/processors/rank.py diff --git a/online_discourse/processors/topics.py b/src/online_discourse/processors/topics.py similarity index 100% rename from online_discourse/processors/topics.py rename to src/online_discourse/processors/topics.py diff --git a/online_discourse/tests.py b/src/online_discourse/tests.py similarity index 100% rename from online_discourse/tests.py rename to src/online_discourse/tests.py diff --git a/online_discourse/urls.py b/src/online_discourse/urls.py similarity index 84% rename from online_discourse/urls.py rename to src/online_discourse/urls.py index 860b8041..4d3b9995 100644 --- a/online_discourse/urls.py +++ b/src/online_discourse/urls.py @@ -13,6 +13,12 @@ kwargs={"community_class": DiscourseSearchCommunity}, name=DiscourseSearchCommunity.get_name() + "_service" ), + url( + r'^search/(?P.+)/$', + views.DiscourseSearchView.as_view(), + kwargs={"community_class": DiscourseSearchCommunity}, + name=DiscourseSearchCommunity.get_name() + "_search" + ), url( r'^order/?$', views.CreateDiscourseOrder.as_view(), diff --git a/online_discourse/views/__init__.py b/src/online_discourse/views/__init__.py similarity index 83% rename from online_discourse/views/__init__.py rename to src/online_discourse/views/__init__.py index 46daacf3..7d131302 100644 --- a/online_discourse/views/__init__.py +++ b/src/online_discourse/views/__init__.py @@ -1,3 +1,4 @@ from .discourse import DiscourseViewSet from .order import CreateDiscourseOrder, DiscourseOrderSerializer from .data import DocumentView, DocumentContentView, CollectionView, CollectionContentView +from .search import DiscourseSearchView diff --git a/online_discourse/views/data.py b/src/online_discourse/views/data.py similarity index 100% rename from online_discourse/views/data.py rename to src/online_discourse/views/data.py diff --git a/online_discourse/views/discourse.py b/src/online_discourse/views/discourse.py similarity index 100% rename from online_discourse/views/discourse.py rename to src/online_discourse/views/discourse.py diff --git a/online_discourse/views/order.py b/src/online_discourse/views/order.py similarity index 100% rename from online_discourse/views/order.py rename to src/online_discourse/views/order.py diff --git a/src/online_discourse/views/search.py b/src/online_discourse/views/search.py new file mode 100644 index 00000000..06da1a91 --- /dev/null +++ b/src/online_discourse/views/search.py @@ -0,0 +1,95 @@ +from copy import copy + +from django.shortcuts import Http404 +from rest_framework import generics +from rest_framework.response import Response + +from core.views import CommunityView +from online_discourse.elastic import get_es_client +from online_discourse.models import ElasticIndex + + +class DiscourseSearchView(generics.GenericAPIView): + + elastic = get_es_client(silent=True) + + @classmethod + def get_query_body(cls, text=None, authors=None, sources=None): + query = {"bool": {"must": []}} + # Adding full text search + if text and text[0]: # bit of a hack because frontend might send: [""] + query["bool"]["must"].append({ + "multi_match": { + "query": " ".join(text), + "fields": ["content", "title"] + } + }) + # Adding term filters + if authors: + query["bool"]["must"].append({"terms": {"author": authors.split("|")}}) + if sources: + query["bool"]["must"].append({"terms": {"source": sources.split("|")}}) + # We always query with a score modification for the argument_score + return { + "query": { + "function_score": { + "query": query, + "field_value_factor": { + "field": "argument_score", + "factor": 1.0, + "missing": 0.0001 + }, + "boost_mode": "multiply" + } + + }, + "from": 0, + "size": 60 + } + + def get_index_or_raise(self, configuration, community_class, path): + signature = community_class.get_signature_from_input( + *path.split('/'), + **configuration + ) + try: + return ElasticIndex.objects.filter(signature=signature).latest("created_at") + except ElasticIndex.DoesNotExist: + raise Http404("Can not find index with signature=".format(signature)) + + def get_response_from_results(self, results): + hits = results["hits"]["hits"] + response_data = copy(CommunityView.RESPONSE_DATA) + documents = [] + for hit in hits: + document = hit["_source"] + document["_id"] = hit["_id"] + documents.append(document) + response_data["results"] = documents + return Response(response_data) + + def get(self, request, community_class, path=""): + configuration, created_at_info = CommunityView.get_configuration_from_request(request) + body = self.get_query_body() + index = self.get_index_or_raise(configuration, community_class, path) + results = self.elastic.search( + body=body, + index=index.remote_name, + _source=["title", "url", "author", "source", "argument_score"] + ) + return self.get_response_from_results(results) + + def post(self, request, community_class, path=""): + configuration, created_at_info = CommunityView.get_configuration_from_request(request) + body = self.get_query_body( + text=configuration.get("keywords", []), + authors=configuration.get("author", None), + sources=configuration.get("source", None) + ) + index = self.get_index_or_raise(configuration, community_class, path) + results = self.elastic.search( + body=body, + index=index.remote_name, + _source=["title", "url", "author", "source", "argument_score"] + ) + return self.get_response_from_results(results) diff --git a/open_data/__init__.py b/src/open_data/__init__.py similarity index 100% rename from open_data/__init__.py rename to src/open_data/__init__.py diff --git a/open_data/admin.py b/src/open_data/admin.py similarity index 100% rename from open_data/admin.py rename to src/open_data/admin.py diff --git a/open_data/apps.py b/src/open_data/apps.py similarity index 100% rename from open_data/apps.py rename to src/open_data/apps.py diff --git a/open_data/migrations/__init__.py b/src/open_data/management/__init__.py similarity index 100% rename from open_data/migrations/__init__.py rename to src/open_data/management/__init__.py diff --git a/setup_utrecht/migrations/__init__.py b/src/open_data/management/commands/__init__.py similarity index 100% rename from setup_utrecht/migrations/__init__.py rename to src/open_data/management/commands/__init__.py diff --git a/open_data/migrations/0001_initial.py b/src/open_data/migrations/0001_initial.py similarity index 100% rename from open_data/migrations/0001_initial.py rename to src/open_data/migrations/0001_initial.py diff --git a/open_data/migrations/0002_auto_20170320_1518.py b/src/open_data/migrations/0002_auto_20170320_1518.py similarity index 100% rename from open_data/migrations/0002_auto_20170320_1518.py rename to src/open_data/migrations/0002_auto_20170320_1518.py diff --git a/open_data/migrations/0003_officialannouncementsdocumentnetherlands_officialannouncementsnetherlands.py b/src/open_data/migrations/0003_officialannouncementsdocumentnetherlands_officialannouncementsnetherlands.py similarity index 100% rename from open_data/migrations/0003_officialannouncementsdocumentnetherlands_officialannouncementsnetherlands.py rename to src/open_data/migrations/0003_officialannouncementsdocumentnetherlands_officialannouncementsnetherlands.py diff --git a/open_data/migrations/0004_auto_20181202_1424.py b/src/open_data/migrations/0004_auto_20181202_1424.py similarity index 100% rename from open_data/migrations/0004_auto_20181202_1424.py rename to src/open_data/migrations/0004_auto_20181202_1424.py diff --git a/open_data/migrations/0005_auto_20190402_0928.py b/src/open_data/migrations/0005_auto_20190402_0928.py similarity index 100% rename from open_data/migrations/0005_auto_20190402_0928.py rename to src/open_data/migrations/0005_auto_20190402_0928.py diff --git a/sources/__init__.py b/src/open_data/migrations/__init__.py similarity index 100% rename from sources/__init__.py rename to src/open_data/migrations/__init__.py diff --git a/open_data/models/__init__.py b/src/open_data/models/__init__.py similarity index 100% rename from open_data/models/__init__.py rename to src/open_data/models/__init__.py diff --git a/open_data/models/dutch_parliament/community.py b/src/open_data/models/dutch_parliament/community.py similarity index 100% rename from open_data/models/dutch_parliament/community.py rename to src/open_data/models/dutch_parliament/community.py diff --git a/open_data/models/dutch_parliament/sources.py b/src/open_data/models/dutch_parliament/sources.py similarity index 100% rename from open_data/models/dutch_parliament/sources.py rename to src/open_data/models/dutch_parliament/sources.py diff --git a/open_data/tests.py b/src/open_data/tests.py similarity index 100% rename from open_data/tests.py rename to src/open_data/tests.py diff --git a/open_data/urls.py b/src/open_data/urls.py similarity index 100% rename from open_data/urls.py rename to src/open_data/urls.py diff --git a/open_data/views.py b/src/open_data/views.py similarity index 100% rename from open_data/views.py rename to src/open_data/views.py diff --git a/package.json b/src/package.json similarity index 96% rename from package.json rename to src/package.json index a38958dd..4840bea9 100644 --- a/package.json +++ b/src/package.json @@ -1,6 +1,6 @@ { "name": "datascope", - "version": "0.15.0", + "version": "0.16.6", "data": { "digital-ocean": { "data/online_discourse/dumps": "online_discourse/dumps" diff --git a/setup_utrecht/README.md b/src/setup_utrecht/README.md similarity index 100% rename from setup_utrecht/README.md rename to src/setup_utrecht/README.md diff --git a/setup_utrecht/__init__.py b/src/setup_utrecht/__init__.py similarity index 100% rename from setup_utrecht/__init__.py rename to src/setup_utrecht/__init__.py diff --git a/setup_utrecht/admin.py b/src/setup_utrecht/admin.py similarity index 100% rename from setup_utrecht/admin.py rename to src/setup_utrecht/admin.py diff --git a/setup_utrecht/apps.py b/src/setup_utrecht/apps.py similarity index 100% rename from setup_utrecht/apps.py rename to src/setup_utrecht/apps.py diff --git a/setup_utrecht/migrations/0001_initial.py b/src/setup_utrecht/migrations/0001_initial.py similarity index 100% rename from setup_utrecht/migrations/0001_initial.py rename to src/setup_utrecht/migrations/0001_initial.py diff --git a/setup_utrecht/migrations/0002_redditlist_redditpermalink_redditscrapecommunity.py b/src/setup_utrecht/migrations/0002_redditlist_redditpermalink_redditscrapecommunity.py similarity index 100% rename from setup_utrecht/migrations/0002_redditlist_redditpermalink_redditscrapecommunity.py rename to src/setup_utrecht/migrations/0002_redditlist_redditpermalink_redditscrapecommunity.py diff --git a/setup_utrecht/migrations/0003_auto_20181202_1424.py b/src/setup_utrecht/migrations/0003_auto_20181202_1424.py similarity index 100% rename from setup_utrecht/migrations/0003_auto_20181202_1424.py rename to src/setup_utrecht/migrations/0003_auto_20181202_1424.py diff --git a/setup_utrecht/migrations/0004_uniformimagedownload.py b/src/setup_utrecht/migrations/0004_uniformimagedownload.py similarity index 100% rename from setup_utrecht/migrations/0004_uniformimagedownload.py rename to src/setup_utrecht/migrations/0004_uniformimagedownload.py diff --git a/setup_utrecht/migrations/0005_redditimagedownload.py b/src/setup_utrecht/migrations/0005_redditimagedownload.py similarity index 100% rename from setup_utrecht/migrations/0005_redditimagedownload.py rename to src/setup_utrecht/migrations/0005_redditimagedownload.py diff --git a/setup_utrecht/migrations/0006_auto_20190402_0928.py b/src/setup_utrecht/migrations/0006_auto_20190402_0928.py similarity index 100% rename from setup_utrecht/migrations/0006_auto_20190402_0928.py rename to src/setup_utrecht/migrations/0006_auto_20190402_0928.py diff --git a/sources/migrations/__init__.py b/src/setup_utrecht/migrations/__init__.py similarity index 100% rename from sources/migrations/__init__.py rename to src/setup_utrecht/migrations/__init__.py diff --git a/setup_utrecht/models/__init__.py b/src/setup_utrecht/models/__init__.py similarity index 100% rename from setup_utrecht/models/__init__.py rename to src/setup_utrecht/models/__init__.py diff --git a/setup_utrecht/models/i48_trolls.py b/src/setup_utrecht/models/i48_trolls.py similarity index 100% rename from setup_utrecht/models/i48_trolls.py rename to src/setup_utrecht/models/i48_trolls.py diff --git a/setup_utrecht/models/sources/files.py b/src/setup_utrecht/models/sources/files.py similarity index 100% rename from setup_utrecht/models/sources/files.py rename to src/setup_utrecht/models/sources/files.py diff --git a/setup_utrecht/models/sources/reddit.py b/src/setup_utrecht/models/sources/reddit.py similarity index 100% rename from setup_utrecht/models/sources/reddit.py rename to src/setup_utrecht/models/sources/reddit.py diff --git a/setup_utrecht/models/womr.py b/src/setup_utrecht/models/womr.py similarity index 100% rename from setup_utrecht/models/womr.py rename to src/setup_utrecht/models/womr.py diff --git a/setup_utrecht/urls.py b/src/setup_utrecht/urls.py similarity index 100% rename from setup_utrecht/urls.py rename to src/setup_utrecht/urls.py diff --git a/setup_utrecht/views.py b/src/setup_utrecht/views.py similarity index 100% rename from setup_utrecht/views.py rename to src/setup_utrecht/views.py diff --git a/sources/models/google/__init__.py b/src/sources/__init__.py similarity index 100% rename from sources/models/google/__init__.py rename to src/sources/__init__.py diff --git a/sources/admin.py b/src/sources/admin.py similarity index 100% rename from sources/admin.py rename to src/sources/admin.py diff --git a/sources/management/commands/analyse_image_downloads.py b/src/sources/management/commands/analyse_image_downloads.py similarity index 100% rename from sources/management/commands/analyse_image_downloads.py rename to src/sources/management/commands/analyse_image_downloads.py diff --git a/sources/management/commands/migrate_image_downloads.py b/src/sources/management/commands/migrate_image_downloads.py similarity index 100% rename from sources/management/commands/migrate_image_downloads.py rename to src/sources/management/commands/migrate_image_downloads.py diff --git a/sources/migrations/0001_initial.py b/src/sources/migrations/0001_initial.py similarity index 100% rename from sources/migrations/0001_initial.py rename to src/sources/migrations/0001_initial.py diff --git a/sources/migrations/0002_wikipediasearch.py b/src/sources/migrations/0002_wikipediasearch.py similarity index 100% rename from sources/migrations/0002_wikipediasearch.py rename to src/sources/migrations/0002_wikipediasearch.py diff --git a/sources/migrations/0003_wikipediarecentchanges.py b/src/sources/migrations/0003_wikipediarecentchanges.py similarity index 100% rename from sources/migrations/0003_wikipediarecentchanges.py rename to src/sources/migrations/0003_wikipediarecentchanges.py diff --git a/sources/migrations/0004_wikipedialistpages.py b/src/sources/migrations/0004_wikipedialistpages.py similarity index 100% rename from sources/migrations/0004_wikipedialistpages.py rename to src/sources/migrations/0004_wikipedialistpages.py diff --git a/sources/migrations/0005_wikipediatranslate.py b/src/sources/migrations/0005_wikipediatranslate.py similarity index 100% rename from sources/migrations/0005_wikipediatranslate.py rename to src/sources/migrations/0005_wikipediatranslate.py diff --git a/sources/migrations/0006_googleimage.py b/src/sources/migrations/0006_googleimage.py similarity index 100% rename from sources/migrations/0006_googleimage.py rename to src/sources/migrations/0006_googleimage.py diff --git a/sources/migrations/0007_googletranslate.py b/src/sources/migrations/0007_googletranslate.py similarity index 100% rename from sources/migrations/0007_googletranslate.py rename to src/sources/migrations/0007_googletranslate.py diff --git a/sources/migrations/0008_imagedownload.py b/src/sources/migrations/0008_imagedownload.py similarity index 100% rename from sources/migrations/0008_imagedownload.py rename to src/sources/migrations/0008_imagedownload.py diff --git a/sources/migrations/0009_auto_20150828_1659.py b/src/sources/migrations/0009_auto_20150828_1659.py similarity index 100% rename from sources/migrations/0009_auto_20150828_1659.py rename to src/sources/migrations/0009_auto_20150828_1659.py diff --git a/sources/migrations/0010_auto_20160225_2307.py b/src/sources/migrations/0010_auto_20160225_2307.py similarity index 100% rename from sources/migrations/0010_auto_20160225_2307.py rename to src/sources/migrations/0010_auto_20160225_2307.py diff --git a/sources/migrations/0011_wikidataitems.py b/src/sources/migrations/0011_wikidataitems.py similarity index 100% rename from sources/migrations/0011_wikidataitems.py rename to src/sources/migrations/0011_wikidataitems.py diff --git a/sources/migrations/0012_auto_20160228_2019.py b/src/sources/migrations/0012_auto_20160228_2019.py similarity index 100% rename from sources/migrations/0012_auto_20160228_2019.py rename to src/sources/migrations/0012_auto_20160228_2019.py diff --git a/sources/migrations/0013_imagefeatures.py b/src/sources/migrations/0013_imagefeatures.py similarity index 100% rename from sources/migrations/0013_imagefeatures.py rename to src/sources/migrations/0013_imagefeatures.py diff --git a/sources/migrations/0014_wikipediacategories_wikipediacategorymembers.py b/src/sources/migrations/0014_wikipediacategories_wikipediacategorymembers.py similarity index 100% rename from sources/migrations/0014_wikipediacategories_wikipediacategorymembers.py rename to src/sources/migrations/0014_wikipediacategories_wikipediacategorymembers.py diff --git a/sources/migrations/0015_wikipediapageviewdetails.py b/src/sources/migrations/0015_wikipediapageviewdetails.py similarity index 100% rename from sources/migrations/0015_wikipediapageviewdetails.py rename to src/sources/migrations/0015_wikipediapageviewdetails.py diff --git a/sources/migrations/0016_wikipediatransclusions.py b/src/sources/migrations/0016_wikipediatransclusions.py similarity index 100% rename from sources/migrations/0016_wikipediatransclusions.py rename to src/sources/migrations/0016_wikipediatransclusions.py diff --git a/sources/migrations/0017_wikipediarevisions.py b/src/sources/migrations/0017_wikipediarevisions.py similarity index 100% rename from sources/migrations/0017_wikipediarevisions.py rename to src/sources/migrations/0017_wikipediarevisions.py diff --git a/sources/migrations/0018_wikipediaedit_wikipedialogin_wikipediatoken.py b/src/sources/migrations/0018_wikipediaedit_wikipedialogin_wikipediatoken.py similarity index 100% rename from sources/migrations/0018_wikipediaedit_wikipedialogin_wikipediatoken.py rename to src/sources/migrations/0018_wikipediaedit_wikipedialogin_wikipediatoken.py diff --git a/sources/migrations/0019_officialannouncementsdocumentnetherlands_officialannouncementsnetherlands.py b/src/sources/migrations/0019_officialannouncementsdocumentnetherlands_officialannouncementsnetherlands.py similarity index 100% rename from sources/migrations/0019_officialannouncementsdocumentnetherlands_officialannouncementsnetherlands.py rename to src/sources/migrations/0019_officialannouncementsdocumentnetherlands_officialannouncementsnetherlands.py diff --git a/sources/migrations/0020_auto_20170428_1123.py b/src/sources/migrations/0020_auto_20170428_1123.py similarity index 100% rename from sources/migrations/0020_auto_20170428_1123.py rename to src/sources/migrations/0020_auto_20170428_1123.py diff --git a/sources/migrations/0021_kledinglisting.py b/src/sources/migrations/0021_kledinglisting.py similarity index 100% rename from sources/migrations/0021_kledinglisting.py rename to src/sources/migrations/0021_kledinglisting.py diff --git a/sources/migrations/0022_googletext.py b/src/sources/migrations/0022_googletext.py similarity index 100% rename from sources/migrations/0022_googletext.py rename to src/sources/migrations/0022_googletext.py diff --git a/sources/migrations/0023_auto_20171003_1235.py b/src/sources/migrations/0023_auto_20171003_1235.py similarity index 100% rename from sources/migrations/0023_auto_20171003_1235.py rename to src/sources/migrations/0023_auto_20171003_1235.py diff --git a/sources/migrations/0024_auto_20171119_1315.py b/src/sources/migrations/0024_auto_20171119_1315.py similarity index 100% rename from sources/migrations/0024_auto_20171119_1315.py rename to src/sources/migrations/0024_auto_20171119_1315.py diff --git a/sources/migrations/0025_auto_20180818_1844.py b/src/sources/migrations/0025_auto_20180818_1844.py similarity index 100% rename from sources/migrations/0025_auto_20180818_1844.py rename to src/sources/migrations/0025_auto_20180818_1844.py diff --git a/sources/migrations/0026_auto_20180818_1854.py b/src/sources/migrations/0026_auto_20180818_1854.py similarity index 100% rename from sources/migrations/0026_auto_20180818_1854.py rename to src/sources/migrations/0026_auto_20180818_1854.py diff --git a/sources/migrations/0027_auto_20180912_0931.py b/src/sources/migrations/0027_auto_20180912_0931.py similarity index 100% rename from sources/migrations/0027_auto_20180912_0931.py rename to src/sources/migrations/0027_auto_20180912_0931.py diff --git a/sources/migrations/0028_auto_20181202_1424.py b/src/sources/migrations/0028_auto_20181202_1424.py similarity index 100% rename from sources/migrations/0028_auto_20181202_1424.py rename to src/sources/migrations/0028_auto_20181202_1424.py diff --git a/sources/models/websites/__init__.py b/src/sources/migrations/__init__.py similarity index 100% rename from sources/models/websites/__init__.py rename to src/sources/migrations/__init__.py diff --git a/sources/models/__init__.py b/src/sources/models/__init__.py similarity index 100% rename from sources/models/__init__.py rename to src/sources/models/__init__.py diff --git a/sources/models/downloads.py b/src/sources/models/downloads.py similarity index 100% rename from sources/models/downloads.py rename to src/sources/models/downloads.py diff --git a/sources/models/wikipedia/__init__.py b/src/sources/models/google/__init__.py similarity index 100% rename from sources/models/wikipedia/__init__.py rename to src/sources/models/google/__init__.py diff --git a/sources/models/google/images.py b/src/sources/models/google/images.py similarity index 100% rename from sources/models/google/images.py rename to src/sources/models/google/images.py diff --git a/sources/models/google/query.py b/src/sources/models/google/query.py similarity index 100% rename from sources/models/google/query.py rename to src/sources/models/google/query.py diff --git a/sources/models/google/text.py b/src/sources/models/google/text.py similarity index 100% rename from sources/models/google/text.py rename to src/sources/models/google/text.py diff --git a/sources/models/google/translations.py b/src/sources/models/google/translations.py similarity index 100% rename from sources/models/google/translations.py rename to src/sources/models/google/translations.py diff --git a/sources/models/micro.py b/src/sources/models/micro.py similarity index 100% rename from sources/models/micro.py rename to src/sources/models/micro.py diff --git a/sources/processors/wikipedia/__init__.py b/src/sources/models/websites/__init__.py similarity index 100% rename from sources/processors/wikipedia/__init__.py rename to src/sources/models/websites/__init__.py diff --git a/sources/models/websites/kleding.py b/src/sources/models/websites/kleding.py similarity index 100% rename from sources/models/websites/kleding.py rename to src/sources/models/websites/kleding.py diff --git a/visual_translations/management/__init__.py b/src/sources/models/wikipedia/__init__.py similarity index 100% rename from visual_translations/management/__init__.py rename to src/sources/models/wikipedia/__init__.py diff --git a/sources/models/wikipedia/base.py b/src/sources/models/wikipedia/base.py similarity index 100% rename from sources/models/wikipedia/base.py rename to src/sources/models/wikipedia/base.py diff --git a/sources/models/wikipedia/categories.py b/src/sources/models/wikipedia/categories.py similarity index 100% rename from sources/models/wikipedia/categories.py rename to src/sources/models/wikipedia/categories.py diff --git a/sources/models/wikipedia/data.py b/src/sources/models/wikipedia/data.py similarity index 100% rename from sources/models/wikipedia/data.py rename to src/sources/models/wikipedia/data.py diff --git a/sources/models/wikipedia/edit.py b/src/sources/models/wikipedia/edit.py similarity index 100% rename from sources/models/wikipedia/edit.py rename to src/sources/models/wikipedia/edit.py diff --git a/sources/models/wikipedia/login.py b/src/sources/models/wikipedia/login.py similarity index 100% rename from sources/models/wikipedia/login.py rename to src/sources/models/wikipedia/login.py diff --git a/sources/models/wikipedia/metrics.py b/src/sources/models/wikipedia/metrics.py similarity index 100% rename from sources/models/wikipedia/metrics.py rename to src/sources/models/wikipedia/metrics.py diff --git a/sources/models/wikipedia/mixins.py b/src/sources/models/wikipedia/mixins.py similarity index 100% rename from sources/models/wikipedia/mixins.py rename to src/sources/models/wikipedia/mixins.py diff --git a/sources/models/wikipedia/pages.py b/src/sources/models/wikipedia/pages.py similarity index 100% rename from sources/models/wikipedia/pages.py rename to src/sources/models/wikipedia/pages.py diff --git a/sources/models/wikipedia/query.py b/src/sources/models/wikipedia/query.py similarity index 100% rename from sources/models/wikipedia/query.py rename to src/sources/models/wikipedia/query.py diff --git a/sources/models/wikipedia/revisions.py b/src/sources/models/wikipedia/revisions.py similarity index 100% rename from sources/models/wikipedia/revisions.py rename to src/sources/models/wikipedia/revisions.py diff --git a/sources/models/wikipedia/search.py b/src/sources/models/wikipedia/search.py similarity index 100% rename from sources/models/wikipedia/search.py rename to src/sources/models/wikipedia/search.py diff --git a/sources/models/wikipedia/transclusion.py b/src/sources/models/wikipedia/transclusion.py similarity index 100% rename from sources/models/wikipedia/transclusion.py rename to src/sources/models/wikipedia/transclusion.py diff --git a/sources/models/wikipedia/translations.py b/src/sources/models/wikipedia/translations.py similarity index 100% rename from sources/models/wikipedia/translations.py rename to src/sources/models/wikipedia/translations.py diff --git a/sources/processors/__init__.py b/src/sources/processors/__init__.py similarity index 100% rename from sources/processors/__init__.py rename to src/sources/processors/__init__.py diff --git a/visual_translations/management/commands/__init__.py b/src/sources/processors/wikipedia/__init__.py similarity index 100% rename from visual_translations/management/commands/__init__.py rename to src/sources/processors/wikipedia/__init__.py diff --git a/sources/processors/wikipedia/compare.py b/src/sources/processors/wikipedia/compare.py similarity index 100% rename from sources/processors/wikipedia/compare.py rename to src/sources/processors/wikipedia/compare.py diff --git a/sources/processors/wikipedia/edit.py b/src/sources/processors/wikipedia/edit.py similarity index 100% rename from sources/processors/wikipedia/edit.py rename to src/sources/processors/wikipedia/edit.py diff --git a/sources/processors/wikipedia/rank.py b/src/sources/processors/wikipedia/rank.py similarity index 100% rename from sources/processors/wikipedia/rank.py rename to src/sources/processors/wikipedia/rank.py diff --git a/sources/tests.py b/src/sources/tests.py similarity index 100% rename from sources/tests.py rename to src/sources/tests.py diff --git a/src/uwsgi.ini b/src/uwsgi.ini new file mode 100644 index 00000000..843d454f --- /dev/null +++ b/src/uwsgi.ini @@ -0,0 +1,9 @@ +[uwsgi] +http = :8000 +chdir = /usr/src/app/ +module = datascope.wsgi +master = 1 +processes = 2 +threads = 1 +uid = app +gid = app diff --git a/visual_translations/README.md b/src/visual_translations/README.md similarity index 100% rename from visual_translations/README.md rename to src/visual_translations/README.md diff --git a/visual_translations/__init__.py b/src/visual_translations/__init__.py similarity index 100% rename from visual_translations/__init__.py rename to src/visual_translations/__init__.py diff --git a/visual_translations/admin.py b/src/visual_translations/admin.py similarity index 100% rename from visual_translations/admin.py rename to src/visual_translations/admin.py diff --git a/visual_translations/apps.py b/src/visual_translations/apps.py similarity index 100% rename from visual_translations/apps.py rename to src/visual_translations/apps.py diff --git a/visual_translations/migrations/__init__.py b/src/visual_translations/management/__init__.py similarity index 100% rename from visual_translations/migrations/__init__.py rename to src/visual_translations/management/__init__.py diff --git a/visual_translations/views/__init__.py b/src/visual_translations/management/commands/__init__.py similarity index 100% rename from visual_translations/views/__init__.py rename to src/visual_translations/management/commands/__init__.py diff --git a/visual_translations/management/commands/delete_image.py b/src/visual_translations/management/commands/delete_image.py similarity index 100% rename from visual_translations/management/commands/delete_image.py rename to src/visual_translations/management/commands/delete_image.py diff --git a/visual_translations/management/commands/delete_word.py b/src/visual_translations/management/commands/delete_word.py similarity index 100% rename from visual_translations/management/commands/delete_word.py rename to src/visual_translations/management/commands/delete_word.py diff --git a/visual_translations/management/commands/generate_eu_visual_translations.py b/src/visual_translations/management/commands/generate_eu_visual_translations.py similarity index 100% rename from visual_translations/management/commands/generate_eu_visual_translations.py rename to src/visual_translations/management/commands/generate_eu_visual_translations.py diff --git a/visual_translations/management/commands/generate_image_grids.py b/src/visual_translations/management/commands/generate_image_grids.py similarity index 100% rename from visual_translations/management/commands/generate_image_grids.py rename to src/visual_translations/management/commands/generate_image_grids.py diff --git a/visual_translations/migrations/0001_initial.py b/src/visual_translations/migrations/0001_initial.py similarity index 100% rename from visual_translations/migrations/0001_initial.py rename to src/visual_translations/migrations/0001_initial.py diff --git a/visual_translations/migrations/0002_auto_20160219_1442.py b/src/visual_translations/migrations/0002_auto_20160219_1442.py similarity index 100% rename from visual_translations/migrations/0002_auto_20160219_1442.py rename to src/visual_translations/migrations/0002_auto_20160219_1442.py diff --git a/visual_translations/migrations/0003_auto_20160225_2307.py b/src/visual_translations/migrations/0003_auto_20160225_2307.py similarity index 100% rename from visual_translations/migrations/0003_auto_20160225_2307.py rename to src/visual_translations/migrations/0003_auto_20160225_2307.py diff --git a/visual_translations/migrations/0004_auto_20160328_1112.py b/src/visual_translations/migrations/0004_auto_20160328_1112.py similarity index 100% rename from visual_translations/migrations/0004_auto_20160328_1112.py rename to src/visual_translations/migrations/0004_auto_20160328_1112.py diff --git a/visual_translations/migrations/0005_visualtranslationsbriccommunity.py b/src/visual_translations/migrations/0005_visualtranslationsbriccommunity.py similarity index 100% rename from visual_translations/migrations/0005_visualtranslationsbriccommunity.py rename to src/visual_translations/migrations/0005_visualtranslationsbriccommunity.py diff --git a/visual_translations/migrations/0006_auto_20160417_2047.py b/src/visual_translations/migrations/0006_auto_20160417_2047.py similarity index 100% rename from visual_translations/migrations/0006_auto_20160417_2047.py rename to src/visual_translations/migrations/0006_auto_20160417_2047.py diff --git a/visual_translations/migrations/0007_webimagedownload.py b/src/visual_translations/migrations/0007_webimagedownload.py similarity index 100% rename from visual_translations/migrations/0007_webimagedownload.py rename to src/visual_translations/migrations/0007_webimagedownload.py diff --git a/visual_translations/migrations/0008_auto_20190402_0928.py b/src/visual_translations/migrations/0008_auto_20190402_0928.py similarity index 100% rename from visual_translations/migrations/0008_auto_20190402_0928.py rename to src/visual_translations/migrations/0008_auto_20190402_0928.py diff --git a/wiki_feed/management/__init__.py b/src/visual_translations/migrations/__init__.py similarity index 100% rename from wiki_feed/management/__init__.py rename to src/visual_translations/migrations/__init__.py diff --git a/visual_translations/models/__init__.py b/src/visual_translations/models/__init__.py similarity index 100% rename from visual_translations/models/__init__.py rename to src/visual_translations/models/__init__.py diff --git a/visual_translations/models/bric.py b/src/visual_translations/models/bric.py similarity index 100% rename from visual_translations/models/bric.py rename to src/visual_translations/models/bric.py diff --git a/visual_translations/models/eu.py b/src/visual_translations/models/eu.py similarity index 100% rename from visual_translations/models/eu.py rename to src/visual_translations/models/eu.py diff --git a/visual_translations/models/sources/files.py b/src/visual_translations/models/sources/files.py similarity index 100% rename from visual_translations/models/sources/files.py rename to src/visual_translations/models/sources/files.py diff --git a/visual_translations/static/visual_translations/css/controller.css b/src/visual_translations/static/visual_translations/css/controller.css similarity index 100% rename from visual_translations/static/visual_translations/css/controller.css rename to src/visual_translations/static/visual_translations/css/controller.css diff --git a/visual_translations/static/visual_translations/css/images/ui-bg_diagonals-thick_18_b81900_40x40.png b/src/visual_translations/static/visual_translations/css/images/ui-bg_diagonals-thick_18_b81900_40x40.png similarity index 100% rename from visual_translations/static/visual_translations/css/images/ui-bg_diagonals-thick_18_b81900_40x40.png rename to src/visual_translations/static/visual_translations/css/images/ui-bg_diagonals-thick_18_b81900_40x40.png diff --git a/visual_translations/static/visual_translations/css/images/ui-bg_diagonals-thick_20_666666_40x40.png b/src/visual_translations/static/visual_translations/css/images/ui-bg_diagonals-thick_20_666666_40x40.png similarity index 100% rename from visual_translations/static/visual_translations/css/images/ui-bg_diagonals-thick_20_666666_40x40.png rename to src/visual_translations/static/visual_translations/css/images/ui-bg_diagonals-thick_20_666666_40x40.png diff --git a/visual_translations/static/visual_translations/css/images/ui-bg_flat_10_000000_40x100.png b/src/visual_translations/static/visual_translations/css/images/ui-bg_flat_10_000000_40x100.png similarity index 100% rename from visual_translations/static/visual_translations/css/images/ui-bg_flat_10_000000_40x100.png rename to src/visual_translations/static/visual_translations/css/images/ui-bg_flat_10_000000_40x100.png diff --git a/visual_translations/static/visual_translations/css/images/ui-bg_glass_100_f6f6f6_1x400.png b/src/visual_translations/static/visual_translations/css/images/ui-bg_glass_100_f6f6f6_1x400.png similarity index 100% rename from visual_translations/static/visual_translations/css/images/ui-bg_glass_100_f6f6f6_1x400.png rename to src/visual_translations/static/visual_translations/css/images/ui-bg_glass_100_f6f6f6_1x400.png diff --git a/visual_translations/static/visual_translations/css/images/ui-bg_glass_100_fdf5ce_1x400.png b/src/visual_translations/static/visual_translations/css/images/ui-bg_glass_100_fdf5ce_1x400.png similarity index 100% rename from visual_translations/static/visual_translations/css/images/ui-bg_glass_100_fdf5ce_1x400.png rename to src/visual_translations/static/visual_translations/css/images/ui-bg_glass_100_fdf5ce_1x400.png diff --git a/visual_translations/static/visual_translations/css/images/ui-bg_glass_65_ffffff_1x400.png b/src/visual_translations/static/visual_translations/css/images/ui-bg_glass_65_ffffff_1x400.png similarity index 100% rename from visual_translations/static/visual_translations/css/images/ui-bg_glass_65_ffffff_1x400.png rename to src/visual_translations/static/visual_translations/css/images/ui-bg_glass_65_ffffff_1x400.png diff --git a/visual_translations/static/visual_translations/css/images/ui-bg_gloss-wave_35_f6a828_500x100.png b/src/visual_translations/static/visual_translations/css/images/ui-bg_gloss-wave_35_f6a828_500x100.png similarity index 100% rename from visual_translations/static/visual_translations/css/images/ui-bg_gloss-wave_35_f6a828_500x100.png rename to src/visual_translations/static/visual_translations/css/images/ui-bg_gloss-wave_35_f6a828_500x100.png diff --git a/visual_translations/static/visual_translations/css/images/ui-bg_highlight-soft_100_eeeeee_1x100.png b/src/visual_translations/static/visual_translations/css/images/ui-bg_highlight-soft_100_eeeeee_1x100.png similarity index 100% rename from visual_translations/static/visual_translations/css/images/ui-bg_highlight-soft_100_eeeeee_1x100.png rename to src/visual_translations/static/visual_translations/css/images/ui-bg_highlight-soft_100_eeeeee_1x100.png diff --git a/visual_translations/static/visual_translations/css/images/ui-bg_highlight-soft_75_ffe45c_1x100.png b/src/visual_translations/static/visual_translations/css/images/ui-bg_highlight-soft_75_ffe45c_1x100.png similarity index 100% rename from visual_translations/static/visual_translations/css/images/ui-bg_highlight-soft_75_ffe45c_1x100.png rename to src/visual_translations/static/visual_translations/css/images/ui-bg_highlight-soft_75_ffe45c_1x100.png diff --git a/visual_translations/static/visual_translations/css/images/ui-icons_222222_256x240.png b/src/visual_translations/static/visual_translations/css/images/ui-icons_222222_256x240.png similarity index 100% rename from visual_translations/static/visual_translations/css/images/ui-icons_222222_256x240.png rename to src/visual_translations/static/visual_translations/css/images/ui-icons_222222_256x240.png diff --git a/visual_translations/static/visual_translations/css/images/ui-icons_228ef1_256x240.png b/src/visual_translations/static/visual_translations/css/images/ui-icons_228ef1_256x240.png similarity index 100% rename from visual_translations/static/visual_translations/css/images/ui-icons_228ef1_256x240.png rename to src/visual_translations/static/visual_translations/css/images/ui-icons_228ef1_256x240.png diff --git a/visual_translations/static/visual_translations/css/images/ui-icons_ef8c08_256x240.png b/src/visual_translations/static/visual_translations/css/images/ui-icons_ef8c08_256x240.png similarity index 100% rename from visual_translations/static/visual_translations/css/images/ui-icons_ef8c08_256x240.png rename to src/visual_translations/static/visual_translations/css/images/ui-icons_ef8c08_256x240.png diff --git a/visual_translations/static/visual_translations/css/images/ui-icons_ffd27a_256x240.png b/src/visual_translations/static/visual_translations/css/images/ui-icons_ffd27a_256x240.png similarity index 100% rename from visual_translations/static/visual_translations/css/images/ui-icons_ffd27a_256x240.png rename to src/visual_translations/static/visual_translations/css/images/ui-icons_ffd27a_256x240.png diff --git a/visual_translations/static/visual_translations/css/images/ui-icons_ffffff_256x240.png b/src/visual_translations/static/visual_translations/css/images/ui-icons_ffffff_256x240.png similarity index 100% rename from visual_translations/static/visual_translations/css/images/ui-icons_ffffff_256x240.png rename to src/visual_translations/static/visual_translations/css/images/ui-icons_ffffff_256x240.png diff --git a/visual_translations/static/visual_translations/css/jquery-ui.min.css b/src/visual_translations/static/visual_translations/css/jquery-ui.min.css similarity index 100% rename from visual_translations/static/visual_translations/css/jquery-ui.min.css rename to src/visual_translations/static/visual_translations/css/jquery-ui.min.css diff --git a/visual_translations/static/visual_translations/css/jquery-ui.theme.min.css b/src/visual_translations/static/visual_translations/css/jquery-ui.theme.min.css similarity index 100% rename from visual_translations/static/visual_translations/css/jquery-ui.theme.min.css rename to src/visual_translations/static/visual_translations/css/jquery-ui.theme.min.css diff --git a/visual_translations/static/visual_translations/css/knob.css b/src/visual_translations/static/visual_translations/css/knob.css similarity index 100% rename from visual_translations/static/visual_translations/css/knob.css rename to src/visual_translations/static/visual_translations/css/knob.css diff --git a/visual_translations/static/visual_translations/css/visual-translations.css b/src/visual_translations/static/visual_translations/css/visual-translations.css similarity index 100% rename from visual_translations/static/visual_translations/css/visual-translations.css rename to src/visual_translations/static/visual_translations/css/visual-translations.css diff --git a/visual_translations/static/visual_translations/geo/europe.topo.json b/src/visual_translations/static/visual_translations/geo/europe.topo.json similarity index 100% rename from visual_translations/static/visual_translations/geo/europe.topo.json rename to src/visual_translations/static/visual_translations/geo/europe.topo.json diff --git a/visual_translations/static/visual_translations/img/bg.jpg b/src/visual_translations/static/visual_translations/img/bg.jpg similarity index 100% rename from visual_translations/static/visual_translations/img/bg.jpg rename to src/visual_translations/static/visual_translations/img/bg.jpg diff --git a/visual_translations/static/visual_translations/img/knob.png b/src/visual_translations/static/visual_translations/img/knob.png similarity index 100% rename from visual_translations/static/visual_translations/img/knob.png rename to src/visual_translations/static/visual_translations/img/knob.png diff --git a/visual_translations/static/visual_translations/img/pointer.png b/src/visual_translations/static/visual_translations/img/pointer.png similarity index 100% rename from visual_translations/static/visual_translations/img/pointer.png rename to src/visual_translations/static/visual_translations/img/pointer.png diff --git a/visual_translations/static/visual_translations/img/triangle-grey.png b/src/visual_translations/static/visual_translations/img/triangle-grey.png similarity index 100% rename from visual_translations/static/visual_translations/img/triangle-grey.png rename to src/visual_translations/static/visual_translations/img/triangle-grey.png diff --git a/visual_translations/static/visual_translations/img/triangle.png b/src/visual_translations/static/visual_translations/img/triangle.png similarity index 100% rename from visual_translations/static/visual_translations/img/triangle.png rename to src/visual_translations/static/visual_translations/img/triangle.png diff --git a/visual_translations/static/visual_translations/js/controller.js b/src/visual_translations/static/visual_translations/js/controller.js similarity index 100% rename from visual_translations/static/visual_translations/js/controller.js rename to src/visual_translations/static/visual_translations/js/controller.js diff --git a/visual_translations/static/visual_translations/js/d3.v3.min.js b/src/visual_translations/static/visual_translations/js/d3.v3.min.js similarity index 100% rename from visual_translations/static/visual_translations/js/d3.v3.min.js rename to src/visual_translations/static/visual_translations/js/d3.v3.min.js diff --git a/visual_translations/static/visual_translations/js/help.js b/src/visual_translations/static/visual_translations/js/help.js similarity index 100% rename from visual_translations/static/visual_translations/js/help.js rename to src/visual_translations/static/visual_translations/js/help.js diff --git a/visual_translations/static/visual_translations/js/jquery-ui-touch-punch.min.js b/src/visual_translations/static/visual_translations/js/jquery-ui-touch-punch.min.js similarity index 100% rename from visual_translations/static/visual_translations/js/jquery-ui-touch-punch.min.js rename to src/visual_translations/static/visual_translations/js/jquery-ui-touch-punch.min.js diff --git a/visual_translations/static/visual_translations/js/jquery-ui.min.js b/src/visual_translations/static/visual_translations/js/jquery-ui.min.js similarity index 100% rename from visual_translations/static/visual_translations/js/jquery-ui.min.js rename to src/visual_translations/static/visual_translations/js/jquery-ui.min.js diff --git a/visual_translations/static/visual_translations/js/knob.jquery.js b/src/visual_translations/static/visual_translations/js/knob.jquery.js similarity index 100% rename from visual_translations/static/visual_translations/js/knob.jquery.js rename to src/visual_translations/static/visual_translations/js/knob.jquery.js diff --git a/visual_translations/static/visual_translations/js/map.js b/src/visual_translations/static/visual_translations/js/map.js similarity index 100% rename from visual_translations/static/visual_translations/js/map.js rename to src/visual_translations/static/visual_translations/js/map.js diff --git a/visual_translations/static/visual_translations/js/socket.js b/src/visual_translations/static/visual_translations/js/socket.js similarity index 100% rename from visual_translations/static/visual_translations/js/socket.js rename to src/visual_translations/static/visual_translations/js/socket.js diff --git a/visual_translations/static/visual_translations/js/topojson.min.js b/src/visual_translations/static/visual_translations/js/topojson.min.js similarity index 100% rename from visual_translations/static/visual_translations/js/topojson.min.js rename to src/visual_translations/static/visual_translations/js/topojson.min.js diff --git a/visual_translations/templates/visual_translations/accepted.html b/src/visual_translations/templates/visual_translations/accepted.html similarity index 100% rename from visual_translations/templates/visual_translations/accepted.html rename to src/visual_translations/templates/visual_translations/accepted.html diff --git a/visual_translations/templates/visual_translations/bad-request.html b/src/visual_translations/templates/visual_translations/bad-request.html similarity index 100% rename from visual_translations/templates/visual_translations/bad-request.html rename to src/visual_translations/templates/visual_translations/bad-request.html diff --git a/visual_translations/templates/visual_translations/controller.html b/src/visual_translations/templates/visual_translations/controller.html similarity index 100% rename from visual_translations/templates/visual_translations/controller.html rename to src/visual_translations/templates/visual_translations/controller.html diff --git a/visual_translations/templates/visual_translations/disambiguation.html b/src/visual_translations/templates/visual_translations/disambiguation.html similarity index 100% rename from visual_translations/templates/visual_translations/disambiguation.html rename to src/visual_translations/templates/visual_translations/disambiguation.html diff --git a/visual_translations/templates/visual_translations/index.html b/src/visual_translations/templates/visual_translations/index.html similarity index 100% rename from visual_translations/templates/visual_translations/index.html rename to src/visual_translations/templates/visual_translations/index.html diff --git a/visual_translations/templates/visual_translations/map.html b/src/visual_translations/templates/visual_translations/map.html similarity index 100% rename from visual_translations/templates/visual_translations/map.html rename to src/visual_translations/templates/visual_translations/map.html diff --git a/visual_translations/templates/visual_translations/no-content.html b/src/visual_translations/templates/visual_translations/no-content.html similarity index 100% rename from visual_translations/templates/visual_translations/no-content.html rename to src/visual_translations/templates/visual_translations/no-content.html diff --git a/visual_translations/templates/visual_translations/ok.html b/src/visual_translations/templates/visual_translations/ok.html similarity index 100% rename from visual_translations/templates/visual_translations/ok.html rename to src/visual_translations/templates/visual_translations/ok.html diff --git a/visual_translations/tests.py b/src/visual_translations/tests.py similarity index 100% rename from visual_translations/tests.py rename to src/visual_translations/tests.py diff --git a/visual_translations/urls.py b/src/visual_translations/urls.py similarity index 100% rename from visual_translations/urls.py rename to src/visual_translations/urls.py diff --git a/wiki_feed/management/commands/__init__.py b/src/visual_translations/views/__init__.py similarity index 100% rename from wiki_feed/management/commands/__init__.py rename to src/visual_translations/views/__init__.py diff --git a/visual_translations/views/community.py b/src/visual_translations/views/community.py similarity index 100% rename from visual_translations/views/community.py rename to src/visual_translations/views/community.py diff --git a/visual_translations/views/eu.py b/src/visual_translations/views/eu.py similarity index 100% rename from visual_translations/views/eu.py rename to src/visual_translations/views/eu.py diff --git a/wiki_feed/README.md b/src/wiki_feed/README.md similarity index 100% rename from wiki_feed/README.md rename to src/wiki_feed/README.md diff --git a/wiki_feed/__init__.py b/src/wiki_feed/__init__.py similarity index 100% rename from wiki_feed/__init__.py rename to src/wiki_feed/__init__.py diff --git a/wiki_feed/admin.py b/src/wiki_feed/admin.py similarity index 100% rename from wiki_feed/admin.py rename to src/wiki_feed/admin.py diff --git a/wiki_feed/apps.py b/src/wiki_feed/apps.py similarity index 100% rename from wiki_feed/apps.py rename to src/wiki_feed/apps.py diff --git a/wiki_feed/fixtures/breaking-news.json b/src/wiki_feed/fixtures/breaking-news.json similarity index 100% rename from wiki_feed/fixtures/breaking-news.json rename to src/wiki_feed/fixtures/breaking-news.json diff --git a/wiki_feed/fixtures/features.json b/src/wiki_feed/fixtures/features.json similarity index 100% rename from wiki_feed/fixtures/features.json rename to src/wiki_feed/fixtures/features.json diff --git a/wiki_feed/migrations/__init__.py b/src/wiki_feed/management/__init__.py similarity index 100% rename from wiki_feed/migrations/__init__.py rename to src/wiki_feed/management/__init__.py diff --git a/wiki_scope/migrations/__init__.py b/src/wiki_feed/management/commands/__init__.py similarity index 100% rename from wiki_scope/migrations/__init__.py rename to src/wiki_feed/management/commands/__init__.py diff --git a/wiki_feed/management/commands/grow_wiki_feed.py b/src/wiki_feed/management/commands/grow_wiki_feed.py similarity index 100% rename from wiki_feed/management/commands/grow_wiki_feed.py rename to src/wiki_feed/management/commands/grow_wiki_feed.py diff --git a/wiki_feed/management/commands/publish_wiki_feed.py b/src/wiki_feed/management/commands/publish_wiki_feed.py similarity index 100% rename from wiki_feed/management/commands/publish_wiki_feed.py rename to src/wiki_feed/management/commands/publish_wiki_feed.py diff --git a/wiki_feed/migrations/0001_initial.py b/src/wiki_feed/migrations/0001_initial.py similarity index 100% rename from wiki_feed/migrations/0001_initial.py rename to src/wiki_feed/migrations/0001_initial.py diff --git a/wiki_feed/migrations/0002_auto_20170201_1741.py b/src/wiki_feed/migrations/0002_auto_20170201_1741.py similarity index 100% rename from wiki_feed/migrations/0002_auto_20170201_1741.py rename to src/wiki_feed/migrations/0002_auto_20170201_1741.py diff --git a/wiki_feed/migrations/0003_wikifeedpublishcommunity.py b/src/wiki_feed/migrations/0003_wikifeedpublishcommunity.py similarity index 100% rename from wiki_feed/migrations/0003_wikifeedpublishcommunity.py rename to src/wiki_feed/migrations/0003_wikifeedpublishcommunity.py diff --git a/wiki_feed/migrations/0004_auto_20190402_0928.py b/src/wiki_feed/migrations/0004_auto_20190402_0928.py similarity index 100% rename from wiki_feed/migrations/0004_auto_20190402_0928.py rename to src/wiki_feed/migrations/0004_auto_20190402_0928.py diff --git a/wiki_scope/views.py b/src/wiki_feed/migrations/__init__.py similarity index 100% rename from wiki_scope/views.py rename to src/wiki_feed/migrations/__init__.py diff --git a/wiki_feed/models.py b/src/wiki_feed/models.py similarity index 100% rename from wiki_feed/models.py rename to src/wiki_feed/models.py diff --git a/wiki_feed/tasks.py b/src/wiki_feed/tasks.py similarity index 100% rename from wiki_feed/tasks.py rename to src/wiki_feed/tasks.py diff --git a/wiki_feed/templates/wiki_feed/accepted.html b/src/wiki_feed/templates/wiki_feed/accepted.html similarity index 100% rename from wiki_feed/templates/wiki_feed/accepted.html rename to src/wiki_feed/templates/wiki_feed/accepted.html diff --git a/wiki_feed/templates/wiki_feed/bad-request.html b/src/wiki_feed/templates/wiki_feed/bad-request.html similarity index 100% rename from wiki_feed/templates/wiki_feed/bad-request.html rename to src/wiki_feed/templates/wiki_feed/bad-request.html diff --git a/wiki_feed/templates/wiki_feed/header.wml b/src/wiki_feed/templates/wiki_feed/header.wml similarity index 100% rename from wiki_feed/templates/wiki_feed/header.wml rename to src/wiki_feed/templates/wiki_feed/header.wml diff --git a/wiki_feed/templates/wiki_feed/index.html b/src/wiki_feed/templates/wiki_feed/index.html similarity index 100% rename from wiki_feed/templates/wiki_feed/index.html rename to src/wiki_feed/templates/wiki_feed/index.html diff --git a/wiki_feed/templates/wiki_feed/no-content.html b/src/wiki_feed/templates/wiki_feed/no-content.html similarity index 100% rename from wiki_feed/templates/wiki_feed/no-content.html rename to src/wiki_feed/templates/wiki_feed/no-content.html diff --git a/wiki_feed/templates/wiki_feed/ok.html b/src/wiki_feed/templates/wiki_feed/ok.html similarity index 100% rename from wiki_feed/templates/wiki_feed/ok.html rename to src/wiki_feed/templates/wiki_feed/ok.html diff --git a/wiki_feed/templates/wiki_feed/section.wml b/src/wiki_feed/templates/wiki_feed/section.wml similarity index 100% rename from wiki_feed/templates/wiki_feed/section.wml rename to src/wiki_feed/templates/wiki_feed/section.wml diff --git a/wiki_feed/templates/wiki_feed/wait.html b/src/wiki_feed/templates/wiki_feed/wait.html similarity index 100% rename from wiki_feed/templates/wiki_feed/wait.html rename to src/wiki_feed/templates/wiki_feed/wait.html diff --git a/wiki_feed/tests/__init__.py b/src/wiki_feed/tests/__init__.py similarity index 100% rename from wiki_feed/tests/__init__.py rename to src/wiki_feed/tests/__init__.py diff --git a/wiki_feed/tests/community.py b/src/wiki_feed/tests/community.py similarity index 100% rename from wiki_feed/tests/community.py rename to src/wiki_feed/tests/community.py diff --git a/wiki_feed/tests/features.py b/src/wiki_feed/tests/features.py similarity index 100% rename from wiki_feed/tests/features.py rename to src/wiki_feed/tests/features.py diff --git a/wiki_feed/urls.py b/src/wiki_feed/urls.py similarity index 100% rename from wiki_feed/urls.py rename to src/wiki_feed/urls.py diff --git a/wiki_feed/views.py b/src/wiki_feed/views.py similarity index 100% rename from wiki_feed/views.py rename to src/wiki_feed/views.py diff --git a/wiki_scope/__init__.py b/src/wiki_scope/__init__.py similarity index 100% rename from wiki_scope/__init__.py rename to src/wiki_scope/__init__.py diff --git a/wiki_scope/admin.py b/src/wiki_scope/admin.py similarity index 100% rename from wiki_scope/admin.py rename to src/wiki_scope/admin.py diff --git a/wiki_scope/apps.py b/src/wiki_scope/apps.py similarity index 100% rename from wiki_scope/apps.py rename to src/wiki_scope/apps.py diff --git a/wiki_scope/migrations/0001_initial.py b/src/wiki_scope/migrations/0001_initial.py similarity index 100% rename from wiki_scope/migrations/0001_initial.py rename to src/wiki_scope/migrations/0001_initial.py diff --git a/wiki_scope/migrations/0002_auto_20190402_0928.py b/src/wiki_scope/migrations/0002_auto_20190402_0928.py similarity index 100% rename from wiki_scope/migrations/0002_auto_20190402_0928.py rename to src/wiki_scope/migrations/0002_auto_20190402_0928.py diff --git a/src/wiki_scope/migrations/__init__.py b/src/wiki_scope/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/wiki_scope/models/__init__.py b/src/wiki_scope/models/__init__.py similarity index 100% rename from wiki_scope/models/__init__.py rename to src/wiki_scope/models/__init__.py diff --git a/wiki_scope/models/category_similarity.py b/src/wiki_scope/models/category_similarity.py similarity index 100% rename from wiki_scope/models/category_similarity.py rename to src/wiki_scope/models/category_similarity.py diff --git a/wiki_scope/models/location.py b/src/wiki_scope/models/location.py similarity index 100% rename from wiki_scope/models/location.py rename to src/wiki_scope/models/location.py diff --git a/wiki_scope/tests.py b/src/wiki_scope/tests.py similarity index 100% rename from wiki_scope/tests.py rename to src/wiki_scope/tests.py diff --git a/wiki_scope/urls.py b/src/wiki_scope/urls.py similarity index 100% rename from wiki_scope/urls.py rename to src/wiki_scope/urls.py diff --git a/src/wiki_scope/views.py b/src/wiki_scope/views.py new file mode 100644 index 00000000..e69de29b diff --git a/tasks.py b/tasks.py new file mode 100644 index 00000000..100e23fa --- /dev/null +++ b/tasks.py @@ -0,0 +1,9 @@ +from invoke import Collection +from build import container +from deploy.tasks import publish_scripts, db_dump, db_load + + +namespace = Collection( + Collection("build", container), + Collection("deploy", publish_scripts, db_dump, db_load) +)