Merge branch 'develop'

README.md

@@ -92,6 +92,12 @@ GRANT USAGE ON SCHEMA blog TO monkey;
 * apply sql functions from `env/functions` directory
 * apply sql stored procedures from `env/stored_procs` directory
 
+### Redis ###
+
+* install redis with `apt-get install redis-server`
+* make sure that redis only listens on localhost
+* change redis.conf and set `maxmemory` (e.g: 500mb) and set `maxmemory-policy` to `allkeys-lru` 
+
 ### Install Bimg ###
 **Windows:**
 * install MSYS2
@@ -116,7 +122,7 @@ GRANT USAGE ON SCHEMA blog TO monkey;
 * modify `conf/nginx/nginx.conf` and replace `imagemonkey.io` and `api.imagemonkey.io` with your own domain names, copy it to `/etc/nginx/nginx.conf` and reload nginx with `service nginx reload`
 
 ### Build Application ###
-**Minimal** required Go version: v1.10.3
+**Minimal** required Go version: v1.11.10
 
 * install git with `apt-get install git`
 * install golang with `apt-get install golang`
@@ -138,9 +144,16 @@ mkdir -p /home/imagemonkey/unverified_donations
 ### Watchdog ###
 * install supervisor with `apt-get install supervisor`
 * add `imagemonkey` user to supervisor group with `adduser imagemonkey supervisor`
-* create logging directories with `mkdir -p /var/log/imagemonkey-api` and `mkdir -p /var/log/imagemonkey-web`
+* create logging directories with `mkdir -p /var/log/imagemonkey-api`, `mkdir -p /var/log/imagemonkey-web` `mkdir -p /var/log/imagemonkey-statworker`, `mkdir -p /var/log/imagemonkey-bot`, `mkdir -p /var/log/imagemonkey-blog-subscription-worker`, `mkdir -p /var/log/imagemonkey-data-processor`, `mkdir -p /var/log/imagemonkey-labelsdownloader`, `mkdir -p /var/log/imagemonkey-trending-labels-worker`
 * copy `conf/supervisor/imagemonkey-api.conf` to `/etc/supervisor/conf.d/imagemonkey-api.conf`
 * copy `conf/supervisor/imagemonkey-web.conf` to `/etc/supervisor/conf.d/imagemonkey-web.conf`
+* copy `conf/supervisor/imagemonkey-statworker.conf` to `/etc/supervisor/conf.d/imagemonkey-statworker.conf`
+* copy `conf/supervisor/imagemonkey-blog-subscription-worker.conf` to `/etc/supervisor/conf.d/imagemonkey-blog-subscription-worker.conf`
+* copy `conf/supervisor/imagemonkey-bot.conf` to `/etc/supervisor/conf.d/imagemonkey-bot.conf`
+* copy `conf/supervisor/imagemonkey-labels-downloader.conf` to `/etc/supervisor/conf.d/imagemonkey-labels-downloader.conf`
+* copy `conf/supervisor/imagemonkey-trending-labels-worker.conf` to `/etc/supervisor/conf.d/imagemonkey-trending-labels-worker.conf`
+* add `EnvironmentFile=/etc/environment` to the service section of the systemctl supervisor config file (see https://stackoverflow.com/questions/47083582/supervisor-not-using-etc-environment)
+* run `systemctl daemon-reload` and `systemctl restart supervisor`
 * run `supervisorctl reread && supervisorctl update && supervisorctl restart all`
 
 

env/docker/Dockerfile.blogsubscriptionworker

@@ -1,4 +1,4 @@
-FROM golang:1.12.5-stretch
+FROM golang:1.12.14-buster
 
 ENV REDIS_ADDRESS=:6379
 ENV SENTRY_DSN=

env/docker/Dockerfile.bot

@@ -1,4 +1,4 @@
-FROM golang:1.12.5-stretch
+FROM golang:1.12.14-buster
 
 RUN mkdir -p /home/go/bin
 ENV GOPATH=/home/go

env/docker/Dockerfile.buildbase

@@ -1,4 +1,4 @@
-FROM golang:1.12.5-stretch
+FROM golang:1.12.14-buster
 
 ENV GOPATH=/home/go
 ENV GOBIN=/home/go/bin

env/docker/Dockerfile.dataprocessor

@@ -1,4 +1,4 @@
-FROM golang:1.12.5-stretch
+FROM golang:1.12.14-buster
 
 ENV USE_SENTRY=true
 ARG SINGLESHOT=false

env/docker/Dockerfile.postgres

@@ -1,7 +1,7 @@
 FROM postgres:9.6.13
 
 ARG TEMPORAL_TABLE_VERSION=1.2.0
-ARG POSTGIS_VERSION=2.3
+ARG POSTGIS_VERSION=2.5
 ENV MONKEY_DB_PASSWORD=
 
 RUN apt-get update && apt-get install -y --no-install-recommends \

env/docker/Dockerfile.statworker

@@ -1,4 +1,4 @@
-FROM golang:1.12.5-stretch
+FROM golang:1.12.14-buster
 
 ENV SENTRY_DSN=
 ENV IMAGEMONKEY_DB_CONNECTION_STRING=

env/docker/Dockerfile.testing

@@ -1,4 +1,7 @@
-FROM golang:1.12.5-stretch
+FROM golang:1.12.14-buster
+
+# when changing the base image, change also the debian version here
+ARG DEBIAN_VERSION=buster
 
 RUN mkdir -p /home/go/bin
 #ENV GOPATH=/home/go
@@ -26,7 +29,9 @@ COPY src/image /tmp/src/image
 COPY src/parser /tmp/src/parser
 COPY env/postgres /tmp/env/postgres
 
-RUN apt-get update \
+RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ ${DEBIAN_VERSION}-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
+    && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
+	&& apt-get update \
 	&& apt-get install -y --no-install-recommends python3 postgresql-client-9.6 python3-pip wget unzip \
 	&& pip3 install selenium requests \
 	&& wget https://chromedriver.storage.googleapis.com/2.44/chromedriver_linux64.zip --directory-prefix=/tmp/ \

env/docker/Dockerfile.trendinglabelsworker

@@ -1,4 +1,4 @@
-FROM golang:1.12.5-stretch
+FROM golang:1.12.14-buster
 
 ENV USE_SENTRY=true
 ENV SINGLESHOT=false