Skip to content

Commit c585008

Browse files
rgaudinrgaudin
authored
Merge pull request #250 from kiwix/update_mirrorbrain
Update mirrorbrain
2 parents 8e1ea29 + d1da8d7 commit c585008

14 files changed

+951
-352
lines changed

‎.github/workflows/mirrorbrain_ci.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ jobs:
1818
uses: openzim/docker-publish-action@v10
1919
with:
2020
image-name: kiwix/mirrorbrain
21-
on-master: latest
21+
on-master: bookworm
2222
restrict-to: kiwix/container-images
2323
context: mirrorbrain
2424
registries: ghcr.io

‎mirrorbrain/Dockerfile

+53-60
Original file line numberDiff line numberDiff line change
@@ -1,76 +1,69 @@
1-
FROM httpd:2.4.43
1+
FROM httpd:2.4.62
22
LABEL org.opencontainers.image.source https://github.com/kiwix/container-images
3-
#
4-
# Author : Florent Kaisser <florent.pro@kaisser.name>
5-
#
6-
# Based on instructions from https://mirrorbrain.org/docs/installation/source/
7-
#
83
LABEL maintainer="kiwix"
94

105
#Set software versions
11-
ENV MB_VERSION 2.18.1
12-
ENV GEOIP_VERSION 1.6.12
13-
ENV MOD_GEOPIP_VERSION 1.2.10
6+
ENV MB_VERSION 3.0.0
7+
ENV MB_USER_OR_ORG kiwix
8+
9+
ENV LIBMAXMINDDB_VERSION 1.11.0
10+
ENV MOD_MAXMINDDB_VERSION 1.2.0
11+
ENV GEOIPUPDATE_VERSION 7.0.1
12+
ENV GEOIPUPDATE_ARCH amd64
1413

1514
#Install needed packages
16-
RUN mkdir -p /usr/share/man/man1/ /usr/share/man/man7/ && apt-get update && apt-get install -y --no-install-recommends wget cron automake libtool unzip libaprutil1-dbd-pgsql postgresql-client build-essential libz-dev python python-dev python-pip python-setuptools python-sqlobject python-formencode python-psycopg2 libconfig-inifiles-perl libwww-perl libdbd-pg-perl libtimedate-perl libdigest-md4-perl
17-
RUN pip install cmdln
15+
RUN mkdir -p /usr/share/man/man1/ /usr/share/man/man7/ && apt-get update && apt-get install -y --no-install-recommends wget cron automake libtool unzip libaprutil1-dbd-pgsql postgresql-client build-essential libz-dev libconfig-inifiles-perl libwww-perl libdbd-pg-perl libtimedate-perl libdigest-md4-perl libaprutil1-dev python3-setuptools python3-pip python3-sqlobject python3-psycopg2 python3-pip python3-dev python3-geoip2 rsync
16+
#Install cmdln manually since it is not packaged
17+
RUN pip install --no-cache-dir --break-system-packages cmdln
1818

1919
#Copy owned base config file for apache
2020
COPY config/apache/httpd.conf conf/httpd.conf
2121

2222
#Install Geolocalisation
23-
RUN { \
24-
cd /tmp ; \
25-
wget -q -O GeoIP-$GEOIP_VERSION.tar.gz https://github.com/maxmind/geoip-api-c/releases/download/v$GEOIP_VERSION/GeoIP-$GEOIP_VERSION.tar.gz && \
26-
tar xzf GeoIP-$GEOIP_VERSION.tar.gz -C /usr/local/src && \
27-
cd /usr/local/src/GeoIP-$GEOIP_VERSION/ && \
28-
aclocal && autoconf && automake --add-missing && ./configure --prefix=/usr/local/geoip && make && make install ; \
29-
cd /tmp ; \
30-
mkdir -p /usr/local/geoip/share/GeoIP/ ; \
31-
# wget -q https://geolite.maxmind.com/download/geoip/database/GeoLite2-City-CSV.zip && \
32-
# unzip GeoLite2-City-CSV.zip && cp GeoLite2-City-*/*.csv /usr/local/geoip/share/GeoIP/ && rm -rf GeoLite2-City-* ; \
33-
# wget -q https://geolite.maxmind.com/download/geoip/database/GeoLite2-Country-CSV.zip && \
34-
# unzip GeoLite2-Country-CSV.zip && cp GeoLite2-Country-*/*.csv /usr/local/geoip/share/GeoIP/ && rm -rf GeoLite2-Country-* ; \
35-
}
36-
37-
#Install Geolocalisation for Apache
38-
RUN { \
39-
cd /tmp ; \
40-
wget -q -O mod_geoip2.tar.gz https://github.com/maxmind/geoip-api-mod_geoip2/archive/$MOD_GEOPIP_VERSION.tar.gz && \
41-
tar xzf mod_geoip2.tar.gz -C /usr/local/src && \
42-
cd /usr/local/src/geoip-api-mod_geoip2-$MOD_GEOPIP_VERSION/ && \
43-
sed s/remote_ip/client_ip/g -i mod_geoip.c && \
44-
apxs -i -a -L/usr/local/geoip/lib -I/usr/local/geoip/include -lGeoIP -c mod_geoip.c ; \
45-
}
23+
RUN \
24+
cd /tmp && \
25+
wget -q -O libmaxminddb-${LIBMAXMINDDB_VERSION}.tar.gz https://github.com/maxmind/libmaxminddb/releases/download/${LIBMAXMINDDB_VERSION}/libmaxminddb-${LIBMAXMINDDB_VERSION}.tar.gz && \
26+
tar xzf libmaxminddb-${LIBMAXMINDDB_VERSION}.tar.gz -C /usr/local/src && \
27+
rm libmaxminddb-${LIBMAXMINDDB_VERSION}.tar.gz && \
28+
cd /usr/local/src/libmaxminddb-${LIBMAXMINDDB_VERSION} && \
29+
./configure && \
30+
make && \
31+
make check && \
32+
make install && \
33+
ldconfig && \
34+
cd /tmp && \
35+
wget -q -O geoipupdate_${GEOIPUPDATE_VERSION}_linux_${GEOIPUPDATE_ARCH}.deb https://github.com/maxmind/geoipupdate/releases/download/v${GEOIPUPDATE_VERSION}/geoipupdate_${GEOIPUPDATE_VERSION}_linux_${GEOIPUPDATE_ARCH}.deb && \
36+
apt-get install -y ./geoipupdate_${GEOIPUPDATE_VERSION}_linux_${GEOIPUPDATE_ARCH}.deb && \
37+
rm /tmp/geoipupdate_${GEOIPUPDATE_VERSION}_linux_${GEOIPUPDATE_ARCH}.deb && \
38+
cd /tmp && \
39+
wget -q -O mod_maxminddb-${MOD_MAXMINDDB_VERSION}.tar.gz https://github.com/maxmind/mod_maxminddb/releases/download/${MOD_MAXMINDDB_VERSION}/mod_maxminddb-${MOD_MAXMINDDB_VERSION}.tar.gz && \
40+
tar xzf mod_maxminddb-${MOD_MAXMINDDB_VERSION}.tar.gz -C /usr/local/src && \
41+
rm mod_maxminddb-${MOD_MAXMINDDB_VERSION}.tar.gz && \
42+
cd /usr/local/src/mod_maxminddb-${MOD_MAXMINDDB_VERSION} && \
43+
./configure && \
44+
make install
4645

4746
#Install MirrorBrain from sources
48-
COPY patch /tmp
49-
RUN { \
50-
wget --no-check-certificate -qO - https://github.com/poeml/mirrorbrain/archive/refs/tags/$MB_VERSION.tar.gz | tar -xz ; \
51-
cd mirrorbrain-$MB_VERSION/mod_mirrorbrain ; \
52-
wget -q http://apache.webthing.com/svn/apache/forms/mod_form.h ; \
53-
wget -q http://apache.webthing.com/svn/apache/forms/mod_form.c ; \
54-
mv /tmp/mod_form.c.patch ./ ; \
55-
apxs -cia -lm mod_form.c ; \
56-
apxs -e -n dbd -a modules/mod_dbd.so ; \
57-
apxs -e -n rewrite -a modules/mod_rewrite.so ; \
58-
apxs -cia -lm mod_mirrorbrain.c ; \
59-
cd ../mod_autoindex_mb ; \
60-
apxs -cia mod_autoindex_mb.c ; \
61-
cd ../tools ; \
62-
gcc -Wall -o geoiplookup_continent geoiplookup_continent.c -L/usr/local/geoip/lib -I/usr/local/geoip/include -lGeoIP ; \
63-
gcc -Wall -o geoiplookup_city geoiplookup_city.c -L/usr/local/geoip/lib -I/usr/local/geoip/include -lGeoIP ; \
64-
install -m 755 geoiplookup_continent /usr/bin/geoiplookup_continent ; \
65-
install -m 755 geoiplookup_city /usr/bin/geoiplookup_city ; \
66-
install -m 755 geoip-lite-update /usr/bin/geoip-lite-update ; \
67-
install -m 755 tnull-rsync /usr/bin/null-rsync ; \
68-
install -m 755 scanner.pl /usr/bin/scanner ; \
69-
cd ../mirrorprobe/ && install -m 755 mirrorprobe.py /usr/bin/mirrorprobe ; \
70-
cd ../mb && python setup.py install ; \
71-
patch /usr/bin/scanner /tmp/scanner.patch ; \
72-
rm /tmp/scanner.patch ; \
73-
}
47+
RUN \
48+
wget --no-check-certificate -qO - https://github.com/$MB_USER_OR_ORG/mirrorbrain/archive/refs/tags/$MB_VERSION.tar.gz | tar -xz && \
49+
cd mirrorbrain-$MB_VERSION/mod_mirrorbrain && \
50+
wget -q http://apache.webthing.com/svn/apache/forms/mod_form.h && \
51+
wget -q http://apache.webthing.com/svn/apache/forms/mod_form.c && \
52+
apxs -cia -lm mod_form.c && \
53+
apxs -e -n dbd -a modules/mod_dbd.so && \
54+
apxs -e -n rewrite -a modules/mod_rewrite.so && \
55+
apxs -cia -lm mod_mirrorbrain.c && \
56+
cd ../mod_autoindex_mb && \
57+
apxs -cia mod_autoindex_mb.c && \
58+
cd ../tools && \
59+
install -m 755 null-rsync /usr/bin/null-rsync && \
60+
install -m 755 scanner.pl /usr/bin/scanner && \
61+
cd ../mirrorprobe/ && install -m 755 mirrorprobe.py /usr/bin/mirrorprobe && \
62+
cd ../mb && pip install --no-cache-dir --break-system-packages . && \
63+
cd ../assets && \
64+
mkdir -p /var/www/static/flags && \
65+
cp famfamfam_flag_icons/png/*.png /var/www/static/flags && \
66+
cp mirrorbrain.css /var/www/static
7467

7568
#Copy files configuration
7669
RUN groupadd -r mirrorbrain && useradd -r -g mirrorbrain -s /bin/bash -c "MirrorBrain user" -d /home/mirrorbrain mirrorbrain

‎mirrorbrain/bin/start.sh

+10
Original file line numberDiff line numberDiff line change
@@ -49,3 +49,13 @@ else
4949
cron -f
5050
fi
5151
fi
52+
53+
if [ ! -z $GEOIPUPDATE ]
54+
then
55+
geoipupdate -v
56+
fi
57+
58+
if [ ! -z $HTTPD_ONLY ]
59+
then
60+
httpd-foreground
61+
fi

‎mirrorbrain/bin/update_mirrorbrain_db.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -61,4 +61,4 @@ scanMirror mirror-sites-ca.mblibrary.info ALLDIRS
6161
scanMirror mirror-sites-in.mblibrary.info ALLDIRS
6262

6363
# Generate HTML mirrors list
64-
mb mirrorlist -f xhtml | grep -v @ > /var/www/download.kiwix.org/mirrors.html
64+
mb mirrorlist -f xhtml --html-header /etc/mirrorlist_header.txt | grep -v @ > /var/www/download.kiwix.org/mirrors.html

‎mirrorbrain/dev/.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
download.kiwix.org
2+
GeoIP.conf

‎mirrorbrain/dev/README.md

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
# Development environment
2+
3+
This folder is not meant to reach production. It is a small stack based on docker compose to help developers
4+
test changes locally before pushing them forward.
5+
6+
The stack deploys:
7+
- our custom Mirrorbrain Docker image (web proxy)
8+
- a PostgreSQL database
9+
10+
The stack contains configuration files which have been extracted from production and adapted to work locally.
11+
12+
This is hence not a 100% realistic setup, for instance redirect maps in Apache have been commented out for now.
13+
14+
## How to start the stack
15+
16+
First thing you will need is a MaxMind configuration file for GeoIP databases v2. This configuration file
17+
should be retrieved from your MaxMind account, must be named `GeoIP.conf` and placed in `dev` folder.
18+
19+
Once this configuration file is in place, you can start the docker compose:
20+
```bash
21+
cd mirrorbrain/dev
22+
docker compose -p mirrorbrain up -d
23+
```
24+
25+
If it is the first time you start the stack, you must initialize DB schema and data. Password of `mirrorbrain` DB
26+
user is `mirrorbrain`.
27+
28+
```
29+
docker exec -it mb_web ./init_mirrorbrain_db_dev.sh
30+
```
31+
32+
You must also update GeoIP database once in a while.
33+
34+
```
35+
docker exec -it mb_web geoipupdate -v
36+
```
37+
38+
## How to test mirror scanning and stuff like that
39+
40+
Mirrorbrain provides a helpfull utility named null-rsync which allows to mirror files locally in the Docker container
41+
with sparse files (no content, no disk usage, only filename and attributes).
42+
43+
```bash
44+
docker exec mb_web null-rsync master.download.kiwix.org::download.kiwix.org/ /var/www/download.kiwix.org/
45+
```
46+
47+
Once this is done, you can run regular `mb` operations. For instance
48+
49+
```bash
50+
docker exec mb_web mb scan -d nightly dotsrc.org
51+
```
52+
53+
You can then check mirror is operating properly (note `X-Forwarded-For` header which is mandatory in our setup to pass end-user IP):
54+
```
55+
curl -H "X-Forwarded-For: 45.82.174.12" "http://localhost:8100/nightly/2023-10-26/kiwix-js-electron_i386_2023-10-26.deb?mirrorlist"
56+
```

‎mirrorbrain/dev/docker-compose.yaml

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
services:
2+
postgresdb:
3+
image: postgres:11
4+
container_name: mb_postgresdb
5+
ports:
6+
- 127.0.0.1:5433:5432
7+
volumes:
8+
- pg_data_mirrorbrain:/var/lib/postgresql/data
9+
environment:
10+
- POSTGRES_DB=mirrorbrain
11+
- POSTGRES_USER=mirrorbrain
12+
- POSTGRES_PASSWORD=mirrorbrain
13+
web:
14+
build: ../
15+
command: ["start.sh"]
16+
container_name: mb_web
17+
ports:
18+
- 127.0.0.1:8100:80
19+
environment:
20+
- GEOIPUPDATE=1
21+
- HTTPD_ONLY=1
22+
volumes:
23+
- ./mirrorbrain.conf:/etc/mirrorbrain.conf
24+
- ./httpd.conf:/usr/local/apache2/conf/httpd.conf
25+
- ./httpd-vhosts.conf:/usr/local/apache2/conf/extra/httpd-vhosts.conf
26+
- ./download.kiwix.org:/var/www/download.kiwix.org
27+
- ./GeoIP.conf:/etc/GeoIP.conf
28+
- ./init_mirrorbrain_db_dev.sh:/usr/local/apache2/init_mirrorbrain_db_dev.sh
29+
- ../sql/mirrors-postgresql.sql:/usr/local/apache2/mirrors-postgresql.sql
30+
# - /home/benoit/Repos/poeml/mirrorbrain/mb/mb:/usr/local/lib/python3.11/dist-packages/mb/
31+
# - /home/benoit/Repos/poeml/mirrorbrain/tools/null-rsync:/usr/bin/null-rsync
32+
- geoip:/usr/share/GeoIP
33+
depends_on:
34+
- postgresdb
35+
36+
volumes:
37+
pg_data_mirrorbrain:
38+
geoip:

‎mirrorbrain/dev/httpd-vhosts.conf

+96
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
<VirtualHost *:80>
2+
ServerName download.kiwix.org
3+
ServerAdmin contact+download@kiwix.org
4+
DocumentRoot /var/www/download.kiwix.org
5+
6+
Alias /flags "/var/www/flags"
7+
8+
MirrorBrainMetalinkPublisher "Kiwix project" https://kiwix.org
9+
MirrorBrainTorrentTrackerURL "http://tracker.openzim.org:6969/announce"
10+
MirrorBrainTorrentTrackerURL "udp://tracker.openzim.org:6969/announce"
11+
# MirrorBrainTorrentTrackerURL "https://opentracker.xyz:443/announce"
12+
# MirrorBrainTorrentTrackerURL "http://torrent.nwps.ws:80/announce"
13+
# MirrorBrainTorrentTrackerURL "udp://tracker.open-internet.nl:6969/announce"
14+
# MirrorBrainTorrentTrackerURL "udp://tracker.coppersurfer.tk:6969/announce"
15+
# MirrorBrainTorrentTrackerURL "udp://tracker.openbittorrent.com:80/announce"
16+
MirrorBrainDHTNode router.bittorrent.com 6881
17+
MirrorBrainDHTNode router.utorrent.com 6881
18+
19+
<IfModule mod_expires.c>
20+
ExpiresActive On
21+
ExpiresDefault "access plus 1 seconds"
22+
ExpiresByType text/html "access plus 1 seconds"
23+
ExpiresByType image/gif "access plus 120 minutes"
24+
ExpiresByType image/jpeg "access plus 120 minutes"
25+
ExpiresByType image/png "access plus 120 minutes"
26+
ExpiresByType text/css "access plus 60 minutes"
27+
ExpiresByType text/javascript "access plus 60 minutes"
28+
ExpiresByType application/x-javascript "access plus 60 minutes"
29+
ExpiresByType text/xml "access plus 60 minutes"
30+
</IfModule>
31+
32+
<IfModule maxminddb_module>
33+
MaxMindDBEnable On
34+
#MaxMindDBFile COUNTRY_DB /usr/share/GeoIP/GeoLite2-Country.mmdb
35+
MaxMindDBFile CITY_DB /usr/share/GeoIP/GeoLite2-City.mmdb
36+
# we use old DBEnv names to not modify mirrorbrain code
37+
MaxMindDBEnv GEOIP_COUNTRY_CODE CITY_DB/country/iso_code
38+
MaxMindDBEnv GEOIP_COUNTRY_NAME CITY_DB/country/names/en
39+
MaxMindDBEnv GEOIP_CONTINENT_CODE CITY_DB/continent/code
40+
MaxMindDBEnv GEOIP_LONGITUDE CITY_DB/location/longitude
41+
MaxMindDBEnv GEOIP_LATITUDE CITY_DB/location/latitude
42+
MaxMindDBEnv GEOIP_REGION CITY_DB/subdivisions/0/iso_code
43+
MaxMindDBEnv GEOIP_REGION_NAME CITY_DB/subdivisions/0/names/en
44+
</IfModule>
45+
46+
# This is a hack to avoid counting twice a same
47+
# download. Therwfore, we need to differenciate - in the Matomo
48+
# log indgester - Mirrorbrain generated redirects and our custom
49+
# permanent links. Therefore Matomo links use HTTP 302 and ours
50+
# HTTP 301 without a cache.
51+
<IfModule mod_rewrite.c>
52+
RewriteEngine on
53+
54+
# RewriteMap redirects-map "txt:/var/www/library.kiwix.org/download.kiwix.org.permalinks"
55+
# RewriteCond ${redirects-map:$1} !=""
56+
# RewriteRule ^(.*)$ ${redirects-map:$1} [last,redirect=301,E=NOCACHE:1]
57+
58+
# RewriteMap releases-map "txt:/data/maps/kiwix-releases.map"
59+
# RewriteCond ${releases-map:$1} !=""
60+
# RewriteRule ^(.*)$ ${releases-map:$1} [last,redirect=301,E=NOCACHE:1]
61+
62+
# RewriteMap zim-map "txt:/data/maps/zim.map"
63+
# RewriteCond ${zim-map:$1} !=""
64+
# RewriteRule ^(.*)$ ${zim-map:$1} [last,redirect=301,E=NOCACHE:1]
65+
66+
# RewriteMap nightly-map "txt:/data/maps/kiwix-nightly.map"
67+
# RewriteCond ${nightly-map:$1} !=""
68+
# RewriteRule ^(.*)$ ${nightly-map:$1} [last,redirect=301,E=NOCACHE:1]
69+
70+
Header always set Cache-Control "no-store, no-cache, must-revalidate" env=NOCACHE
71+
</IfModule>
72+
73+
<Directory /var/www/download.kiwix.org>
74+
MirrorBrainEngine On
75+
MirrorBrainDebug Off
76+
FormGET On
77+
MirrorBrainHandleHEADRequestLocally Off
78+
MirrorBrainMinSize 2048
79+
MirrorBrainExcludeUserAgent rpm/4.4.2*
80+
MirrorBrainExcludeUserAgent *APT-HTTP*
81+
MirrorBrainExcludeMimeType application/pgp-keys
82+
Options FollowSymLinks Indexes
83+
IndexOptions FancyIndexing
84+
AllowOverride All
85+
Order allow,deny
86+
Allow from all
87+
Header set Access-Control-Allow-Origin "*"
88+
</Directory>
89+
90+
<Directory /var/www/download.kiwix.org/nightly>
91+
AllowOverride none
92+
Options +Indexes
93+
IndexOptions FancyIndexing
94+
IndexOrderDefault Descending Date
95+
</Directory>
96+
</VirtualHost>

0 commit comments

Comments
 (0)