diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 9c156d5afc8e3..5701c79c8e718 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -192,7 +192,7 @@ jobs: echo "export USER_GID=$(id -g)" >> .envrc - name: Test make dev run: | - make DOCKER_LOCAL_DATA="$(pwd)" dev + make DOCKER_LOCAL_DATA="$(pwd)" SKIP_SAMPLE_IMAGES=1 dev make status - name: Test all is running run: make livecheck || ( tail -n 300 logs/apache2/*error*log; docker compose logs; false ) diff --git a/Makefile b/Makefile index 02af5e0539132..df0ae5bab9a6a 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,8 @@ SHELL := $(shell which bash) ENV_FILE ?= .env NAME = "ProductOpener" MOUNT_POINT ?= /mnt +# in CI, in make dev we want to skip downloading sample images (too slow) +SKIP_SAMPLE_IMAGES ?= SKIP_SAMPLE_IMAGES DOCKER_LOCAL_DATA_DEFAULT = /srv/off/docker_data DOCKER_LOCAL_DATA ?= $(DOCKER_LOCAL_DATA_DEFAULT) OS := $(shell uname) diff --git a/scripts/gen_feeds_daily.sh b/scripts/gen_feeds_daily.sh index 4c96561ebe533..4ddb000b86469 100755 --- a/scripts/gen_feeds_daily.sh +++ b/scripts/gen_feeds_daily.sh @@ -69,15 +69,20 @@ cd $OFF_SCRIPTS_DIR ./mongodb_dump.sh $OFF_PUBLIC_DATA_DIR $PRODUCT_OPENER_FLAVOR $MONGODB_HOST $PRODUCT_OPENER_FLAVOR_SHORT # Small products data and images export for Docker dev environments -# for about 1/10000th of the products contained in production. -./export_products_data_and_images.pl --sample-mod 10000,0 \ +# for about 1/100000th of the products contained in production. +./export_products_data_and_images.pl --sample-mod 100000,0 \ + --products-file $OFF_PUBLIC_EXPORTS_DIR/products.random-modulo-100000.tar.gz \ + --images-file $OFF_PUBLIC_EXPORTS_DIR/products.random-modulo-100000.images.tar.gz \ + --jsonl-file $OFF_PUBLIC_EXPORTS_DIR/products.random-modulo-100000.jsonl.gz \ + --mongo-file $OFF_PUBLIC_EXPORTS_DIR/products.random-modulo-100000.mongodbdump.gz +# On saturday, export modulo 1000 and 10000 for larger sample +if [ "$(date +%u)" = "6" ] +then + ./export_products_data_and_images.pl --sample-mod 10000,0 \ --products-file $OFF_PUBLIC_EXPORTS_DIR/products.random-modulo-10000.tar.gz \ --images-file $OFF_PUBLIC_EXPORTS_DIR/products.random-modulo-10000.images.tar.gz \ --jsonl-file $OFF_PUBLIC_EXPORTS_DIR/products.random-modulo-10000.jsonl.gz \ --mongo-file $OFF_PUBLIC_EXPORTS_DIR/products.random-modulo-10000.mongodbdump.gz -# On saturday, export modulo 1000 for larger sample -if [ "$(date +%u)" = "6" ] -then ./export_products_data_and_images.pl --sample-mod 1000,0 \ --products-file $OFF_PUBLIC_EXPORTS_DIR/products.random-modulo-1000.tar.gz \ --images-file $OFF_PUBLIC_EXPORTS_DIR/products.random-modulo-1000.images.tar.gz \ diff --git a/scripts/import_sample_data.sh b/scripts/import_sample_data.sh index ede03d176bfc5..965d476c7a0f0 100755 --- a/scripts/import_sample_data.sh +++ b/scripts/import_sample_data.sh @@ -7,14 +7,19 @@ cd /tmp echo "\033[32m------------------ 1/ Retrieve products -----------------\033[0m"; # explicitly specify the wget output file name so that wget does not append .1 if already present # e.g. if the tar command failed and the script was stopped -wget -O products.tar.gz https://static.openfoodfacts.org/exports/products.random-modulo-10000.tar.gz 2>&1 +wget -O products.tar.gz https://static.openfoodfacts.org/exports/products.random-modulo-100000.tar.gz 2>&1 tar -xzvf products.tar.gz -C /mnt/podata/products rm products.tar.gz -echo "\033[32m------------------ 2/ Retrieve product images -------------------\033[0m"; -wget -O products.images.tar.gz https://static.openfoodfacts.org/exports/products.random-modulo-10000.images.tar.gz 2>&1 -tar -xzvf products.images.tar.gz -C /opt/product-opener/html/images/products/ -rm products.images.tar.gz +if [[ -z "${SKIP_SAMPLE_IMAGES}" ]] +then + echo "\033[32m------------------ 2/ Retrieve product images -------------------\033[0m"; + wget -O products.images.tar.gz https://static.openfoodfacts.org/exports/products.random-modulo-100000.images.tar.gz 2>&1 + tar -xzvf products.images.tar.gz -C /opt/product-opener/html/images/products/ + rm products.images.tar.gz +else + echo "\033[32m------------------ SKIPPED product images -------------------\033[0m"; +fi echo "\033[32m------------------ 3/ Import products -------------------\033[0m"; perl -I/opt/product-opener/lib /opt/product-opener/scripts/update_all_products_from_dir_in_mongodb.pl diff --git a/taxonomies/food/categories.txt b/taxonomies/food/categories.txt index f1cd85bd9e3cb..c9f46fffd940d 100644 --- a/taxonomies/food/categories.txt +++ b/taxonomies/food/categories.txt @@ -68440,10 +68440,6 @@ tr: Armut intake24_category_code:en: PEAR wikidata:en: Q13099586 -< en:Pears -en: Pears (Guyot) -fr: Poires Guyot - < en:Pears it: Pera dell'Emilia Romagna origins:en: en:italy @@ -68458,7 +68454,7 @@ hr: Konferencijske kruške nl: Conference peren < en:Pears -en: Guyot pears +en: Guyot pears, Pears (Guyot) xx: Guyot fr: Poires Guyot wikidata:en: Q3033517