Merge branch 'master' into TAN-430-restructure-back-office-project-view

.circleci/config.yml

@@ -327,6 +327,8 @@ jobs:
     parameters:
       ssh_host:
         type: string
+      cluster_name:
+        type: string
       ssh_user:
         type: string
         default: ubuntu
@@ -343,7 +345,7 @@ jobs:
       - run:
           command: ssh << parameters.ssh_user >>@<< parameters.ssh_host >> -o StrictHostKeyChecking=no "docker pull citizenlabdotco/back-ee:$CIRCLE_BRANCH && docker run --env-file cl2-deployment/<< parameters.env_file >> citizenlabdotco/back-ee:$CIRCLE_BRANCH bin/rake db:migrate_if_pending cl2back:clean_tenant_settings email_campaigns:assure_campaign_records fix_existing_tenants:update_permissions cl2back:clear_cache_store email_campaigns:remove_deprecated"
       - deploy:
-          command: ssh << parameters.ssh_user >>@<< parameters.ssh_host >> -o StrictHostKeyChecking=no "cd cl2-deployment && docker stack deploy --compose-file << parameters.compose_file >> << parameters.stack_name >> --with-registry-auth"
+          command: ssh << parameters.ssh_user >>@<< parameters.ssh_host >> -o StrictHostKeyChecking=no "cd cl2-deployment && CL_CLUSTER_NAME=<<parameters.cluster_name>> docker stack deploy --compose-file << parameters.compose_file >> << parameters.stack_name >> --with-registry-auth"
 
   back-generate-tenant-templates:
     resource_class: small
@@ -840,6 +842,7 @@ workflows:
           compose_file: docker-compose-staging.yml
           stack_name: "cl2"
           env_file: ".env-web"
+          cluster_name: "stg"
 
   back-deploy-production:
     when: << pipeline.parameters.back >>
@@ -877,6 +880,7 @@ workflows:
           compose_file: docker-compose-production-benelux.yml
           stack_name: cl2-prd-bnlx-stack
           env_file: ".env-production-benelux"
+          cluster_name: "eu"
       - back-deploy-to-swarm:
           name: Deploy to Canada
           requires:
@@ -889,6 +893,7 @@ workflows:
           compose_file: docker-compose-production.yml
           env_file: ".env-web"
           stack_name: "cl2"
+          cluster_name: "can"
       - back-deploy-to-swarm:
           name: Deploy to South-America
           requires:
@@ -901,6 +906,7 @@ workflows:
           compose_file: docker-compose-production.yml
           env_file: ".env-web"
           stack_name: "cl2"
+          cluster_name: "sam"
       - back-deploy-to-swarm:
           name: Deploy to US-West
           requires:
@@ -913,6 +919,7 @@ workflows:
           compose_file: docker-compose-production.yml
           env_file: ".env-web"
           stack_name: "cl2"
+          cluster_name: "usw"
       - back-deploy-to-swarm:
           name: Deploy to UK
           requires:
@@ -925,6 +932,7 @@ workflows:
           compose_file: docker-compose-production.yml
           env_file: ".env-web"
           stack_name: "cl2"
+          cluster_name: "uk"
 
   # FRONT-END
   front:

Makefile

@@ -109,6 +109,23 @@ e2e-ci-env-run-test:
 	cd e2e && \
 	docker-compose run --rm --name cypress_run front npm run cypress:run -- --config baseUrl=http://e2e.front:3000 --spec ${spec}
 
+e2e-ci-env-db-dump:
+	cd e2e && \
+	docker compose exec postgres pg_dumpall -c -U postgres > dump.sql
+
+e2e-ci-env-db-restore:
+	cd e2e && \
+	docker compose exec postgres psql -U postgres -d cl2_back_development -c "SELECT 1" 1> /dev/null && \
+	docker compose exec postgres psql -U postgres -d cl2_back_development -c "DROP SCHEMA IF EXISTS e2e_front,public CASCADE" 1> /dev/null 2> /dev/null && \
+	docker compose exec postgres psql -U postgres -d cl2_back_development -c "CREATE SCHEMA public" && \
+	cat dump.sql | docker compose exec -T postgres psql --quiet -U postgres 1> /dev/null 2> /dev/null
+
+e2e-ci-env-reproduce-flaky-test:
+	for i in $(shell seq 1 10); do \
+		make e2e-ci-env-db-restore && \
+		make e2e-ci-env-run-test spec="${spec}"; \
+	done
+
 # =================
 # CircleCI
 # =================

back/app/controllers/web_api/v1/home_pages_controller.rb

@@ -52,7 +52,8 @@ def home_page_params
       banner_signed_out_subheader_multiloc: CL2_SUPPORTED_LOCALES,
       banner_cta_signed_out_text_multiloc: CL2_SUPPORTED_LOCALES,
       bottom_info_section_multiloc: CL2_SUPPORTED_LOCALES,
-      pinned_admin_publication_ids: []
+      pinned_admin_publication_ids: [],
+      craftjs_json: {}
     )
   end
 end

back/app/models/home_page.rb

@@ -29,6 +29,7 @@
 #  created_at                               :datetime         not null
 #  updated_at                               :datetime         not null
 #  header_bg                                :string
+#  craftjs_json                             :jsonb            not null
 #
 class HomePage < ApplicationRecord
   has_many :pins, as: :page, inverse_of: :page, dependent: :destroy

back/app/models/notifications/voting_basket_submitted.rb

@@ -78,7 +78,8 @@ def self.make_notifications_on(activity)
         [new(
           recipient_id: recipient_id,
           basket: basket,
-          project: basket.participation_context.project
+          project: basket.participation_context.project,
+          phase: basket.participation_context.phase? ? basket.participation_context : nil
         )]
       else
         []

back/app/serializers/web_api/v1/home_page_serializer.rb

@@ -20,7 +20,8 @@ class WebApi::V1::HomePageSerializer < WebApi::V1::BaseSerializer
     :banner_cta_signed_out_text_multiloc,
     :banner_cta_signed_out_type,
     :banner_cta_signed_out_url,
-    :pinned_admin_publication_ids
+    :pinned_admin_publication_ids,
+    :craftjs_json
 
   attribute :header_bg do |object|
     object.header_bg && object.header_bg.versions.to_h { |k, v| [k.to_s, v.url] }

back/app/services/cl2_data_listing_service.rb

@@ -25,13 +25,14 @@ def cl2_schema_models
     views = ActiveRecord::Base.connection.execute(
       "SELECT table_name FROM information_schema.tables WHERE table_type = 'VIEW'"
     ).pluck('table_name')
-    ActiveRecord::Base.descendants.select do |claz|
-      [
-        *ActiveRecord::Base.subclasses.map(&:name),
-        Tenant.name
-      ].exclude? claz.name
-    end.select do |claz|
-      views.exclude? claz.table_name
+    subclasses = [
+      *ActiveRecord::Base.subclasses.map(&:name),
+      Tenant.name
+    ]
+    ActiveRecord::Base.descendants.reject do |claz|
+      subclasses.include?(claz.name) ||
+        views.include?(claz.table_name) ||
+        claz.abstract_class
     end
   end
 

back/app/services/location/service.rb

@@ -7,8 +7,16 @@ def autocomplete(input, language)
   end
 
   def geocode(address, language)
-    response = HTTParty.get("https://maps.googleapis.com/maps/api/geocode/json?address=#{CGI.escape(address)}&key=#{api_key}&language=#{language}")
-    { location: response['results'].first['geometry']['location'] }
+    # Pass through unaltered if valid co-ordinates are entered
+    coordinates_regex = /^[-+]?([1-8]?\d(\.\d+)?|90(\.0+)?),\s*[-+]?(180(\.0+)?|((1[0-7]\d)|([1-9]?\d))(\.\d+)?)$/
+    if coordinates_regex.match? address
+      split_coordinates = address.split(',')
+      location = { lat: split_coordinates[0].to_f, lng: split_coordinates[1].to_f }
+    else
+      response = HTTParty.get("https://maps.googleapis.com/maps/api/geocode/json?address=#{CGI.escape(address)}&key=#{api_key}&language=#{language}")
+      location = response['results'].first['geometry']['location']
+    end
+    { location: location }
   end
 
   def reverse_geocode(lat, lng, language)

back/db/migrate/20231018083110_add_craftjs_json_to_home_pages.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddCraftjsJsonToHomePages < ActiveRecord::Migration[7.0]
+  def change
+    add_column :home_pages, :craftjs_json, :jsonb, default: {}, null: false
+  end
+end

back/db/migrate/20231109101517_add_memberships_count_to_groups.rb

@@ -0,0 +1,5 @@
+class AddMembershipsCountToGroups < ActiveRecord::Migration[7.0]
+  def change
+    add_column :groups, :memberships_count, :integer, null: false, default: 0, if_not_exists: true
+  end
+end

back/db/structure.sql

@@ -2431,7 +2431,8 @@ CREATE TABLE public.home_pages (
     banner_cta_signed_out_url character varying,
     created_at timestamp(6) without time zone NOT NULL,
     updated_at timestamp(6) without time zone NOT NULL,
-    header_bg character varying
+    header_bg character varying,
+    craftjs_json jsonb DEFAULT '{}'::jsonb NOT NULL
 );
 
 
@@ -7958,6 +7959,8 @@ INSERT INTO "schema_migrations" (version) VALUES
 ('20230913121819'),
 ('20230915391649'),
 ('20230927135924'),
-('20231003095622');
+('20231003095622'),
+('20231018083110'),
+('20231109101517');
 
 

back/engines/commercial/analysis/app/lib/analysis/LLM/gpt_4_turbo.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Analysis
+  module LLM
+    class GPT4Turbo < OpenAIGPT
+      def context_window
+        128_000
+      end
+
+      def enabled?
+        true
+      end
+
+      def accuracy
+        0.7
+      end
+
+      def gpt_model
+        'gpt-4-1106-preview'
+      end
+    end
+  end
+end

back/engines/commercial/analysis/app/lib/analysis/summarization_method/base.rb

@@ -11,6 +11,7 @@ class SummarizationMethod::Base
     LLMS = [
       LLM::GPT48k.new,
       LLM::GPT432k.new,
+      LLM::GPT4Turbo.new,
       LLM::GPT3516k.new
     ]
 

back/engines/commercial/id_criipto/app/lib/id_criipto/criipto_verification.rb

@@ -69,7 +69,7 @@ def exposed_config_parameters
     def profile_to_uid(auth)
       case config[:identity_source]
       when DK_MIT_ID
-        auth['uuid']
+        auth['uid']
       else
         raise "Unsupported identity source #{config[:identity_source]}"
       end