diff --git a/Gemfile b/Gemfile index a3b576eaf..83f31bcfd 100644 --- a/Gemfile +++ b/Gemfile @@ -100,6 +100,9 @@ gem "interactor", "~> 3.0" gem "sentry-rails" gem "sentry-ruby" +# Utilities +gem "http" + group :development, :test do gem "debug", platforms: %i[mri mingw x64_mingw] gem "factory_bot_rails" diff --git a/Gemfile.lock b/Gemfile.lock index 67231c7cd..95a0c7708 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -159,10 +159,11 @@ GEM ast (2.4.2) attr_extras (7.1.0) bcrypt (3.1.20) + bigdecimal (3.1.4) bindex (0.8.1) bootsnap (1.17.0) msgpack (~> 1.2) - brakeman (6.0.1) + brakeman (6.1.0) builder (3.2.4) bullet (7.1.4) activesupport (>= 3.0.0) @@ -209,11 +210,11 @@ GEM carrierwave (>= 2.2.1) marcel (~> 1.0.0) mime-types (~> 3.0) - chartkick (5.0.4) + chartkick (5.0.5) choice (0.2.0) concurrent-ruby (1.2.2) connection_pool (2.4.1) - countries (5.7.0) + countries (5.7.1) unaccent (~> 0.3) crack (0.4.5) rexml @@ -239,6 +240,7 @@ GEM warden (~> 1.2.3) diff-lcs (1.5.0) docile (1.4.0) + domain_name (0.6.20231109) dotenv (2.8.1) dotenv-rails (2.8.1) dotenv (= 2.8.1) @@ -262,6 +264,9 @@ GEM webrick (~> 1.7) websocket-driver (>= 0.6, < 0.8) ffi (1.16.3) + ffi-compiler (1.0.1) + ffi (>= 1.0.0) + rake formtastic (4.0.0) actionpack (>= 5.2.0) formtastic_i18n (0.7.0) @@ -278,6 +283,14 @@ GEM activesupport (>= 5.2) hashdiff (1.0.1) htmlentities (4.3.4) + http (5.1.1) + addressable (~> 2.8) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.4.0) + http-cookie (1.0.5) + domain_name (~> 0.5) + http-form_data (2.3.0) i18n (1.14.1) concurrent-ruby (~> 1.0) i18n_generators (2.2.2) @@ -293,14 +306,14 @@ GEM responders (>= 2) interactor (3.1.2) io-console (0.6.0) - irb (1.9.1) + irb (1.10.1) rdoc reline (>= 0.3.8) jquery-rails (4.6.0) rails-dom-testing (>= 1, < 3) railties (>= 4.2.0) thor (>= 0.14, < 2.0) - json (2.7.0) + json (2.7.1) jsonapi-resources (0.9.12) activerecord (>= 4.1) concurrent-ruby @@ -329,6 +342,9 @@ GEM railties (>= 5.2) rexml lint_roller (1.1.0) + llhttp-ffi (0.4.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) loofah (2.22.0) crass (~> 1.0.2) nokogiri (>= 1.12.0) @@ -342,7 +358,7 @@ GEM method_source (1.0.0) mime-types (3.5.1) mime-types-data (~> 3.2015) - mime-types-data (3.2023.1003) + mime-types-data (3.2023.1205) mini_magick (4.12.0) mini_mime (1.1.5) mini_portile2 (2.8.5) @@ -368,7 +384,8 @@ GEM racc (~> 1.4) nokogiri (1.15.5-x86_64-linux) racc (~> 1.4) - oj (3.16.1) + oj (3.16.2) + bigdecimal (~> 3.1) oj_mimic_json (1.0.1) optimist (3.1.0) orm_adapter (0.5.0) @@ -395,6 +412,8 @@ GEM rack (>= 2.0.0) rack-mini-profiler (2.3.4) rack (>= 1.2.0) + rack-session (1.0.2) + rack (< 3) rack-test (2.1.0) rack (>= 1.3) rails (7.0.8) @@ -436,21 +455,21 @@ GEM activerecord (>= 6.1.5) activesupport (>= 6.1.5) i18n - rdoc (6.6.0) + rdoc (6.6.1) psych (>= 4.0.0) redis (5.0.8) redis-client (>= 0.17.0) - redis-actionpack (5.3.0) + redis-actionpack (5.4.0) actionpack (>= 5, < 8) - redis-rack (>= 2.1.0, < 3) + redis-rack (>= 2.1.0, < 4) redis-store (>= 1.1.0, < 2) redis-activesupport (5.3.0) activesupport (>= 3, < 8) redis-store (>= 1.3, < 2) - redis-client (0.18.0) + redis-client (0.19.0) connection_pool - redis-rack (2.1.4) - rack (>= 2.0.8, < 3) + redis-rack (3.0.0) + rack-session (>= 0.2.0) redis-store (>= 1.2, < 2) redis-rails (5.0.2) redis-actionpack (>= 5.0, < 6) @@ -458,7 +477,7 @@ GEM redis-store (>= 1.2, < 2) redis-store (1.10.0) redis (>= 4, < 6) - regexp_parser (2.8.2) + regexp_parser (2.8.3) reline (0.4.1) io-console (~> 0.5) request_store (1.5.1) @@ -551,10 +570,10 @@ GEM sendgrid-ruby (~> 6.4) sendgrid-ruby (6.7.0) ruby_http_client (~> 3.4) - sentry-rails (5.14.0) + sentry-rails (5.15.0) railties (>= 5.0) - sentry-ruby (~> 5.14.0) - sentry-ruby (5.14.0) + sentry-ruby (~> 5.15.0) + sentry-ruby (5.15.0) concurrent-ruby (~> 1.0, >= 1.0.2) shoulda-matchers (4.0.1) activesupport (>= 4.2.0) @@ -580,7 +599,7 @@ GEM net-scp (>= 1.1.2) net-ssh (>= 2.8.0) ssrf_filter (1.1.2) - standard (1.32.0) + standard (1.32.1) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.0) rubocop (~> 1.57.2) @@ -677,6 +696,7 @@ DEPENDENCIES globalize globalize-versioning! groupdate + http i18n_generators interactor (~> 3.0) jsonapi-resources (= 0.9.12) diff --git a/app/admin/custom_admin_header.rb b/app/admin/custom_admin_header.rb index 883587bae..533a4ebeb 100644 --- a/app/admin/custom_admin_header.rb +++ b/app/admin/custom_admin_header.rb @@ -70,6 +70,7 @@ def build(namespace, menu) # rubocop:disable Metrics/AbcSize text_node content_tag "a", t("active_admin.menu.private_sector.settings.settings"), class: "-with-children" ul do li { link_to t("active_admin.menu.private_sector.settings.countries"), admin_countries_path } + li { link_to t("active_admin.menu.private_sector.settings.protected_areas"), admin_protected_areas_path } li { link_to t("active_admin.menu.private_sector.settings.fmus"), admin_fmus_path } li { link_to t("active_admin.menu.private_sector.settings.fmu_allocations"), admin_fmu_operators_path } end diff --git a/app/admin/fmu.rb b/app/admin/fmu.rb index 599192802..7c7d02e2f 100644 --- a/app/admin/fmu.rb +++ b/app/admin/fmu.rb @@ -11,19 +11,6 @@ config.order_clause controller do - def preview - file = params["file"] - max_file_size = 200_000 - response = if file.blank? || file.size > max_file_size - {errors: "File must exist and be smaller than #{max_file_size / 1000} KB"} - else - Fmu.file_upload(file) - end - respond_to do |format| - format.json { render json: response } - end - end - def scoped_collection end_of_association_chain.with_translations(I18n.locale).includes(:country, :operator) end @@ -88,8 +75,13 @@ def scoped_collection row :certification_fsc_cw row :certification_tlv row :certification_ls + if resource.geojson && resource.centroid.present? + row :map do |r| + render partial: "map", locals: {center: [r.centroid.x, r.centroid.y], center_marker: false, geojson: r.geojson, bbox: r.bbox} + end + end row(:geojson) { |fmu| fmu.geojson.to_json } - row(:properties) { |fmu| fmu.geojson&.dig("properties") } + row(:properties) { |fmu| fmu.geojson&.dig("properties")&.to_json } row :created_at row :updated_at row :deleted_at @@ -114,28 +106,47 @@ def scoped_collection form do |f| f.semantic_errors(*f.object.errors.attribute_names) - f.inputs I18n.t("active_admin.shared.fmu_details") do - f.input :country, input_html: {disabled: object.persisted?}, required: true - f.input :esri_shapefiles_zip, as: :file, input_html: {accept: ".zip"} - render partial: "zip_hint" - f.input :forest_type, as: :select, - collection: ForestType::TYPES.map { |key, v| [v[:label], key] }, - input_html: {disabled: object.persisted?} - f.input :certification_fsc - f.input :certification_pefc - f.input :certification_olb - f.input :certification_pafc - f.input :certification_fsc_cw - f.input :certification_tlv - f.input :certification_ls - end + columns class: "d-flex" do + column max_width: "500px" do + f.inputs I18n.t("active_admin.shared.fmu_details") do + f.input :country, input_html: {disabled: object.persisted?}, required: true + f.input :forest_type, as: :select, + collection: ForestType::TYPES.map { |key, v| [v[:label], key] }, + input_html: {disabled: object.persisted?} + f.input :certification_fsc + f.input :certification_pefc + f.input :certification_olb + f.input :certification_pafc + f.input :certification_fsc_cw + f.input :certification_tlv + f.input :certification_ls + end + + f.inputs I18n.t("activerecord.models.operator"), for: [:fmu_operator, f.object.fmu_operator || FmuOperator.new] do |fo| + fo.input :operator_id, label: I18n.t("activerecord.attributes.fmu/translation.name"), as: :select, + collection: Operator.active.map { |o| [o.name, o.id] }, + input_html: {disabled: object.persisted?}, required: false + fo.input :start_date, input_html: {disabled: object.persisted?}, required: false + fo.input :end_date, input_html: {disabled: object.persisted?} + end + end - f.inputs I18n.t("activerecord.models.operator"), for: [:fmu_operator, f.object.fmu_operator || FmuOperator.new] do |fo| - fo.input :operator_id, label: I18n.t("activerecord.attributes.fmu/translation.name"), as: :select, - collection: Operator.active.map { |o| [o.name, o.id] }, - input_html: {disabled: object.persisted?}, required: false - fo.input :start_date, input_html: {disabled: object.persisted?}, required: false - fo.input :end_date, input_html: {disabled: object.persisted?} + column class: "flex-1" do + f.inputs Fmu.human_attribute_name(:geometry) do + f.input :esri_shapefiles_zip, as: :esri_shapefile_zip + + render partial: "upload_geometry_map", + locals: { + file_input_id: "fmu_esri_shapefiles_zip", + geojson: f.resource.geojson, + bbox: f.resource.bbox, + present: f.resource.geojson.present?, + host: Rails.env.development? ? request.base_url : request.base_url + "/api", + show_fmus: true, + api_key: ENV["API_KEY"] + } + end + end end f.inputs I18n.t("active_admin.shared.translated_fields") do @@ -144,14 +155,5 @@ def scoped_collection end end f.actions - - render partial: "form", - locals: { - geojson: f.resource.geojson, - bbox: f.resource.bbox, - present: f.resource.geojson.present?, - host: Rails.env.development? ? request.base_url : request.base_url + "/api", - api_key: ENV["API_KEY"] - } end end diff --git a/app/admin/protected_area.rb b/app/admin/protected_area.rb new file mode 100644 index 000000000..8880210b0 --- /dev/null +++ b/app/admin/protected_area.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +ActiveAdmin.register ProtectedArea do + extend BackRedirectable + + menu false + + permit_params [:name, :country_id, :esri_shapefiles_zip, :geojson, :wdpa_pid] + + filter :wdpa_pid + filter :name + filter :country, + as: :select, + collection: -> { Country.active.with_translations(I18n.locale) } + + index do + column :id + column :wdpa_pid + column :country + column :name + + actions + end + + form do |f| + f.semantic_errors(*f.object.errors.attribute_names) + + columns class: "d-flex" do + column max_width: "500px" do + f.inputs I18n.t("active_admin.details", model: I18n.t("activerecord.models.protected_area")) do + f.input :wdpa_pid + f.input :country + f.input :name + end + end + + column class: "flex-1" do + f.inputs ProtectedArea.human_attribute_name(:geometry) do + f.input :esri_shapefiles_zip, as: :esri_shapefile_zip + render partial: "upload_geometry_map", + locals: { + file_input_id: "protected_area_esri_shapefiles_zip", + geojson: f.resource.geojson, + bbox: f.resource.bbox, + present: f.resource.geojson.present?, + host: Rails.env.development? ? request.base_url : request.base_url + "/api", + show_fmus: false, + api_key: ENV["API_KEY"] + } + end + end + end + + f.actions + end + + show do + attributes_table do + row :wdpa_pid + row :name + row :country + row :map do |r| + render partial: "map", locals: {center: [r.centroid.x, r.centroid.y], center_marker: false, geojson: r.geojson, bbox: r.bbox} + end + row :geojson do |r| + r.geojson.to_json + end + end + active_admin_comments + end +end diff --git a/app/assets/stylesheets/active_admin.scss b/app/assets/stylesheets/active_admin.scss index 0bfdd951f..0860fce7e 100644 --- a/app/assets/stylesheets/active_admin.scss +++ b/app/assets/stylesheets/active_admin.scss @@ -156,7 +156,7 @@ body.active_admin { font-size: 1.1em; } - .fmu-hints { + .form-input-hint { font-size: 0.95em; font-style: italic; color: #666; diff --git a/app/assets/stylesheets/helpers.scss b/app/assets/stylesheets/helpers.scss index 8f95c1ed8..9213e5c31 100644 --- a/app/assets/stylesheets/helpers.scss +++ b/app/assets/stylesheets/helpers.scss @@ -6,3 +6,13 @@ margin-top: 5px; margin-bottom: 5px; } + +.d-flex { + display: flex;; +} + +.flex-1 { + flex: 1; +} + + diff --git a/app/controllers/admin/geometry_previews_controller.rb b/app/controllers/admin/geometry_previews_controller.rb new file mode 100644 index 000000000..2a21a67b1 --- /dev/null +++ b/app/controllers/admin/geometry_previews_controller.rb @@ -0,0 +1,25 @@ +class Admin::GeometryPreviewsController < ApplicationController + before_action :authenticate_user! + + def create + file = params["file"] + max_file_size = 200_000 + response = if file.blank? || file.size > max_file_size + {errors: "File must exist and be smaller than #{max_file_size / 1000} KB"} + else + geojson = EsriShapefileUpload.geojson_from_file(file.path) + if geojson.nil? + {errors: "No geojson found in zip file"} + else + geometry = RGeo::GeoJSON.decode geojson + bbox = RGeo::Cartesian::BoundingBox.create_from_geometry(geometry.geometry) + { + geojson: geojson, + bbox: [bbox.min_x, bbox.min_y, bbox.max_x, bbox.max_y] + } + end + end + + render json: response + end +end diff --git a/app/controllers/v1/fmus_controller.rb b/app/controllers/v1/fmus_controller.rb index 91cd98edc..2513c9294 100644 --- a/app/controllers/v1/fmus_controller.rb +++ b/app/controllers/v1/fmus_controller.rb @@ -18,8 +18,7 @@ def index end def tiles - tile = Fmu.vector_tiles params[:z], params[:x], params[:y], params[:operator_id] - send_data tile + send_data FmuVectorTile.fetch params[:x], params[:y], params[:z], params[:operator_id] end private diff --git a/app/controllers/v1/protected_areas_controller.rb b/app/controllers/v1/protected_areas_controller.rb new file mode 100644 index 000000000..2e25c6ee0 --- /dev/null +++ b/app/controllers/v1/protected_areas_controller.rb @@ -0,0 +1,11 @@ +module V1 + class ProtectedAreasController < APIController + include ErrorSerializer + + skip_before_action :authenticate, only: [:tiles] + + def tiles + send_data ProtectedAreaVectorTile.fetch params[:x], params[:y], params[:z] + end + end +end diff --git a/app/inputs/esri_shapefile_zip_input.rb b/app/inputs/esri_shapefile_zip_input.rb new file mode 100644 index 000000000..40b75fa55 --- /dev/null +++ b/app/inputs/esri_shapefile_zip_input.rb @@ -0,0 +1,13 @@ +class EsriShapefileZipInput < Formtastic::Inputs::FileInput + def input_html_options + {accept: ".zip"}.merge(super) + end + + def hint_html + builder.template.content_tag(:div) do + I18n.t("active_admin.esri_shapefile_zip_input.hint").split('\n').map do |line| + builder.template.content_tag(:p, line, class: "form-input-hint") + end.join.html_safe + end + end +end diff --git a/app/models/concerns/esri_shapefile_upload.rb b/app/models/concerns/esri_shapefile_upload.rb new file mode 100644 index 000000000..0581f9065 --- /dev/null +++ b/app/models/concerns/esri_shapefile_upload.rb @@ -0,0 +1,20 @@ +module EsriShapefileUpload + extend ActiveSupport::Concern + + included do + attr_reader :esri_shapefiles_zip + end + + def esri_shapefiles_zip=(esri_shapefiles_zip) + self.geojson = geojson_from_file(esri_shapefiles_zip.path) + @esri_shapefiles_zip = esri_shapefiles_zip + end + + def geojson_from_file(filepath) + FileDataImport::Parser::Zip.new(filepath).foreach_with_line do |attributes, index| + # takes only the first feature from the Esri shapefile. + return attributes[:geojson].slice("type", "geometry").merge("properties" => {}) + end + end + module_function :geojson_from_file +end diff --git a/app/models/fmu.rb b/app/models/fmu.rb index 68528de71..923d45f24 100644 --- a/app/models/fmu.rb +++ b/app/models/fmu.rb @@ -27,12 +27,11 @@ class Fmu < ApplicationRecord has_paper_trail acts_as_paranoid + include EsriShapefileUpload include ValidationHelper include Translatable translates :name, paranoia: true, touch: true, versioning: :paper_trail - attr_reader :esri_shapefiles_zip - active_admin_translates :name do validates :name, presence: true end @@ -57,7 +56,7 @@ class Fmu < ApplicationRecord validates :name, presence: true validates :forest_type, presence: true - validate :geojson_correctness, if: :geojson_changed? + validates :geojson, geojson: true, if: :geojson_changed? after_save :update_geometry, if: :saved_change_to_geojson? @@ -90,44 +89,6 @@ def fetch_all(options) fmus = fmus.filter_by_free if free fmus end - - # Returns a vector tile for the X,Y,Z provided - def vector_tiles(param_z, param_x, param_y, operator_id) - begin - x, y, z = Integer(param_x), Integer(param_y), Integer(param_z) - rescue ArgumentError, TypeError - return nil - end - - operator_condition = operator_id.present? ? sanitize_sql(["AND operator_id=?", operator_id]) : "" - - query = <<~SQL - SELECT ST_ASMVT(tile.*, 'layer0', 4096, 'mvtgeometry', 'id') as tile - FROM ( - SELECT id, geojson -> 'properties' as properties, ST_AsMVTGeom(the_geom_webmercator, ST_TileEnvelope(#{z},#{x},#{y}), 4096, 256, true) AS mvtgeometry - FROM ( - SELECT fmus.*, st_transform(geometry, 3857) as the_geom_webmercator - FROM fmus - LEFT JOIN fmu_operators fo on fo.fmu_id = fmus.id and fo.current = true - WHERE fmus.deleted_at IS NULL #{operator_condition} - ) as data - WHERE ST_AsMVTGeom(the_geom_webmercator, ST_TileEnvelope(#{z},#{x},#{y}),4096,0,true) IS NOT NULL - ) AS tile; - SQL - - tile = ActiveRecord::Base.connection.execute query - ActiveRecord::Base.connection.unescape_bytea tile.getvalue(0, 0) - end - end - - def esri_shapefiles_zip=(esri_shapefiles_zip) - FileDataImport::Parser::Zip.new(esri_shapefiles_zip.path).foreach_with_line do |attributes, index| - # takes only the first feature from the Esri shapefile. - self.geojson = attributes[:geojson].slice("type", "geometry").merge("properties" => {}) - break - end - - @esri_shapefiles_zip = esri_shapefiles_zip end def update_geojson_properties @@ -161,42 +122,17 @@ def properties geojson["properties"] end - def bbox - query = <<~SQL - SELECT st_astext(st_envelope(geometry)) - FROM fmus - where id = #{id} - SQL - envelope = - ActiveRecord::Base.connection.execute(query)[0]["st_astext"][9..-3] - .split(/ |,/).map(&:to_f).each_slice(2).to_a - [envelope[0], envelope[2]] + def centroid + RGeo::GeoJSON.decode(geojson["properties"]["centroid"]) rescue nil end - def self.file_upload(esri_shapefiles_zip) - PaperTrail.request.disable_model(Fmu) - PaperTrail.request.disable_model(Fmu::Translation) - tmp_fmu = Fmu.new(name: "Test #{Time.now.to_i}", country_id: Country.first.id) - FileDataImport::Parser::Zip.new(esri_shapefiles_zip.path).foreach_with_line do |attributes, _index| - tmp_fmu.geojson = attributes[:geojson].slice("type", "geometry").merge("properties" => {}) - break - end - tmp_fmu.save(validate: false) - - response = { - geojson: tmp_fmu.geojson, - bbox: tmp_fmu.bbox - } - tmp_fmu.really_destroy! - PaperTrail.request.enable_model(Fmu) - PaperTrail.request.enable_model(Fmu::Translation) - response - rescue => e - PaperTrail.request.enable_model(Fmu) - PaperTrail.request.enable_model(Fmu::Translation) - {errors: e.message} + def bbox + return nil if geometry.nil? + + bbox = RGeo::Cartesian::BoundingBox.create_from_geometry(geometry) + [bbox.min_x, bbox.min_y, bbox.max_x, bbox.max_y] end def cache_key @@ -216,39 +152,11 @@ def cache_key private def update_geometry - query = <<~SQL - update fmus - set geometry = ST_GeomFromGeoJSON(geojson -> 'geometry') - where fmus.id = :fmu_id - SQL - ActiveRecord::Base.connection.update(Fmu.sanitize_sql_for_assignment([query, fmu_id: id])) + self.class.unscoped.where(id: id).update_all("geometry = ST_GeomFromGeoJSON(geojson -> 'geometry')") update_centroid end def update_centroid - query = <<~SQL - update fmus - set geojson = jsonb_set(geojson, '{properties,centroid}', ST_AsGeoJSON(st_centroid(geometry))::jsonb, true) - where fmus.id = :fmu_id; - SQL - ActiveRecord::Base.connection.update(Fmu.sanitize_sql_for_assignment([query, fmu_id: id])) - end - - def geojson_correctness - return if geojson.blank? - - temp_geometry = RGeo::GeoJSON.decode geojson - bbox = RGeo::Cartesian::BoundingBox.create_from_geometry(temp_geometry.geometry) - validate_bbox bbox - rescue RGeo::Error::InvalidGeometry - errors.add(:geojson, "Failed linear ring test") - rescue => e - errors.add(:geojson, "Error: #{e.message}") - end - - def validate_bbox(bbox) - return if bbox.max_x <= 180 && bbox.min_x >= -180 && bbox.max_y <= 90 && bbox.min_y >= -90 - - errors.add(:geojson, "The FMU's bbox is bigger than the globe. Please make sure your projection is 4326") + self.class.unscoped.where(id: id).update_all("geojson = jsonb_set(geojson, '{properties,centroid}', ST_AsGeoJSON(st_centroid(geometry))::jsonb, true)") end end diff --git a/app/models/fmu_vector_tile.rb b/app/models/fmu_vector_tile.rb new file mode 100644 index 000000000..1a087f212 --- /dev/null +++ b/app/models/fmu_vector_tile.rb @@ -0,0 +1,28 @@ +class FmuVectorTile + def self.fetch(param_x, param_y, param_z, operator_id) + begin + x, y, z = Integer(param_x), Integer(param_y), Integer(param_z) + rescue ArgumentError, TypeError + return nil + end + + operator_condition = operator_id.present? ? ActiveRecord::Base.sanitize_sql(["AND operator_id=?", operator_id]) : "" + + query = <<~SQL + SELECT ST_ASMVT(tile.*, 'layer0', 4096, 'mvtgeometry', 'id') as tile + FROM ( + SELECT id, geojson -> 'properties' as properties, ST_AsMVTGeom(the_geom_webmercator, ST_TileEnvelope(#{z},#{x},#{y}), 4096, 256, true) AS mvtgeometry + FROM ( + SELECT fmus.*, st_transform(geometry, 3857) as the_geom_webmercator + FROM fmus + LEFT JOIN fmu_operators fo on fo.fmu_id = fmus.id and fo.current = true + WHERE fmus.deleted_at IS NULL #{operator_condition} + ) as data + WHERE ST_AsMVTGeom(the_geom_webmercator, ST_TileEnvelope(#{z},#{x},#{y}),4096,0,true) IS NOT NULL + ) AS tile; + SQL + + tile = ActiveRecord::Base.connection.execute query + ActiveRecord::Base.connection.unescape_bytea tile.getvalue(0, 0) + end +end diff --git a/app/models/protected_area.rb b/app/models/protected_area.rb new file mode 100644 index 000000000..58bec700f --- /dev/null +++ b/app/models/protected_area.rb @@ -0,0 +1,57 @@ +# == Schema Information +# +# Table name: protected_areas +# +# id :bigint not null, primary key +# country_id :bigint not null +# name :string not null +# wdpa_pid :string not null +# geojson :jsonb not null +# geometry :geometry geometry, 0 +# centroid :geometry point, 0 +# created_at :datetime not null +# updated_at :datetime not null +# +class ProtectedArea < ApplicationRecord + include EsriShapefileUpload + + belongs_to :country + + validates :wdpa_pid, presence: true + validates :name, presence: true + validates :geojson, presence: true + validates :geojson, geojson: true + + after_save :update_geometry, if: :saved_change_to_geojson? + + def geojson=(value) + geojson = case value + when String + ActiveSupport::JSON.decode(value) + when Hash + value.with_indifferent_access + else + value + end + if geojson.present? && geojson["type"] != "Feature" + geojson = { + type: "Feature", + geometry: geojson + } + end + super(geojson) + end + + def bbox + return nil if geometry.nil? + + bbox = RGeo::Cartesian::BoundingBox.create_from_geometry(geometry) + [bbox.min_x, bbox.min_y, bbox.max_x, bbox.max_y] + end + + private + + def update_geometry + self.class.unscoped.where(id: id).update_all("geometry = ST_GeomFromGeoJSON(geojson -> 'geometry')") + end +end diff --git a/app/models/protected_area_vector_tile.rb b/app/models/protected_area_vector_tile.rb new file mode 100644 index 000000000..36c5a36a3 --- /dev/null +++ b/app/models/protected_area_vector_tile.rb @@ -0,0 +1,24 @@ +class ProtectedAreaVectorTile + def self.fetch(param_x, param_y, param_z) + begin + x, y, z = Integer(param_x), Integer(param_y), Integer(param_z) + rescue ArgumentError, TypeError + return nil + end + + query = <<~SQL + SELECT ST_ASMVT(tile.*, 'layer0', 4096, 'mvtgeometry', 'id') as tile + FROM ( + SELECT id, json_build_object('name', name) as properties, ST_AsMVTGeom(the_geom_webmercator, ST_TileEnvelope(#{z},#{x},#{y}), 4096, 256, true) AS mvtgeometry + FROM ( + SELECT protected_areas.*, st_transform(geometry, 3857) as the_geom_webmercator + FROM protected_areas + ) as data + WHERE ST_AsMVTGeom(the_geom_webmercator, ST_TileEnvelope(#{z},#{x},#{y}),4096,0,true) IS NOT NULL + ) AS tile; + SQL + + tile = ActiveRecord::Base.connection.execute query + ActiveRecord::Base.connection.unescape_bytea tile.getvalue(0, 0) + end +end diff --git a/app/validators/geojson_validator.rb b/app/validators/geojson_validator.rb new file mode 100644 index 000000000..67565e73a --- /dev/null +++ b/app/validators/geojson_validator.rb @@ -0,0 +1,24 @@ +class GeojsonValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + return if value.blank? + temp_geometry = RGeo::GeoJSON.decode value + if temp_geometry.geometry.present? + bbox = RGeo::Cartesian::BoundingBox.create_from_geometry(temp_geometry.geometry) + validate_bbox record, attribute, bbox + else + record.errors.add(attribute, "No geometry found in geojson") + end + rescue RGeo::Error::InvalidGeometry + record.errors.add(attribute, "Failed linear ring test") + rescue + record.errors.add(attribute, "Incorrect geojson") + end + + private + + def validate_bbox(record, attribute, bbox) + return if bbox.max_x <= 180 && bbox.min_x >= -180 && bbox.max_y <= 90 && bbox.min_y >= -90 + + record.errors.add(attribute, "The FMU's bbox is bigger than the globe. Please make sure your projection is 4326") + end +end diff --git a/app/views/admin/observations/_attributes_table.html.arb b/app/views/active_admin/resource/_attributes_table.html.arb similarity index 100% rename from app/views/admin/observations/_attributes_table.html.arb rename to app/views/active_admin/resource/_attributes_table.html.arb diff --git a/app/views/admin/observations/_map.html.erb b/app/views/active_admin/resource/_map.html.erb similarity index 85% rename from app/views/admin/observations/_map.html.erb rename to app/views/active_admin/resource/_map.html.erb index fe85cfa03..c79c15b07 100644 --- a/app/views/admin/observations/_map.html.erb +++ b/app/views/active_admin/resource/_map.html.erb @@ -6,6 +6,8 @@ .map-wrapper { width: 100%; padding: 16px; } +<% show_center_marker = true unless local_assigns.key?(:center_marker) %> +
- <%= line %> -
-<% end %> \ No newline at end of file diff --git a/config/brakeman.ignore b/config/brakeman.ignore index 480016a61..3269fc673 100644 --- a/config/brakeman.ignore +++ b/config/brakeman.ignore @@ -1,5 +1,28 @@ { "ignored_warnings": [ + { + "warning_type": "SQL Injection", + "warning_code": 0, + "fingerprint": "09f84900e658e1395df00795db5bc5db44f9f7f0bb6828932616d32930872bb0", + "check_name": "SQL", + "message": "Possible SQL injection", + "file": "app/models/fmu_vector_tile.rb", + "line": 25, + "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", + "code": "ActiveRecord::Base.connection.execute(\"SELECT ST_ASMVT(tile.*, 'layer0', 4096, 'mvtgeometry', 'id') as tile\\n FROM (\\n SELECT id, geojson -> 'properties' as properties, ST_AsMVTGeom(the_geom_webmercator, ST_TileEnvelope(#{Integer(param_z)},#{Integer(param_x)},#{Integer(param_y)}), 4096, 256, true) AS mvtgeometry\\n FROM (\\n SELECT fmus.*, st_transform(geometry, 3857) as the_geom_webmercator\\n FROM fmus\\n LEFT JOIN fmu_operators fo on fo.fmu_id = fmus.id and fo.current = true\\n WHERE fmus.deleted_at IS NULL #{(ActiveRecord::Base.sanitize_sql([\"AND operator_id=?\", operator_id]) or \"\")}\\n ) as data\\n WHERE ST_AsMVTGeom(the_geom_webmercator, ST_TileEnvelope(#{Integer(param_z)},#{Integer(param_x)},#{Integer(param_y)}),4096,0,true) IS NOT NULL\\n ) AS tile;\\n\")", + "render_path": null, + "location": { + "type": "method", + "class": "FmuVectorTile", + "method": "FmuVectorTile.fetch" + }, + "user_input": "Integer(param_z)", + "confidence": "Medium", + "cwe_id": [ + 89 + ], + "note": "" + }, { "warning_type": "SQL Injection", "warning_code": 0, @@ -7,7 +30,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "app/resources/v1/observation_resource.rb", - "line": 63, + "line": 62, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "records.joins(:observation_report).where(\"extract(year from observation_reports.publication_date) in (#{years})\")", "render_path": null, @@ -26,18 +49,18 @@ { "warning_type": "SQL Injection", "warning_code": 0, - "fingerprint": "23d495db5516c045871325ae5e92b8e853e562ebeb5a87bc3486378ded9ae019", + "fingerprint": "27286ce5352a110cc3f1c9ebd509311407e55c697f3abc47e6896a0cd5bfecc0", "check_name": "SQL", "message": "Possible SQL injection", - "file": "app/models/fmu.rb", - "line": 118, + "file": "app/models/protected_area_vector_tile.rb", + "line": 21, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", - "code": "ActiveRecord::Base.connection.execute(\"SELECT ST_ASMVT(tile.*, 'layer0', 4096, 'mvtgeometry', 'id') as tile\\n FROM (\\n SELECT id, geojson -> 'properties' as properties, ST_AsMVTGeom(the_geom_webmercator, ST_TileEnvelope(#{Integer(param_z)},#{Integer(param_x)},#{Integer(param_y)}), 4096, 256, true) AS mvtgeometry\\n FROM (\\n SELECT fmus.*, st_transform(geometry, 3857) as the_geom_webmercator\\n FROM fmus\\n LEFT JOIN fmu_operators fo on fo.fmu_id = fmus.id and fo.current = true\\n WHERE fmus.deleted_at IS NULL #{(sanitize_sql([\"AND operator_id=?\", operator_id]) or \"\")}\\n ) as data\\n WHERE ST_AsMVTGeom(the_geom_webmercator, ST_TileEnvelope(#{Integer(param_z)},#{Integer(param_x)},#{Integer(param_y)}),4096,0,true) IS NOT NULL\\n ) AS tile;\\n\")", + "code": "ActiveRecord::Base.connection.execute(\"SELECT ST_ASMVT(tile.*, 'layer0', 4096, 'mvtgeometry', 'id') as tile\\n FROM (\\n SELECT id, json_build_object('name', name) as properties, ST_AsMVTGeom(the_geom_webmercator, ST_TileEnvelope(#{Integer(param_z)},#{Integer(param_x)},#{Integer(param_y)}), 4096, 256, true) AS mvtgeometry\\n FROM (\\n SELECT protected_areas.*, st_transform(geometry, 3857) as the_geom_webmercator\\n FROM protected_areas\\n ) as data\\n WHERE ST_AsMVTGeom(the_geom_webmercator, ST_TileEnvelope(#{Integer(param_z)},#{Integer(param_x)},#{Integer(param_y)}),4096,0,true) IS NOT NULL\\n ) AS tile;\\n\")", "render_path": null, "location": { "type": "method", - "class": "Fmu", - "method": "vector_tiles" + "class": "ProtectedAreaVectorTile", + "method": "ProtectedAreaVectorTile.fetch" }, "user_input": "Integer(param_z)", "confidence": "Medium", @@ -191,7 +214,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "app/resources/v1/operator_resource.rb", - "line": 88, + "line": 67, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "records.joins(:country).joins([{ :country => :translations }]).where(\"lower(country_translations.name) like #{sanitized_value}\")", "render_path": null, @@ -435,6 +458,6 @@ "note": "" } ], - "updated": "2023-06-02 14:31:06 +0200", - "brakeman_version": "5.4.1" + "updated": "2023-11-20 15:52:38 +0100", + "brakeman_version": "6.0.0" } diff --git a/config/locales/en.yml b/config/locales/en.yml index 66b600801..0e4413175 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -176,6 +176,8 @@ en: user_deactivated: 'User deactivated' permissions_changed: 'Permissions changed' operator_activated: 'Operator activated' + new_shape: 'New shape' + current_shape: 'Current shape' menu: dashboard: dashboard: 'Dashboard' @@ -213,6 +215,7 @@ en: countries: 'Countries' fmus: 'Fmus' fmu_allocations: 'Fmu allocations' + protected_areas: 'Protected areas' government_sector: government_sector: 'Government sector' required_document_group: 'Required document group' @@ -425,8 +428,8 @@ en: action_label: "%{title} Selected" labels: destroy: "Delete" - fmus: - esri_file_upload: "The zip file must contain at least the following files:\\n + esri_shapefile_zip_input: + hint: "The zip file must contain at least the following files:\\n .shp — shape format; the feature geometry itself {content-type: x-gis/x-shapefile}\\n .shx — shape index format; a positional index of the feature geometry to allow seeking forwards and backwards quickly {content-type: x-gis/x-shapefile}\\n .dbf — attribute format; columnar attributes for each shape, in dBase IV format {content-type: application/octet-stream OR text/plain}\\n @@ -514,6 +517,7 @@ en: operator_document_statistic: Operator document statistic #g paper_trail/version: Paper trail/version #g partner: Partner #g + protected_area: Protected Area #g required_gov_document: Required gov document #g required_gov_document/translation: Required gov document/translation #g required_gov_document_group: Required gov document group #g @@ -1195,6 +1199,13 @@ en: website: Website #g name: Name + protected_area: + country: :activerecord.models.country #g + geojson: Geojson #g + geometry: Geometry #g + name: Name #g + wdpa_pid: WDPA pid #g + required_gov_document: country: :activerecord.models.country #g deleted_at: Deleted at #g diff --git a/config/locales/fr.yml b/config/locales/fr.yml index 249d3c669..5c8373b32 100644 --- a/config/locales/fr.yml +++ b/config/locales/fr.yml @@ -186,6 +186,8 @@ fr: user_deactivated: 'User deactivated' permissions_changed: 'Permissions changed' operator_activated: 'Operator activated' + new_shape: 'Nouvelle forme' + current_shape: 'Forme actuelle' menu: dashboard: dashboard: 'Tableau de bord' @@ -223,6 +225,7 @@ fr: countries: 'Pays' fmus: 'UFA' fmu_allocations: 'Attribution des UFA' + protected_areas: 'Zones protégées' government_sector: government_sector: 'Gouvernement' required_document_group: 'Groupe de documents requis' @@ -434,12 +437,12 @@ fr: action_label: "%{title} les éléments sélectionnés" labels: destroy: "Supprimer" - fmus: - esri_file_upload: "The zip file must contain at least the following files:\\n - .shp — shape format; the feature geometry itself {content-type: x-gis/x-shapefile}\\n - .shx — shape index format; a positional index of the feature geometry to allow seeking forwards and backwards quickly {content-type: x-gis/x-shapefile}\\n - .dbf — attribute format; columnar attributes for each shape, in dBase IV format {content-type: application/octet-stream OR text/plain}\\n - .prj — projection description, using a well-known text representation of coordinate reference systems {content-type: text/plain OR application/text}" + esri_shapefile_zip_input: + hint: "Le fichier zip doit contenir au moins les fichiers suivants:\\n + .shp — format de la forme ; la géométrie de l'entité elle-même {content-type: x-gis/x-shapefile}\\n + .shx — stocke l'index de la géométrie ; un index de position de la géométrie de l'élément pour permettre une recherche rapide vers l'avant et vers l'arrière {content-type: x-gis/x-shapefile}\\n + .dbf — contient les données attributaires relatives aux objets contenus dans le shapefile; attributs en colonnes pour chaque forme, au format dBase IV {content-type: application/octet-stream OR text/plain}\\n + .prj — description de la projection, utilisant une représentation textuelle bien connue des systèmes de référence de coordonnées {content-type: text/plain OR application/text}" globalize: translations: 'Traductions' language: @@ -524,6 +527,7 @@ fr: operator_document_history: Historique des documents de l'opérateur #g operator_document_statistic: Statistique des documents de l'opérateur #g partner: Partenaire #g + protected_area: Aire Protégée required_gov_document: Document gouvernemental requis #g required_gov_document_group: Groupe de documents gouvernementaux requis #g required_operator_document: Document de l'opérateur requis #g @@ -1101,6 +1105,13 @@ fr: website: Site Internet #g name: Nom + protected_area: + country: :activerecord.models.country #g + geojson: Géojson #g + geometry: Géométrie #g + name: Nom #g + wdpa_pid: WDPA pid #g + required_gov_document: country: :activerecord.models.country #g deleted_at: Supprimé le #g diff --git a/config/routes.rb b/config/routes.rb index eac33353e..f585c634c 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -10,7 +10,9 @@ root to: "home#index" - post "admin/fmus/preview" => "admin/fmus#preview" + namespace :admin do + resources :geometry_previews, only: [:create] + end require "sidekiq/web" @@ -82,6 +84,7 @@ resources :fmus, only: [:index, :update] do get "tiles/:z/:x/:y", to: "fmus#tiles", on: :collection end + get "protected_areas/tiles/:z/:x/:y", to: "protected_areas#tiles" resources :imports, only: :create diff --git a/db/migrate/20231020090304_create_protected_areas.rb b/db/migrate/20231020090304_create_protected_areas.rb new file mode 100644 index 000000000..3284c1353 --- /dev/null +++ b/db/migrate/20231020090304_create_protected_areas.rb @@ -0,0 +1,16 @@ +class CreateProtectedAreas < ActiveRecord::Migration[7.0] + def change + create_table :protected_areas do |t| + t.references :country, foreign_key: {on_delete: :cascade}, index: true, null: false + + t.string :name, null: false + t.string :wdpa_pid, null: false + + t.jsonb :geojson, null: false + t.geometry :geometry + t.virtual :centroid, type: :st_point, as: "ST_Centroid(geometry)", stored: true + + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 6870db4ec..a95e4f220 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -770,6 +770,18 @@ t.index ["slug"], name: "index_pages_on_slug", unique: true end + create_table "protected_areas", force: :cascade do |t| + t.bigint "country_id", null: false + t.string "name", null: false + t.string "wdpa_pid", null: false + t.jsonb "geojson", null: false + t.geometry "geometry", limit: {:srid=>0, :type=>"geometry"} + t.virtual "centroid", type: :geometry, limit: {:srid=>0, :type=>"st_point"}, as: "st_centroid(geometry)", stored: true + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["country_id"], name: "index_protected_areas_on_country_id" + end + create_table "required_gov_document_group_translations", id: :serial, force: :cascade do |t| t.integer "required_gov_document_group_id", null: false t.string "locale", null: false @@ -1145,6 +1157,7 @@ add_foreign_key "operator_documents", "required_operator_documents" add_foreign_key "operator_documents", "users", on_delete: :nullify add_foreign_key "operators", "holdings", on_delete: :nullify + add_foreign_key "protected_areas", "countries", on_delete: :cascade add_foreign_key "required_gov_document_groups", "required_gov_document_groups", column: "parent_id" add_foreign_key "required_gov_documents", "countries", on_delete: :cascade add_foreign_key "required_gov_documents", "required_gov_document_groups", on_delete: :cascade diff --git a/lib/tasks/data_import.rake b/lib/tasks/data_import.rake index c5707dff2..e48ca5a09 100644 --- a/lib/tasks/data_import.rake +++ b/lib/tasks/data_import.rake @@ -583,4 +583,37 @@ namespace :import do end end end + + desc "Loads protected areas from GFW data API" + task protected_areas: :environment do + countries = if ENV["COUNTRIES"] + Country.where(iso: ENV["COUNTRIES"].split(",")) + else + Country.active + end + + abort "No counties found" if countries.empty? + + countries_iso_string = countries.pluck(:iso).uniq.map { |iso| "'#{iso}'" }.join(", ") + sql = "SELECT wdpa_pid, name, gfw_geojson, iso3 FROM data where marine = '0' and iso3 IN (#{countries_iso_string})" + + ProtectedArea.where(country: countries).delete_all + ProtectedArea.delete_all if ENV["CLEAR_ALL"] + + response = HTTP.post( + "https://data-api.globalforestwatch.org/dataset/wdpa_protected_areas/v202302/query/json", + json: {sql: sql} + ) + response_json = JSON.parse(response.body) + response_json["data"].each do |protected_area| + country = Country.find_by(iso: protected_area["iso3"]) + ProtectedArea.create!( + name: protected_area["name"], + geojson: protected_area["gfw_geojson"], + wdpa_pid: protected_area["wdpa_pid"], + country: country + ) + puts "Country #{country.name} protected area #{protected_area["name"]} imported" + end + end end diff --git a/spec/factories/protected_areas.rb b/spec/factories/protected_areas.rb new file mode 100644 index 000000000..829d55b02 --- /dev/null +++ b/spec/factories/protected_areas.rb @@ -0,0 +1,65 @@ +# == Schema Information +# +# Table name: protected_areas +# +# id :bigint not null, primary key +# country_id :bigint not null +# name :string not null +# wdpa_pid :string not null +# geojson :jsonb not null +# geometry :geometry geometry, 0 +# centroid :geometry point, 0 +# created_at :datetime not null +# updated_at :datetime not null +# +FactoryBot.define do + factory :protected_area do + country + wdpa_pid { "1232" } + name { "Name" } + geojson { + { + type: "Feature", + geometry: { + type: "Polygon", + coordinates: [ + [ + [ + 18.03955078125, + 51.11041991029264 + ], + [ + 17.42431640625, + 50.331436330838834 + ], + [ + 17.99560546875, + 49.79544988802771 + ], + [ + 19.62158203125, + 49.89463439573421 + ], + [ + 20.0830078125, + 50.42951794712287 + ], + [ + 19.6875, + 51.08282186160978 + ], + [ + 18.91845703125, + 51.31688050404585 + ], + [ + 18.03955078125, + 51.11041991029264 + ] + ] + ] + } + } + } + end +end diff --git a/spec/models/protected_area_spec.rb b/spec/models/protected_area_spec.rb new file mode 100644 index 000000000..170321eeb --- /dev/null +++ b/spec/models/protected_area_spec.rb @@ -0,0 +1,29 @@ +# == Schema Information +# +# Table name: protected_areas +# +# id :bigint not null, primary key +# country_id :bigint not null +# name :string not null +# wdpa_pid :string not null +# geojson :jsonb not null +# geometry :geometry geometry, 0 +# centroid :geometry point, 0 +# created_at :datetime not null +# updated_at :datetime not null +# +require "rails_helper" + +RSpec.describe ProtectedArea, type: :model do + subject(:protected_area) { build(:protected_area) } + + it "is valid with valid attributes" do + expect(protected_area).to be_valid + end + + describe "Validations" do + it { is_expected.to validate_presence_of(:name) } + it { is_expected.to validate_presence_of(:wdpa_pid) } + it { is_expected.to validate_presence_of(:geojson) } + end +end