Merge branch 'feat/observation-multiple-evidences' into staging

app/admin/observation_document.rb

@@ -24,10 +24,8 @@
 
   csv do
     column :id
-    column :observation do |od|
-      od.observation&.id
-    end
     column :name
+    column :document_type
     column :user do |od|
       od.user&.name
     end
@@ -38,8 +36,10 @@
 
   index do
     column :id
-    column :observation, sortable: "observation_id"
+    column :observation_report
+    column :observations
     column :name
+    column :document_type
     column :attachment do |o|
       link_to o&.name, o.attachment&.url if o.attachment&.url
     end
@@ -62,7 +62,10 @@
     end
   end
 
-  filter :observation, as: :select, collection: -> { Observation.joins(:observation_documents).distinct.order(:id).pluck(:id) }
+  filter :observations, as: :select, collection: -> { Observation.joins(:observation_documents).distinct.order(:id).pluck(:id) }
+  filter :observation_report,
+    label: -> { I18n.t("activerecord.models.observation_report") }, as: :select,
+    collection: -> { ObservationReport.where(id: ObservationDocument.select(:observation_report_id)).order(:title) }
   filter :name, as: :select
   filter :attachment, as: :select
   filter :user
@@ -73,9 +76,11 @@
   form do |f|
     f.semantic_errors(*f.object.errors.attribute_names)
     f.inputs do
-      f.input :observation, collection: Observation.all.map { |o| [o.id, o.id] }, input_html: {disabled: true}
+      f.input :observation_report, input_html: {disabled: true}
+      f.input :observations, input_html: {disabled: true}
       f.input :user, input_html: {disabled: true}
       f.input :name
+      f.input :document_type
       f.input :attachment, as: :file, hint: f.object&.attachment&.file&.filename
 
       f.actions
@@ -85,7 +90,9 @@
   show do
     attributes_table do
       row :id
-      row :observation
+      row :observation_report
+      row :observations
+      row :document_type
       row :attachment do |o|
         link_to o&.name, o.attachment&.url if o.attachment&.url
       end
@@ -99,7 +106,7 @@
 
   controller do
     def scoped_collection
-      end_of_association_chain.includes(:user)
+      end_of_association_chain.includes(:user, :observations)
     end
   end
 end

app/importers/observations_importer.rb

@@ -4,7 +4,7 @@
 class ObservationsImporter < FileDataImport::BaseImporter
   PERMITTED_ATTRIBUTES = %i[
     observation_type publication_date pv is_active location_accuracy
-    lat lng fmu_id evidence_type location_information actions_taken
+    lat lng fmu_id location_information actions_taken
     evidence_on_report validation_status is_physical_place user_id
     observer_ids
   ].freeze

app/models/observation.rb

@@ -26,7 +26,6 @@
 #  law_id                :integer
 #  location_information  :string
 #  is_physical_place     :boolean          default(TRUE), not null
-#  evidence_type         :integer
 #  location_accuracy     :integer
 #  evidence_on_report    :string
 #  hidden                :boolean          default(FALSE), not null
@@ -50,18 +49,16 @@ class Observation < ApplicationRecord
   translates :details, :concern_opinion, :litigation_status, touch: true, versioning: :paper_trail, paranoia: true
   active_admin_translates :details, :concern_opinion, :litigation_status
 
+  enum evidence_type: {"No evidence" => 0, "Uploaded documents" => 1, "Evidence presented in the report" => 2}
   enum observation_type: {"operator" => 0, "government" => 1}
   enum validation_status: {"Created" => 0, "Ready for QC" => 1, "QC in progress" => 2, "Approved" => 3,
                            "Rejected" => 4, "Needs revision" => 5, "Ready for publication" => 6,
                            "Published (no comments)" => 7, "Published (not modified)" => 8,
                            "Published (modified)" => 9}
-  enum evidence_type: {"Government Documents" => 0, "Company Documents" => 1, "Photos" => 2,
-                       "Testimony from local communities" => 3, "Other" => 4, "Evidence presented in the report" => 5,
-                       "Maps" => 6}
   enum location_accuracy: {"Estimated location" => 0, "GPS coordinates extracted from photo" => 1,
                            "Accurate GPS coordinates" => 2}
 
-  validate_enum_attributes :observation_type, :evidence_type, :location_accuracy
+  validate_enum_attributes :evidence_type, :observation_type, :location_accuracy
 
   STATUS_TRANSITIONS = {
     monitor: {
@@ -109,7 +106,7 @@ class Observation < ApplicationRecord
   has_many :observation_operators, dependent: :destroy
   has_many :relevant_operators, through: :observation_operators, source: :operator
 
-  has_many :observation_documents, dependent: :destroy
+  has_and_belongs_to_many :observation_documents, inverse_of: :observations
 
   accepts_nested_attributes_for :observation_documents, allow_destroy: true
   accepts_nested_attributes_for :observation_report, allow_destroy: true
@@ -127,7 +124,7 @@ class Observation < ApplicationRecord
 
   validates :lat, numericality: {greater_than_or_equal_to: -90, less_than_or_equal_to: 90, allow_blank: true}
   validates :lng, numericality: {greater_than_or_equal_to: -180, less_than_or_equal_to: 180, allow_blank: true}
-  validate :evidence_presented_in_the_report
+  validates :evidence_on_report, presence: true, if: -> { evidence_type == "Evidence presented in the report" }
   validate :status_changes, if: -> { user_type.present? }
 
   validates :observers, presence: true
@@ -137,6 +134,7 @@ class Observation < ApplicationRecord
   validates :admin_comment, presence: true, if: -> { validation_status == "Needs revision" }
 
   before_validation :assign_observers_from_report, if: :observation_report_changed?
+  before_validation :nullify_evidence_on_report, if: -> { evidence_type != "Evidence presented in the report" }
 
   before_save :set_active_status
   before_save :nullify_fmu_and_coordinates, unless: :is_physical_place
@@ -152,7 +150,7 @@ class Observation < ApplicationRecord
 
   after_save :create_history, if: :saved_changes?
 
-  after_save :remove_documents, if: -> { evidence_type == "Evidence presented in the report" }
+  after_save :remove_documents, if: -> { evidence_type != "Uploaded documents" }
   after_save :update_fmu_geojson
 
   after_commit :notify_about_creation, on: :create
@@ -221,6 +219,10 @@ def published?
 
   private
 
+  def nullify_evidence_on_report
+    self.evidence_on_report = nil
+  end
+
   def nullify_fmu_and_coordinates
     self.lat = nil
     self.lng = nil
@@ -296,17 +298,8 @@ def status_changes
       "Invalid validation change for #{@user_type}. Can't move from '#{validation_status_was}' to '#{validation_status}'")
   end
 
-  def evidence_presented_in_the_report
-    if evidence_type == "Evidence presented in the report" && evidence_on_report.blank?
-      errors.add(:evidence_on_report, "You must add information on where to find the evidence on the report")
-    end
-    if evidence_type != "Evidence presented in the report" && evidence_on_report.present?
-      errors.add(:evidence_on_report, "This field can only be present when the evidence is presented on the report")
-    end
-  end
-
   def remove_documents
-    ObservationDocument.where(observation_id: id).destroy_all
+    self.observation_documents = []
   end
 
   # If the observation is for an fmu, it updates its geojson with the new count

app/models/observation_document.rb

@@ -16,14 +16,20 @@
 
 class ObservationDocument < ApplicationRecord
   has_paper_trail
+  acts_as_paranoid
   mount_base64_uploader :attachment, ObservationDocumentUploader
   include MoveableAttachment
 
-  acts_as_paranoid
+  enum document_type: {
+    "Government Documents" => 0, "Company Documents" => 1, "Photos" => 2,
+    "Testimony from local communities" => 3, "Other" => 4, "Maps" => 5
+  }
+  validate_enum_attributes :document_type
 
   # TODO: only nils in the database, maybe that should be removed?
   belongs_to :user, inverse_of: :observation_documents, touch: true, optional: true
-  belongs_to :observation, inverse_of: :observation_documents, touch: true
+  belongs_to :observation_report, inverse_of: :observation_documents, touch: true, optional: true
+  has_and_belongs_to_many :observations, inverse_of: :observation_documents
 
   skip_callback :commit, :after, :remove_attachment!
   after_destroy :move_attachment_to_private_directory

app/models/observation_history.rb

@@ -33,9 +33,7 @@ class ObservationHistory < ApplicationRecord
                            "Rejected" => 4, "Needs revision" => 5, "Ready for publication" => 6,
                            "Published (no comments)" => 7, "Published (not modified)" => 8,
                            "Published (modified)" => 9}
-  enum evidence_type: {"Government Documents" => 0, "Company Documents" => 1, "Photos" => 2,
-                       "Testimony from local communities" => 3, "Other" => 4, "Evidence presented in the report" => 5,
-                       "Maps" => 6}
+  enum evidence_type: {"No evidence" => 0, "Uploaded documents" => 1, "Evidence presented in the report" => 2}
   enum location_accuracy: {"Estimated location" => 0, "GPS coordinates extracted from photo" => 1,
                            "Accurate GPS coordinates" => 2}
   enum fmu_forest_type: ForestType::TYPES_WITH_CODE

app/models/observation_report.rb

@@ -27,6 +27,7 @@ class ObservationReport < ApplicationRecord
   has_many :observation_report_observers, dependent: :destroy
   has_many :observers, through: :observation_report_observers, after_add: :self_touch, after_remove: :self_touch
   has_many :observations, dependent: :destroy
+  has_many :observation_documents, inverse_of: :observation_report, dependent: :destroy
 
   skip_callback :commit, :after, :remove_attachment!
   after_destroy :move_attachment_to_private_directory

app/resources/v1/observation_document_resource.rb

@@ -4,10 +4,15 @@ module V1
   class ObservationDocumentResource < BaseResource
     caching
 
-    attributes :name, :attachment, :created_at, :updated_at
+    attributes :name, :attachment, :document_type, :created_at, :updated_at
 
-    has_one :observation
+    has_many :observations
+    has_one :observation_report
 
-    filters :observation_id, :name
+    filters :observation_report_id, :name
+
+    filter :observation_id, apply: ->(records, value, _options) {
+      records.where(id: ObservationDocument.joins(:observations).where(observations: value))
+    }
   end
 end

app/uploaders/observation_document_uploader.rb

@@ -14,7 +14,7 @@ def filename
     else
       [
         model.id,
-        (model.observation.evidence_type || "other").parameterize
+        (model.document_type || "other").parameterize
       ].join("-")
     end
 

app/views/active_admin/resource/_attributes_table.html.arb

@@ -27,6 +27,8 @@ panel I18n.t("active_admin.observations_page.details") do
         render partial: "map", locals: {center: [r.lng, r.lat], geojson: r.fmu&.geojson, bbox: r.fmu&.bbox}
       end
     end
+    row :evidence_type
+    row :evidence_on_report
     row :actions_taken
     row :concern_opinion
     row :observation_report

config/environments/development.rb

@@ -65,4 +65,6 @@
 
   # Uncomment if you wish to allow Action Cable access from any origin.
   # config.action_cable.disable_request_forgery_protection = true
+
+  # config.active_record.migration_error = false
 end

db/migrate/20231222154005_create_observation_document_observations.rb

@@ -0,0 +1,34 @@
+class CreateObservationDocumentObservations < ActiveRecord::Migration[7.0]
+  def up
+    create_table :observation_documents_observations do |t|
+      t.belongs_to :observation_document, foreign_key: {on_delete: :cascade}, index: {name: "observation_documents_observations_doc_index"}, null: false
+      t.belongs_to :observation, foreign_key: {on_delete: :cascade}, index: {name: "observation_documents_observations_obs_index"}, null: false
+      t.index [:observation_document_id, :observation_id], unique: true, name: "observation_documents_observations_double_index"
+      t.timestamps
+    end
+
+    query = <<-SQL
+      INSERT INTO observation_documents_observations (observation_document_id, observation_id, created_at, updated_at)
+      SELECT
+        od.id, o.id, NOW(), NOW()
+      FROM
+        observation_documents od JOIN observations o ON od.observation_id = o.id
+    SQL
+    execute(query)
+
+    remove_reference :observation_documents, :observation
+  end
+
+  def down
+    add_reference :observation_documents, :observation, foreign_key: true, index: true
+
+    query = <<-SQL
+      UPDATE observation_documents SET observation_id = odo.observation_id
+      FROM (SELECT DISTINCT ON (observation_document_id) observation_document_id, observation_id FROM observation_documents_observations) odo
+      WHERE observation_documents.id = odo.observation_document_id
+    SQL
+    execute(query)
+
+    drop_table :observation_documents_observations
+  end
+end

db/migrate/20231222171303_add_document_type_to_observation_document.rb

@@ -0,0 +1,79 @@
+class AddDocumentTypeToObservationDocument < ActiveRecord::Migration[7.0]
+  def up
+    add_column :observation_documents, :document_type, :integer, default: 0, null: false
+    # 5 is evidence on report in observations table
+    # changing Maps from 6 to 5
+    query = <<~SQL
+      UPDATE observation_documents SET document_type = odo.evidence_type, updated_at = NOW()
+      FROM (
+        SELECT DISTINCT ON (observation_document_id) observation_document_id, evidence_type
+        FROM observation_documents_observations
+        INNER JOIN observations ON observations.id = observation_documents_observations.observation_id
+        WHERE evidence_type <> 5 AND evidence_type IS NOT NULL
+      ) odo
+      WHERE observation_documents.id = odo.observation_document_id
+    SQL
+    execute(query)
+    execute("UPDATE observation_documents SET document_type = 5 WHERE document_type = 6")
+    execute("UPDATE observations SET evidence_type = 0 WHERE evidence_type <> 5") # set no evidence for all except evidence on report
+    execute("UPDATE observations SET evidence_type = 2 WHERE evidence_type = 5") # update for evidence on report
+    execute(
+      <<~SQL
+        UPDATE observations SET evidence_type = 1 WHERE EXISTS (
+          SELECT * FROM observation_documents_observations odo
+            INNER JOIN observation_documents od ON od.id = odo.observation_document_id
+          WHERE observation_id = observations.id AND od.deleted_at IS NULL
+        )
+      SQL
+    ) # update for linked with non deleted documents
+
+    # Testing data after migration
+    # I was testing migration correctness with those queries so I will leave them here
+    result = execute(
+      "SELECT id FROM observations WHERE evidence_type <> 2 AND coalesce(evidence_on_report, '') <> ''"
+    )
+    raise "Mismatched type when evidence_on_report present" if result.count.positive?
+    result = execute(
+      <<~SQL
+        SELECT id FROM observations WHERE evidence_type <> 1 AND EXISTS (
+          SELECT * FROM
+            observation_documents_observations odo
+          INNER JOIN observation_documents od on od.id = odo.observation_document_id
+          WHERE odo.observation_id = observations.id AND od.deleted_at IS NULL
+        )
+      SQL
+    )
+    raise "Documents exists for observations with no linked documents" if result.count.positive?
+    result = execute(
+      <<~SQL
+        SELECT id FROM observations WHERE evidence_type = 1 AND NOT EXISTS (
+          SELECT * FROM
+            observation_documents_observations odo
+          INNER JOIN observation_documents od on od.id = odo.observation_document_id
+          WHERE odo.observation_id = observations.id AND od.deleted_at IS NULL
+        )
+      SQL
+    )
+    raise "No documents when evidence type is 1: #{result.values.flatten}" if result.count.positive?
+    result = execute(
+      "SELECT id FROM observations WHERE evidence_type = 0 AND validation_status <> 0"
+    )
+    Rails.logger.debug "No evidence for non created (non drafted) observations #{result.values.flatten}" if result.count.positive?
+  end
+
+  def down
+    query = <<~SQL
+      UPDATE observations SET evidence_type = odo.document_type, updated_at = NOW()
+      FROM (
+        SELECT DISTINCT ON (observation_id) observation_id, document_type
+        FROM observation_documents_observations
+        INNER JOIN observation_documents ON observation_documents.id = observation_documents_observations.observation_document_id
+      ) odo
+      WHERE observations.id = odo.observation_id
+    SQL
+    execute(query)
+    execute("UPDATE observations SET evidence_type = 6 WHERE evidence_type = 5")
+    execute("UPDATE observations SET evidence_type = 5 WHERE evidence_type is null and coalesce(evidence_on_report, '') <> ''")
+    remove_column :observation_documents, :document_type
+  end
+end

db/migrate/20240105172847_add_observation_report_to_observation_document.rb

@@ -0,0 +1,18 @@
+class AddObservationReportToObservationDocument < ActiveRecord::Migration[7.0]
+  def change
+    add_reference :observation_documents, :observation_report, foreign_key: true, index: true
+
+    reversible do |dir|
+      dir.up do
+        execute(
+          <<~SQL
+            UPDATE observation_documents SET observation_report_id = o.observation_report_id
+            FROM observations o
+            INNER JOIN observation_documents_observations odo ON odo.observation_id = o.id
+            WHERE odo.observation_document_id = observation_documents.id
+          SQL
+        )
+      end
+    end
+  end
+end