pkp/pkp-lib#9822 Increase timeout and improve DB table indexes for us…

…age stats jobs

classes/migration/install/MetricsMigration.php

@@ -320,7 +320,7 @@ public function up(): void
             $table->string('city', 255)->default('');
             $table->string('load_id', 50);
 
-            $table->index(['load_id', 'context_id', 'ip'], 'ust_load_id_context_id_ip');
+            $table->index(['load_id', 'context_id', 'ip', 'user_agent', 'canonical_url'], 'ust_load_id_context_id_ip_ua_url');
         });
 
         // Usage stats unique book and chapter item investigations temporary records
@@ -361,7 +361,7 @@ public function up(): void
             $table->string('city', 255)->default('');
             $table->string('load_id', 50);
 
-            $table->index(['load_id', 'context_id', 'ip'], 'usii_load_id_context_id_ip');
+            $table->index(['load_id', 'context_id', 'ip', 'user_agent'], 'usii_load_id_context_id_ip_ua');
         });
 
         // Usage stats unique book and chapter item requests temporary records
@@ -402,7 +402,7 @@ public function up(): void
             $table->string('city', 255)->default('');
             $table->string('load_id', 50);
 
-            $table->index(['load_id', 'context_id', 'ip'], 'usir_load_id_context_id_ip');
+            $table->index(['load_id', 'context_id', 'ip', 'user_agent'], 'usir_load_id_context_id_ip_ua');
         });
 
         // Usage stats unique title investigations temporary records
@@ -442,7 +442,7 @@ public function up(): void
             $table->string('city', 255)->default('');
             $table->string('load_id', 50);
 
-            $table->index(['load_id', 'context_id', 'ip'], 'usti_load_id_context_id_ip');
+            $table->index(['load_id', 'context_id', 'ip', 'user_agent'], 'usti_load_id_context_id_ip_ua');
         });
 
         // Usage stats unique title requests temporary records
@@ -483,7 +483,7 @@ public function up(): void
             $table->string('city', 255)->default('');
             $table->string('load_id', 50);
 
-            $table->index(['load_id', 'context_id', 'ip'], 'ustr_load_id_context_id_ip');
+            $table->index(['load_id', 'context_id', 'ip', 'user_agent'], 'ustr_load_id_context_id_ip_ua');
         });
 
         // Usage stats institution temporary records

classes/migration/upgrade/v3_4_0/I9822_ChangeUsageStatsTemporaryTablesIndexes.php

@@ -0,0 +1,59 @@
+<?php
+
+/**
+ * @file classes/migration/upgrade/v3_4_0/I9822_ChangeUsageStatsTemporaryTablesIndexes.php
+ *
+ * Copyright (c) 2025 Simon Fraser University
+ * Copyright (c) 2025 John Willinsky
+ * Distributed under the GNU GPL v3. For full terms see the file docs/COPYING.
+ *
+ * @class I9822_ChangeUsageStatsTemporaryTablesIndexes
+ *
+ * @brief Consider aditional columns user_agent and canonical_url for the index on temporary usage stats tables to fix/improve the removeDoubleClicks and compileUniqueClicks query.
+ */
+
+namespace APP\migration\upgrade\v3_4_0;
+
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+use PKP\install\DowngradeNotSupportedException;
+
+class I9822_ChangeUsageStatsTemporaryTablesIndexes extends \PKP\migration\upgrade\v3_4_0\I9822_ChangeUsageStatsTemporaryTablesIndexes
+{
+    /**
+     * Run the migration.
+     */
+    public function up(): void
+    {
+        $sm = Schema::getConnection()->getDoctrineSchemaManager();
+        Schema::table('usage_stats_unique_title_investigations_temporary_records', function (Blueprint $table) use ($sm) {
+            $indexesFound = $sm->listTableIndexes('usage_stats_unique_title_investigations_temporary_records');
+            if (array_key_exists('usti_load_id_context_id_ip', $indexesFound)) {
+                $table->dropIndex('usti_load_id_context_id_ip');
+            }
+            if (!array_key_exists('usti_load_id_context_id_ip_ua', $indexesFound)) {
+                $table->index(['load_id', 'context_id', 'ip', 'user_agent'], 'usti_load_id_context_id_ip_ua');
+            }
+        });
+        Schema::table('usage_stats_unique_title_requests_temporary_records', function (Blueprint $table) use ($sm) {
+            $indexesFound = $sm->listTableIndexes('usage_stats_unique_title_requests_temporary_records');
+            if (array_key_exists('ustr_load_id_context_id_ip', $indexesFound)) {
+                $table->dropIndex('ustr_load_id_context_id_ip');
+            }
+            if (!array_key_exists('ustr_load_id_context_id_ip_ua', $indexesFound)) {
+                $table->index(['load_id', 'context_id', 'ip', 'user_agent'], 'ustr_load_id_context_id_ip_ua');
+            }
+        });
+        parent::up();
+    }
+
+    /**
+     * Reverse the downgrades
+     *
+     * @throws DowngradeNotSupportedException
+     */
+    public function down(): void
+    {
+        throw new DowngradeNotSupportedException();
+    }
+}

dbscripts/xml/upgrade.xml

@@ -114,6 +114,7 @@
 	<upgrade minversion="3.4.0.0" maxversion="3.4.0.5">
 		<!-- Only new 3.4 installations need this fix -->
 		<migration class="PKP\migration\upgrade\v3_4_0\I8592_SiteNotificationSubscriptions"/>
+		<migration class="APP\migration\upgrade\v3_4_0\I9822_ChangeUsageStatsTemporaryTablesIndexes"/>
 	</upgrade>
 
 	<upgrade minversion="3.1.0.0" maxversion="3.4.9.9">

jobs/statistics/CompileUniqueInvestigations.php

@@ -23,6 +23,8 @@
 
 class CompileUniqueInvestigations extends BaseJob
 {
+    public int $timeout = 600;
+
     /**
      * Create a new job instance.
      *

jobs/statistics/CompileUniqueRequests.php

@@ -23,6 +23,8 @@
 
 class CompileUniqueRequests extends BaseJob
 {
+    public int $timeout = 600;
+
     /**
      * Create a new job instance.
      *