Merge pull request #441 from lega0208/feedback-relevance

Feedback relevance

.eslintrc.json

@@ -25,7 +25,8 @@
               }
             ]
           }
-        ]
+        ],
+        "@nx/dependency-checks": "warn"
       }
     },
     {
@@ -37,7 +38,8 @@
       "rules": {
         "@typescript-eslint/no-explicit-any": "warn",
         "@typescript-eslint/no-unused-vars": "warn",
-        "@typescript-eslint/ban-ts-comment": "warn"
+        "@typescript-eslint/ban-ts-comment": "warn",
+        "@nx/dependency-checks": "warn"
       }
     },
     {

apps/api/src/app/app.module.ts

@@ -11,6 +11,7 @@ import { environment } from '../environments/environment';
 import { DbModule } from '@dua-upd/db';
 import { InternalSearchModule } from '../internal-search/internal-search.module';
 import { ReportsModule } from '../reports/reports.module';
+import { FeedbackModule } from '@dua-upd/api/feedback';
 
 @Module({
   imports: [
@@ -23,6 +24,7 @@ import { ReportsModule } from '../reports/reports.module';
     ReportsModule,
     CustomReportsModule.register(environment.production),
     QueryModule,
+    FeedbackModule,
     BullModule.forRoot({
       connection: {
         host: environment.redisHost,

apps/api/src/overall/overall.controller.ts

@@ -1,4 +1,4 @@
-import { Controller, Get, Query } from '@nestjs/common';
+import { Controller, Get, ParseBoolPipe, Query } from '@nestjs/common';
 import { OverallService } from './overall.service';
 
 @Controller('overall')
@@ -9,7 +9,16 @@ export class OverallController {
   getMetrics(
     @Query('dateRange') dateRange: string,
     @Query('comparisonDateRange') comparisonDateRange: string,
+    @Query('ipd', ParseBoolPipe) ipd: boolean,
   ) {
-    return this.overallService.getMetrics({ dateRange, comparisonDateRange });
+    if (ipd) {
+      console.log('IPD MODE ACTIVATED 🤖');
+    }
+
+    return this.overallService.getMetrics({
+      dateRange,
+      comparisonDateRange,
+      ipd,
+    });
   }
 }

apps/api/src/overall/overall.module.ts

@@ -4,9 +4,10 @@ import { OverallService } from './overall.service';
 import { OverallController } from './overall.controller';
 import { DbModule, DbService } from '@dua-upd/db';
 import { hours } from '@dua-upd/utils-common';
+import { FeedbackModule } from '@dua-upd/api/feedback';
 
 @Module({
-  imports: [CacheModule.register({ ttl: hours(12) }), DbModule],
+  imports: [CacheModule.register({ ttl: hours(12) }), DbModule, FeedbackModule],
   controllers: [OverallController],
   providers: [OverallService, DbService],
 })

apps/api/src/overall/overall.service.ts

@@ -12,12 +12,12 @@ import type {
   ProjectDocument,
   TaskDocument,
   UxTestDocument,
-  FeedbackDocument,
   PageMetricsModel,
   PageDocument,
   SearchAssessmentDocument,
   AnnotationsDocument,
   GcTasksDocument,
+  FeedbackModel,
 } from '@dua-upd/db';
 import {
   DbService,
@@ -50,7 +50,10 @@ import {
   dateRangeSplit,
   getImprovedKpiSuccessRates,
   getLatestTestData,
+  parseDateRangeString,
+  percentChange,
 } from '@dua-upd/utils-common';
+import { FeedbackService } from '@dua-upd/api/feedback';
 
 dayjs.extend(utc);
 dayjs.extend(quarterOfYear);
@@ -117,19 +120,21 @@ export class OverallService {
     @InjectModel(CallDriver.name, 'defaultConnection')
     private calldriversModel: CallDriverModel,
     @InjectModel(Feedback.name, 'defaultConnection')
-    private feedbackModel: Model<FeedbackDocument>,
+    private feedbackModel: FeedbackModel,
     @InjectModel(SearchAssessment.name, 'defaultConnection')
     private searchAssessmentModel: Model<SearchAssessmentDocument>,
     @InjectModel(GcTasks.name, 'defaultConnection')
     private gcTasksModel: Model<GcTasksDocument>,
     @Inject(CACHE_MANAGER) private cacheManager: Cache,
     @InjectModel(Annotations.name, 'defaultConnection')
     private annotationsModel: Model<AnnotationsDocument>,
+    private feedbackService: FeedbackService,
   ) {}
 
   @AsyncLogTiming
   async getMetrics(params: ApiParams): Promise<OverviewData> {
-    const cacheKey = `OverviewMetrics-${params.dateRange}`;
+    const cacheKey = `OverviewMetrics-${params.dateRange}-${params['ipd']}`;
+
     const cachedData =
       await this.cacheManager.store.get<OverviewData>(cacheKey);
 
@@ -185,6 +190,34 @@ export class OverallService {
 
     const improvedTasksKpi = getImprovedKpiSuccessRates(uxTests);
 
+    const mostRelevantCommentsAndWords =
+      await this.feedbackService.getMostRelevantCommentsAndWords({
+        dateRange: parseDateRangeString(params.dateRange),
+        ipd: params.ipd as boolean,
+      });
+
+    const numComments =
+      mostRelevantCommentsAndWords.en.comments.length +
+      mostRelevantCommentsAndWords.fr.comments.length;
+
+    const { start: prevDateRangeStart, end: prevDateRangeEnd } =
+      parseDateRangeString(params.comparisonDateRange);
+
+    const numPreviousComments = await this.feedbackModel
+      .countDocuments({
+        date: { $gte: prevDateRangeStart, $lte: prevDateRangeEnd },
+      })
+      .exec();
+
+    const numCommentsPercentChange = !params.ipd && numPreviousComments
+      ? percentChange(numComments, numPreviousComments)
+      : null;
+
+    const commentsByPage = await this.feedbackModel.getCommentsByPageWithComparison(
+      params.dateRange,
+      params.comparisonDateRange
+    );
+
     const results = {
       dateRange: params.dateRange,
       comparisonDateRange: params.comparisonDateRange,
@@ -220,13 +253,17 @@ export class OverallService {
       ),
       projects: await getProjects(this.projectModel, this.uxTestModel),
       uxTests,
-      improvedTasksKpi,
+      improvedTasksKpi: getImprovedKpiSuccessRates(uxTests),
       ...(await getUxData(testsSince2018)),
       calldriverTopics,
       top5IncreasedCalldriverTopics,
       top5DecreasedCalldriverTopics,
       searchTermsEn: await this.getTopSearchTerms(params, 'en'),
       searchTermsFr: await this.getTopSearchTerms(params, 'fr'),
+      mostRelevantCommentsAndWords,
+      numComments,
+      numCommentsPercentChange,
+      commentsByPage,
     };
 
     await this.cacheManager.set(cacheKey, results);
@@ -712,7 +749,7 @@ async function getOverviewMetrics(
   overallModel: Model<OverallDocument>,
   PageMetricsModel: PageMetricsModel,
   calldriversModel: CallDriverModel,
-  feedbackModel: Model<FeedbackDocument>,
+  feedbackModel: FeedbackModel,
   pageModel: Model<PageDocument>,
   annotationsModel: Model<AnnotationsDocument>,
   db: DbService,
@@ -824,83 +861,6 @@ async function getOverviewMetrics(
     .limit(10)
     .exec();
 
-  const totalFeedback = await feedbackModel
-    .aggregate()
-    .project({
-      date: 1,
-      url: 1,
-      main_section: 1,
-    })
-    .sort({ date: 1 })
-    .match({
-      $and: [
-        {
-          date: dateQuery,
-        },
-        {
-          url: {
-            $regex:
-              '/en/revenue-agency|/fr/agence-revenu|/en/services/taxes|/fr/services/impots',
-          },
-        },
-      ],
-    })
-    .group({
-      _id: '$main_section',
-      sum: { $sum: 1 },
-    })
-    .sort({ sum: -1 })
-    .project({
-      _id: 0,
-      main_section: '$_id',
-      sum: 1,
-    })
-    .exec();
-
-  const feedbackPages = await feedbackModel
-    .aggregate<{ _id: string; title: string; url: string; sum: number }>()
-    .project({
-      date: 1,
-      url: 1,
-    })
-    .match({
-      $and: [
-        { date: dateQuery },
-        // todo: remove url filter once there is logic in place to remove non-CRA pages from feedback collection
-        {
-          url: {
-            $regex:
-              '/en/revenue-agency|/fr/agence-revenu|/en/services/taxes|/fr/services/impots',
-          },
-        },
-      ],
-    })
-    .group({
-      _id: '$url',
-      sum: { $sum: 1 },
-    })
-    .project({
-      _id: 0,
-      url: '$_id',
-      sum: 1,
-    })
-    .sort({ sum: -1 })
-    .lookup({
-      from: 'pages',
-      localField: 'url',
-      foreignField: 'url',
-      as: 'page',
-    })
-    .unwind('$page')
-    .addFields({
-      _id: '$page._id',
-      title: '$page.title',
-    })
-    .project({
-      page: 0,
-    })
-    .exec();
-
   const aggregatedMetrics = await overallModel
     .aggregate<{
       visitors: number;
@@ -1007,9 +967,21 @@ async function getOverviewMetrics(
     .group({
       _id: { gc_task: '$gc_task', theme: '$theme' },
       total_entries: { $sum: 1 },
-      satisfaction: { $avg: { $cond: [{ $in: ['$satisfaction', ['Very satisfied', 'Satisfied']] }, 1, 0] } },
-      ease: { $avg: { $cond: [{ $in: ['$ease', ['Very easy', 'Easy']] }, 1, 0] } },
-      able_to_complete: { $avg: { $cond: [{ $eq: ['$able_to_complete', 'Yes'] }, 1, 0] } },
+      satisfaction: {
+        $avg: {
+          $cond: [
+            { $in: ['$satisfaction', ['Very satisfied', 'Satisfied']] },
+            1,
+            0,
+          ],
+        },
+      },
+      ease: {
+        $avg: { $cond: [{ $in: ['$ease', ['Very easy', 'Easy']] }, 1, 0] },
+      },
+      able_to_complete: {
+        $avg: { $cond: [{ $eq: ['$able_to_complete', 'Yes'] }, 1, 0] },
+      },
     })
     .project({
       gc_task: '$_id.gc_task',
@@ -1019,17 +991,63 @@ async function getOverviewMetrics(
       ease: 1,
       able_to_complete: 1,
       margin_of_error: {
-            $divide: [
+        $divide: [
+          {
+            $add: [
               {
-                $add: [
-                  { $multiply: [1.96, { $sqrt: { $divide: [{ $multiply: ['$satisfaction', { $subtract: [1, '$satisfaction'] }] }, '$total_entries'] } }] },
-                  { $multiply: [1.96, { $sqrt: { $divide: [{ $multiply: ['$ease', { $subtract: [1, '$ease'] }] }, '$total_entries'] } }] },
-                  { $multiply: [1.96, { $sqrt: { $divide: [{ $multiply: ['$able_to_complete', { $subtract: [1, '$able_to_complete'] }] }, '$total_entries'] } }] },
-                ]
+                $multiply: [
+                  1.96,
+                  {
+                    $sqrt: {
+                      $divide: [
+                        {
+                          $multiply: [
+                            '$satisfaction',
+                            { $subtract: [1, '$satisfaction'] },
+                          ],
+                        },
+                        '$total_entries',
+                      ],
+                    },
+                  },
+                ],
               },
-              3
-            ]
-          }
+              {
+                $multiply: [
+                  1.96,
+                  {
+                    $sqrt: {
+                      $divide: [
+                        { $multiply: ['$ease', { $subtract: [1, '$ease'] }] },
+                        '$total_entries',
+                      ],
+                    },
+                  },
+                ],
+              },
+              {
+                $multiply: [
+                  1.96,
+                  {
+                    $sqrt: {
+                      $divide: [
+                        {
+                          $multiply: [
+                            '$able_to_complete',
+                            { $subtract: [1, '$able_to_complete'] },
+                          ],
+                        },
+                        '$total_entries',
+                      ],
+                    },
+                  },
+                ],
+              },
+            ],
+          },
+          3,
+        ],
+      },
     })
     .sort({ total_entries: -1 })
     .project({ _id: 0 });
@@ -1041,10 +1059,8 @@ async function getOverviewMetrics(
     calldriversEnquiry,
     searchAssessmentData,
     ...aggregatedMetrics[0],
-    totalFeedback,
     topPagesVisited,
     top10GSC,
-    feedbackPages,
     annotations,
     gcTasksData,
     gcTasksComments,

apps/api/src/pages/pages.module.ts

@@ -4,9 +4,10 @@ import { PagesService } from './pages.service';
 import { PagesController } from './pages.controller';
 import { DbModule, DbService } from '@dua-upd/db';
 import { hours } from '@dua-upd/utils-common';
+import { FeedbackModule } from '@dua-upd/api/feedback';
 
 @Module({
-  imports: [CacheModule.register({ ttl: hours(12) }), DbModule],
+  imports: [CacheModule.register({ ttl: hours(12) }), DbModule, FeedbackModule],
   controllers: [PagesController],
   providers: [PagesService, DbService],
 })