From 4451b047626411b3cb26e59135911f4be0969345 Mon Sep 17 00:00:00 2001 From: sophiamersmann Date: Fri, 26 Jan 2024 09:12:16 +0000 Subject: [PATCH] :hammer: (db) migrate Pageview to knex --- adminSiteServer/apiRouter.ts | 13 +++++++--- baker/algolia/indexChartsToAlgolia.ts | 4 +-- baker/algolia/indexExplorersToAlgolia.ts | 4 +-- baker/algolia/indexToAlgolia.tsx | 4 +-- db/model/Pageview.ts | 32 ++++++++++++++++++------ 5 files changed, 39 insertions(+), 18 deletions(-) diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts index 99f8116509b..b21edbd9814 100644 --- a/adminSiteServer/apiRouter.ts +++ b/adminSiteServer/apiRouter.ts @@ -58,6 +58,7 @@ import { DbRawVariable, DbRawOrigin, parseOriginsRow, + AnalyticsPageviewsTableName, } from "@ourworldindata/types" import { getVariableDataRoute, @@ -67,7 +68,6 @@ import { getDatasetById, setTagsForDataset } from "../db/model/Dataset.js" import { User } from "../db/model/User.js" import { GdocPost } from "../db/model/Gdoc/GdocPost.js" import { GdocBase, Tag as TagEntity } from "../db/model/Gdoc/GdocBase.js" -import { Pageview } from "../db/model/Pageview.js" import { syncDatasetToGitRepo, removeDatasetFromGitRepo, @@ -480,9 +480,14 @@ apiRouter.get( ).then((chart) => chart?.config?.slug) if (!slug) return {} - const pageviewsByUrl = await Pageview.findOneBy({ - url: `https://ourworldindata.org/grapher/${slug}`, - }) + const pageviewsByUrl = await db.knexRawFirst( + "select * from ?? where url = ?", + db.knexInstance(), + [ + AnalyticsPageviewsTableName, + `https://ourworldindata.org/grapher/${slug}`, + ] + ) return { pageviews: pageviewsByUrl ?? undefined, diff --git a/baker/algolia/indexChartsToAlgolia.ts b/baker/algolia/indexChartsToAlgolia.ts index 4f56c4941ae..999d9459e62 100644 --- a/baker/algolia/indexChartsToAlgolia.ts +++ b/baker/algolia/indexChartsToAlgolia.ts @@ -5,7 +5,7 @@ import { isPathRedirectedToExplorer } from "../../explorerAdminServer/ExplorerRe import { ChartRecord, SearchIndexName } from "../../site/search/searchTypes.js" import { KeyChartLevel, OwidGdocLinkType, isNil } from "@ourworldindata/utils" import { MarkdownTextWrap } from "@ourworldindata/components" -import { Pageview } from "../../db/model/Pageview.js" +import { getAnalyticsPageviewsByUrlObj } from "../../db/model/Pageview.js" import { Link } from "../../db/model/Link.js" import { getRelatedArticles } from "../../db/model/Post.js" import { Knex } from "knex" @@ -59,7 +59,7 @@ const getChartsRecords = async ( ) } - const pageviews = await Pageview.getViewsByUrlObj() + const pageviews = await getAnalyticsPageviewsByUrlObj(knex) const records: ChartRecord[] = [] for (const c of chartsToIndex) { diff --git a/baker/algolia/indexExplorersToAlgolia.ts b/baker/algolia/indexExplorersToAlgolia.ts index cab27ac9de9..f1987b3c8b2 100644 --- a/baker/algolia/indexExplorersToAlgolia.ts +++ b/baker/algolia/indexExplorersToAlgolia.ts @@ -9,7 +9,7 @@ import { import { getAlgoliaClient } from "./configureAlgolia.js" import * as db from "../../db/db.js" import { ALGOLIA_INDEXING } from "../../settings/serverSettings.js" -import { Pageview } from "../../db/model/Pageview.js" +import { getAnalyticsPageviewsByUrlObj } from "../../db/model/Pageview.js" import { chunkParagraphs } from "../chunk.js" import { SearchIndexName } from "../../site/search/searchTypes.js" import { Chart } from "../../db/model/Chart.js" @@ -112,7 +112,7 @@ function getNullishJSONValueAsPlaintext(value: string): string { } const getExplorerRecords = async (): Promise => { - const pageviews = await Pageview.getViewsByUrlObj() + const pageviews = await getAnalyticsPageviewsByUrlObj(db.knexInstance()) // Fetch info about all charts used in explorers, as linked by the explorer_charts table const graphersUsedInExplorers = await db diff --git a/baker/algolia/indexToAlgolia.tsx b/baker/algolia/indexToAlgolia.tsx index becde364d59..3c59758cfe2 100644 --- a/baker/algolia/indexToAlgolia.tsx +++ b/baker/algolia/indexToAlgolia.tsx @@ -22,7 +22,7 @@ import { PageType, SearchIndexName, } from "../../site/search/searchTypes.js" -import { Pageview } from "../../db/model/Pageview.js" +import { getAnalyticsPageviewsByUrlObj } from "../../db/model/Pageview.js" import { GdocPost } from "../../db/model/Gdoc/GdocPost.js" import { ArticleBlocks } from "../../site/gdocs/components/ArticleBlocks.js" import React from "react" @@ -196,7 +196,7 @@ function generateGdocRecords( // Generate records for countries, WP posts (not including posts that have been succeeded by Gdocs equivalents), and Gdocs const getPagesRecords = async (knex: Knex) => { - const pageviews = await Pageview.getViewsByUrlObj() + const pageviews = await getAnalyticsPageviewsByUrlObj(db.knexInstance()) const gdocs = await GdocPost.getPublishedGdocs() const publishedGdocsBySlug = keyBy(gdocs, "slug") // TODO: the knex instance should be handed down as a parameter diff --git a/db/model/Pageview.ts b/db/model/Pageview.ts index 325d940ba71..5407e41bf3f 100644 --- a/db/model/Pageview.ts +++ b/db/model/Pageview.ts @@ -1,6 +1,12 @@ import { keyBy } from "lodash" import { Entity, Column, BaseEntity } from "typeorm" import { RawPageview } from "@ourworldindata/utils" +import * as db from "../db.js" +import { Knex } from "knex" +import { + DbPlainAnalyticsPageview, + AnalyticsPageviewsTableName, +} from "@ourworldindata/types" @Entity("analytics_pageviews") export class Pageview extends BaseEntity implements RawPageview { @@ -18,13 +24,23 @@ export class Pageview extends BaseEntity implements RawPageview { @Column() views_14d!: number /** Sum of pageviews over the last 365 days. */ @Column() views_365d!: number +} - static async getViewsByUrlObj(): Promise<{ [url: string]: RawPageview }> { - const pageviews = await Pageview.find() +export async function getAnalyticsPageviewsByUrlObj( + knex: Knex +): Promise<{ + [url: string]: DbPlainAnalyticsPageview +}> { + const pageviews = await db.knexRaw( + "SELECT * FROM ??", + knex, + [AnalyticsPageviewsTableName] + ) - // Normalize URLs to be relative to the root of the site. - // This also filters out any URLs that don't start with ourworldindata.org. - const pageviewsNormalized: RawPageview[] = pageviews.flatMap((p) => { + // Normalize URLs to be relative to the root of the site. + // This also filters out any URLs that don't start with ourworldindata.org. + const pageviewsNormalized = pageviews.flatMap( + (p: DbPlainAnalyticsPageview) => { if (p.url.startsWith("https://ourworldindata.org/")) return [ { @@ -36,8 +52,8 @@ export class Pageview extends BaseEntity implements RawPageview { }, ] else return [] - }) + } + ) - return keyBy(pageviewsNormalized, (p) => p.url) - } + return keyBy(pageviewsNormalized, (p) => p.url) }