diff --git a/.gitignore b/.gitignore index 80358aaa44..d52d7a531b 100755 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ dist/ **/tsup.config.bundled*.mjs cfstorage/ vite.*.mjs +archive/* # Sentry Config File .env.sentry-build-plugin diff --git a/baker/GrapherBaker.tsx b/baker/GrapherBaker.tsx index aeaa1189c1..da8af226cc 100644 --- a/baker/GrapherBaker.tsx +++ b/baker/GrapherBaker.tsx @@ -33,6 +33,7 @@ import { DbPlainChart, DbRawChartConfig, DbEnrichedImage, + AssetMap, } from "@ourworldindata/types" import ProgressBar from "progress" import { @@ -58,7 +59,15 @@ const renderDatapageIfApplicable = async ( grapher: GrapherInterface, isPreviewing: boolean, knex: db.KnexReadonlyTransaction, - imageMetadataDictionary?: Record + { + imageMetadataDictionary, + staticAssetMap, + runtimeAssetMap, + }: { + imageMetadataDictionary?: Record + staticAssetMap?: AssetMap + runtimeAssetMap?: AssetMap + } = {} ) => { const variable = await getVariableOfDatapageIfApplicable(grapher) @@ -81,6 +90,8 @@ const renderDatapageIfApplicable = async ( useIndicatorGrapherConfigs: false, pageGrapher: grapher, imageMetadataDictionary, + staticAssetMap, + runtimeAssetMap, }, knex ) @@ -94,16 +105,23 @@ const renderDatapageIfApplicable = async ( export const renderDataPageOrGrapherPage = async ( grapher: GrapherInterface, knex: db.KnexReadonlyTransaction, - imageMetadataDictionary?: Record + { + imageMetadataDictionary, + staticAssetMap, + runtimeAssetMap, + }: { + imageMetadataDictionary?: Record + staticAssetMap?: AssetMap + runtimeAssetMap?: AssetMap + } = {} ): Promise => { - const datapage = await renderDatapageIfApplicable( - grapher, - false, - knex, - imageMetadataDictionary - ) + const datapage = await renderDatapageIfApplicable(grapher, false, knex, { + imageMetadataDictionary, + staticAssetMap, + runtimeAssetMap, + }) if (datapage) return datapage - return renderGrapherPage(grapher, knex) + return renderGrapherPage(grapher, knex, { staticAssetMap, runtimeAssetMap }) } export async function renderDataPageV2( @@ -114,6 +132,8 @@ export async function renderDataPageV2( useIndicatorGrapherConfigs, pageGrapher, imageMetadataDictionary = {}, + staticAssetMap, + runtimeAssetMap, }: { variableId: number variableMetadata: OwidVariableWithSource @@ -121,6 +141,8 @@ export async function renderDataPageV2( useIndicatorGrapherConfigs: boolean pageGrapher?: GrapherInterface imageMetadataDictionary?: Record + staticAssetMap?: AssetMap + runtimeAssetMap?: AssetMap }, knex: db.KnexReadonlyTransaction ) { @@ -217,6 +239,8 @@ export async function renderDataPageV2( imageMetadata={imageMetadata} faqEntries={faqEntries} tagToSlugMap={tagToSlugMap} + staticAssetMap={staticAssetMap} + runtimeAssetMap={runtimeAssetMap} /> ) } @@ -237,7 +261,14 @@ export const renderPreviewDataPageOrGrapherPage = async ( const renderGrapherPage = async ( grapher: GrapherInterface, - knex: db.KnexReadonlyTransaction + knex: db.KnexReadonlyTransaction, + { + staticAssetMap, + runtimeAssetMap, + }: { + staticAssetMap?: AssetMap + runtimeAssetMap?: AssetMap + } = {} ) => { const postSlug = urlToSlug(grapher.originUrl || "") as string | undefined // TODO: update this to use gdocs posts @@ -258,10 +289,46 @@ const renderGrapherPage = async ( relatedArticles={relatedArticles} baseUrl={BAKED_BASE_URL} baseGrapherUrl={BAKED_GRAPHER_URL} + staticAssetMap={staticAssetMap} + runtimeAssetMap={runtimeAssetMap} /> ) } +export const bakeSingleGrapherPageForArchival = async ( + bakedSiteDir: string, + grapher: GrapherInterface, + knex: db.KnexReadonlyTransaction, + { + imageMetadataDictionary, + staticAssetMap, + runtimeAssetMap, + }: { + imageMetadataDictionary?: Record + staticAssetMap?: AssetMap + runtimeAssetMap?: AssetMap + } = {} +) => { + const outPathHtml = `${bakedSiteDir}/grapher/${grapher.slug}.html` + await fs.writeFile( + outPathHtml, + await renderDataPageOrGrapherPage(grapher, knex, { + imageMetadataDictionary, + staticAssetMap, + runtimeAssetMap, + }) + ) + const outPathManifest = `${bakedSiteDir}/grapher/${grapher.slug}.manifest.json` + + // TODO: right now, this only contains the asset maps. it may in the future also contain input + // hashes of the config and data files. + await fs.writeFile( + outPathManifest, + JSON.stringify({ staticAssetMap, runtimeAssetMap }, undefined, 2) + ) + console.log(outPathHtml, outPathManifest) +} + const bakeGrapherPage = async ( bakedSiteDir: string, imageMetadataDictionary: Record, diff --git a/baker/buildLocalArchivalBake.ts b/baker/buildLocalArchivalBake.ts new file mode 100644 index 0000000000..3248d222d7 --- /dev/null +++ b/baker/buildLocalArchivalBake.ts @@ -0,0 +1,275 @@ +#! /usr/bin/env node + +// This should be imported as early as possible so the global error handler is +// set up before any errors are thrown. +import "../serverUtils/instrument.js" + +import yargs from "yargs" +import { hideBin } from "yargs/helpers" +import fs from "fs-extra" +import path, { normalize } from "path" +import * as db from "../db/db.js" +import { bakeSingleGrapherPageForArchival } from "./GrapherBaker.js" +import { keyBy } from "lodash" +import { getAllImages } from "../db/model/Image.js" +import { getChartConfigBySlug } from "../db/model/Chart.js" +import findProjectBaseDir from "../settings/findBaseDir.js" +import { AssetMap } from "@ourworldindata/types" +import { getVariableData } from "../db/model/Variable.js" +import dayjs from "dayjs" +import { hashBase36, hashBase36FromStream } from "../serverUtils/hash.js" +import * as Sentry from "@sentry/node" + +const DATE_TIME_FORMAT = "YYYYMMDD-HHmmss" +const DIR = "archive" + +const projBaseDir = findProjectBaseDir(__dirname) +if (!projBaseDir) throw new Error("Could not find project base directory") + +const archiveDir = path.join(projBaseDir, DIR) + +const hashFile = async (file: string) => { + const stream = await fs.createReadStream(file) + return await hashBase36FromStream(stream) +} + +const hashAndWriteFile = async (filename: string, content: string) => { + const hash = hashBase36(content) + const targetFilename = filename.replace(/^(.*\/)?([^.]+\.)/, `$1$2${hash}.`) + console.log(`Writing ${targetFilename}`) + const fullTargetFilename = path.resolve(archiveDir, targetFilename) + await fs.mkdirp(path.dirname(fullTargetFilename)) + await fs.writeFile(fullTargetFilename, content) + return path.relative(archiveDir, fullTargetFilename) +} + +const hashAndCopyFile = async (srcFile: string, targetDir: string) => { + const hash = await hashFile(srcFile) + const targetFilename = path + .basename(srcFile) + .replace(/^(.*\/)?([^.]+\.)/, `$1$2${hash}.`) + const targetFile = path.resolve(archiveDir, targetDir, targetFilename) + console.log(`Copying ${srcFile} to ${targetFile}`) + await fs.copyFile(srcFile, targetFile) + return path.relative(archiveDir, targetFile) +} + +const IGNORED_FILES_PATTERNS = [ + /^_headers$/, + /\.DS_Store$/, + /^images(\/.*)?$/, + /^sdg.*$/, + /^identifyadmin.html$/, + /^robots.txt$/, +] +const copyPublicDir = async () => { + const publicDir = path.join(projBaseDir, "public") + const targetDir = archiveDir + const ignoredFilesPattern = new RegExp( + IGNORED_FILES_PATTERNS.map((p) => p.source).join("|") + ) + await fs.copy(publicDir, targetDir, { + overwrite: true, + filter: (src) => { + const relativePath = path.relative(publicDir, src) + if (ignoredFilesPattern.test(relativePath)) { + console.log(`Ignoring ${relativePath}`) + return false + } + return true + }, + }) +} + +const bakeDods = async () => { + const srcFile = path.join(projBaseDir, "localBake/dods.json") + const targetDir = path.join(archiveDir, "assets") + + const newFilename = await hashAndCopyFile(srcFile, targetDir) + return { "dods.json": `/${newFilename}` } +} + +const bakeVariableDataFiles = async (variableId: number) => { + const { data, metadata } = await getVariableData(variableId) + const dataStringified = JSON.stringify(data) + const metadataStringified = JSON.stringify(metadata) + + const dataFilename = `api/v1/indicators/${variableId}.data.json` + const metadataFilename = `api/v1/indicators/${variableId}.metadata.json` + + const [dataFilenameWithHash, metadataFilenameWithHash] = await Promise.all([ + hashAndWriteFile(dataFilename, dataStringified), + hashAndWriteFile(metadataFilename, metadataStringified), + ]) + + return { + [path.basename(dataFilename)]: `/${dataFilenameWithHash}`, + [path.basename(metadataFilename)]: `/${metadataFilenameWithHash}`, + } +} + +const bakeOwidMjsFile = async ( + srcPath: string, + destPath: string, + sourceMapFilename: string +) => { + const mjsContents = await fs.readFile(srcPath, "utf8") + const withSourceMap = mjsContents.replace( + /\/\/# sourceMappingURL=owid.mjs.map/g, + `//# sourceMappingURL=${sourceMapFilename}` + ) + + if (withSourceMap === mjsContents) { + console.error("Failed to replace sourceMappingURL in owid.mjs") + } + return await hashAndWriteFile(destPath, withSourceMap) +} + +// we explicitly want owid.mjs.map to come first, so that we can replace the sourceMappingURL in owid.mjs +// after we know the content-hashed filename of owid.map.mjs +const ASSET_FILES = ["owid.mjs.map", "owid.mjs", "owid.css"] +const IGNORED_FILES = [".vite"] +const bakeAssets = async () => { + const srcDir = path.join(projBaseDir, "dist/assets") + const targetDir = path.join(projBaseDir, DIR, "assets") + + await fs.mkdirp(targetDir) + + const staticAssetMap: AssetMap = {} + + const filesInDir = await fs.readdir(srcDir, { withFileTypes: true }) + + for await (const filename of ASSET_FILES) { + if (!filesInDir.some((dirent) => dirent.name === filename)) { + throw new Error(`Could not find ${filename} in ${srcDir}`) + } + const srcFile = path.join(srcDir, filename) + + let outFilename: string + if (filename === "owid.mjs") { + const sourceMapFilename = path.basename( + staticAssetMap["owid.mjs.map"] ?? "" + ) + if (!sourceMapFilename) + throw new Error("Could not find owid.mjs.map in staticAssetMap") + + outFilename = await bakeOwidMjsFile( + srcFile, + path.join(targetDir, filename), + sourceMapFilename + ) + } else { + outFilename = await hashAndCopyFile(srcFile, targetDir) + } + staticAssetMap[filename] = `/${outFilename}` + } + + const additionalFiles = new Set( + filesInDir.map((dirent) => dirent.name) + ).difference(new Set([...ASSET_FILES, ...IGNORED_FILES])) + if (additionalFiles.size > 0) { + console.warn( + `Found additional files in ${srcDir}:`, + Array.from(additionalFiles).join(", ") + ) + } + + return { staticAssetMap } +} + +const bakeDomainToFolder = async ( + baseUrl = "http://localhost:3000/", + dir = DIR, + { copyToLatestDir = false }: { copyToLatestDir?: boolean } = {} +) => { + const dateTimeFormatted = dayjs().utc().format(DATE_TIME_FORMAT) + dir = path.join(normalize(dir), dateTimeFormatted) + await fs.mkdirp(dir) + await fs.mkdirp(path.join(dir, "grapher")) + + await copyPublicDir() + const { staticAssetMap } = await bakeAssets() + + console.log(`Baking site locally with baseUrl '${baseUrl}' to dir '${dir}'`) + + const SLUGS = ["life-expectancy", "environmental-footprint-milks"] + + const commonRuntimeFiles = await bakeDods() + + await db.knexReadonlyTransaction(async (trx) => { + const imageMetadataDictionary = await getAllImages(trx).then((images) => + keyBy(images, "filename") + ) + + for (const chartSlug of SLUGS) { + const chart = await getChartConfigBySlug(trx, chartSlug) + + const runtimeFiles = { ...commonRuntimeFiles } + + for (const dim of chart.config.dimensions ?? []) { + if (dim.variableId) { + const variableId = dim.variableId + const variableFiles = + await bakeVariableDataFiles(variableId) + Object.assign(runtimeFiles, variableFiles) + } + } + + await bakeSingleGrapherPageForArchival(dir, chart.config, trx, { + imageMetadataDictionary, + staticAssetMap, + runtimeAssetMap: runtimeFiles, + }) + } + }, db.TransactionCloseMode.Close) + + if (copyToLatestDir) { + const latestDir = path.join(DIR, "latest") + await fs.remove(latestDir) + await fs.copy(dir, latestDir) + console.log(`Copied ${dir} to ${latestDir}`) + } +} + +void yargs(hideBin(process.argv)) + .command<{ + baseUrl: string + dir: string + latestDir?: boolean + }>( + "$0 [baseUrl] [dir]", + "Bake the site to a local folder", + (yargs) => { + yargs + .positional("baseUrl", { + type: "string", + default: "http://localhost:3000/", + describe: "Base URL of the site", + }) + .positional("dir", { + type: "string", + default: "archive", + describe: "Directory to save the baked site", + }) + .option("latestDir", { + type: "boolean", + description: + "Copy the baked site to a 'latest' directory, for ease of testing", + }) + }, + async ({ baseUrl, dir, latestDir }) => { + await bakeDomainToFolder(baseUrl, dir, { + copyToLatestDir: latestDir, + }).catch(async (e) => { + console.error("Error in buildLocalArchivalBake:", e) + Sentry.captureException(e) + await Sentry.close() + process.exit(1) + }) + + process.exit(0) + } + ) + .help() + .alias("help", "h") + .strict().argv diff --git a/packages/@ourworldindata/grapher/src/core/Grapher.tsx b/packages/@ourworldindata/grapher/src/core/Grapher.tsx index f25460ab13..965b35869d 100644 --- a/packages/@ourworldindata/grapher/src/core/Grapher.tsx +++ b/packages/@ourworldindata/grapher/src/core/Grapher.tsx @@ -115,6 +115,7 @@ import { SeriesName, ChartViewInfo, OwidChartDimensionInterfaceWithMandatorySlug, + AssetMap, } from "@ourworldindata/types" import { BlankOwidTable, @@ -274,10 +275,11 @@ async function loadVariablesDataAdmin( async function loadVariablesDataSite( variableIds: number[], - dataApiUrl: string + dataApiUrl: string, + assetMap?: AssetMap ): Promise { const loadVariableDataPromises = variableIds.map((variableId) => - loadVariableDataAndMetadata(variableId, dataApiUrl) + loadVariableDataAndMetadata(variableId, dataApiUrl, assetMap) ) const variablesData: OwidVariableDataMetadataDimensions[] = await Promise.all(loadVariableDataPromises) @@ -338,6 +340,7 @@ export interface GrapherProgrammaticInterface extends GrapherInterface { ChartViewInfo, "parentChartSlug" | "queryParamsForParentChart" > + runtimeAssetMap?: AssetMap manager?: GrapherManager instanceRef?: React.RefObject @@ -518,6 +521,8 @@ export class Grapher "name" | "parentChartSlug" | "queryParamsForParentChart" > = undefined + runtimeAssetMap?: AssetMap = this.props.runtimeAssetMap + selection = this.manager?.selection ?? new SelectionArray( @@ -1154,7 +1159,8 @@ export class Grapher } else { variablesDataMap = await loadVariablesDataSite( this.variableIds, - this.dataApiUrl + this.dataApiUrl, + this.runtimeAssetMap ) } this.createPerformanceMeasurement("downloadVariablesData", startMark) @@ -2429,7 +2435,8 @@ export class Grapher } static renderSingleGrapherOnGrapherPage( - jsonConfig: GrapherInterface + jsonConfig: GrapherInterface, + { runtimeAssetMap }: { runtimeAssetMap?: AssetMap } = {} ): void { const container = document.getElementsByTagName("figure")[0] try { @@ -2439,6 +2446,7 @@ export class Grapher bindUrlToWindow: true, enableKeyboardShortcuts: true, queryStr: window.location.search, + runtimeAssetMap, }, container ) diff --git a/packages/@ourworldindata/grapher/src/core/loadVariable.ts b/packages/@ourworldindata/grapher/src/core/loadVariable.ts index 226d455b09..03524a5463 100644 --- a/packages/@ourworldindata/grapher/src/core/loadVariable.ts +++ b/packages/@ourworldindata/grapher/src/core/loadVariable.ts @@ -1,13 +1,21 @@ -import { OwidVariableDataMetadataDimensions } from "@ourworldindata/types" -import { fetchWithRetry } from "@ourworldindata/utils" +import { + AssetMap, + OwidVariableDataMetadataDimensions, +} from "@ourworldindata/types" +import { fetchWithRetry, readFromAssetMap } from "@ourworldindata/utils" export const getVariableDataRoute = ( dataApiUrl: string, - variableId: number + variableId: number, + assetMap?: AssetMap ): string => { if (dataApiUrl.includes("v1/indicators/")) { - // fetching from Data API, e.g. https://api.ourworldindata.org/v1/indicators/123.data.json - return `${dataApiUrl}${variableId}.data.json` + const filename = `${variableId}.data.json` + return readFromAssetMap(assetMap, { + path: filename, + // fetching from Data API, e.g. https://api.ourworldindata.org/v1/indicators/123.data.json + fallback: `${dataApiUrl}${filename}`, + }) } else { throw new Error(`dataApiUrl format not supported: ${dataApiUrl}`) } @@ -15,11 +23,16 @@ export const getVariableDataRoute = ( export const getVariableMetadataRoute = ( dataApiUrl: string, - variableId: number + variableId: number, + assetMap?: AssetMap ): string => { if (dataApiUrl.includes("v1/indicators/")) { - // fetching from Data API, e.g. https://api.ourworldindata.org/v1/indicators/123.metadata.json - return `${dataApiUrl}${variableId}.metadata.json` + const filename = `${variableId}.metadata.json` + return readFromAssetMap(assetMap, { + path: filename, + // fetching from Data API, e.g. https://api.ourworldindata.org/v1/indicators/123.metadata.json + fallback: `${dataApiUrl}${filename}`, + }) } else { throw new Error(`dataApiUrl format not supported: ${dataApiUrl}`) } @@ -27,13 +40,14 @@ export const getVariableMetadataRoute = ( export async function loadVariableDataAndMetadata( variableId: number, - dataApiUrl: string + dataApiUrl: string, + assetMap?: AssetMap ): Promise { const dataPromise = fetchWithRetry( - getVariableDataRoute(dataApiUrl, variableId) + getVariableDataRoute(dataApiUrl, variableId, assetMap) ) const metadataPromise = fetchWithRetry( - getVariableMetadataRoute(dataApiUrl, variableId) + getVariableMetadataRoute(dataApiUrl, variableId, assetMap) ) const [dataResponse, metadataResponse] = await Promise.all([ dataPromise, diff --git a/packages/@ourworldindata/types/src/domainTypes/Site.ts b/packages/@ourworldindata/types/src/domainTypes/Site.ts index 646f66b7c8..ab140121e8 100644 --- a/packages/@ourworldindata/types/src/domainTypes/Site.ts +++ b/packages/@ourworldindata/types/src/domainTypes/Site.ts @@ -6,3 +6,5 @@ export interface BreadcrumbItem { export interface KeyValueProps { [key: string]: string | boolean | undefined } + +export type AssetMap = Record diff --git a/packages/@ourworldindata/types/src/index.ts b/packages/@ourworldindata/types/src/index.ts index 7fc78018c3..e3a4d69a12 100644 --- a/packages/@ourworldindata/types/src/index.ts +++ b/packages/@ourworldindata/types/src/index.ts @@ -22,7 +22,11 @@ export { type QueryParams, R2GrapherConfigDirectory, } from "./domainTypes/Various.js" -export { type BreadcrumbItem, type KeyValueProps } from "./domainTypes/Site.js" +export { + type BreadcrumbItem, + type KeyValueProps, + type AssetMap, +} from "./domainTypes/Site.js" export { type FormattedPost, type IndexPost, diff --git a/packages/@ourworldindata/utils/src/Util.ts b/packages/@ourworldindata/utils/src/Util.ts index 5209f0e670..57d0d460aa 100644 --- a/packages/@ourworldindata/utils/src/Util.ts +++ b/packages/@ourworldindata/utils/src/Util.ts @@ -178,6 +178,7 @@ import { DimensionProperty, GRAPHER_CHART_TYPES, DbPlainTag, + AssetMap, } from "@ourworldindata/types" import { PointVector } from "./PointVector.js" import * as React from "react" @@ -2076,3 +2077,30 @@ export function isArrayDifferentFromReference( if (array.length !== referenceArray.length) return true return difference(array, referenceArray).length > 0 } + +// When reading from an asset map, we want a very particular behavior: +// If the asset map is entirely undefined, then we want to just fail silently and return the fallback. +// If the asset map is defined but the asset is not found, however, then we want to throw an error. +// This is to avoid invisible errors that'll lead to runtime errors or 404s. + +export function readFromAssetMap( + assetMap: AssetMap | undefined, + { path, fallback }: { path: string; fallback: string } +): string + +export function readFromAssetMap( + assetMap: AssetMap | undefined, + { path, fallback }: { path: string; fallback?: string } +): string | undefined + +export function readFromAssetMap( + assetMap: AssetMap | undefined, + { path, fallback }: { path: string; fallback?: string } +): string | undefined { + if (!assetMap) return fallback + + const assetValue = assetMap[path] + if (assetValue === undefined) + throw new Error(`Entry for asset not found in asset map: ${path}`) + return assetValue +} diff --git a/packages/@ourworldindata/utils/src/index.ts b/packages/@ourworldindata/utils/src/index.ts index 0da9e6e3e7..a69dbb3471 100644 --- a/packages/@ourworldindata/utils/src/index.ts +++ b/packages/@ourworldindata/utils/src/index.ts @@ -128,6 +128,7 @@ export { lazy, getParentVariableIdFromChartConfig, isArrayDifferentFromReference, + readFromAssetMap, } from "./Util.js" export { diff --git a/serverUtils/hash.test.ts b/serverUtils/hash.test.ts new file mode 100644 index 0000000000..e26597d27c --- /dev/null +++ b/serverUtils/hash.test.ts @@ -0,0 +1,30 @@ +import { PassThrough } from "node:stream" +import { hashBase36, hashBase36FromStream } from "./hash.js" + +describe(hashBase36, () => { + it("hashes a string", () => { + const hash = hashBase36("hello") + expect(hash).toBe("14bu24ea") + }) + + it("hashes a string with a non-default hash length", () => { + const hash = hashBase36("hello", 12) + expect(hash).toBe("14bu24ea7cq4") + }) + + it("hashes a buffer", () => { + const hash = hashBase36(Buffer.from("hello")) + expect(hash).toBe("14bu24ea") + }) + + it("hashes a stream", async () => { + const stream = new PassThrough() + const hashPromise = hashBase36FromStream(stream) + + stream.emit("data", "hello") + stream.end() + + const hash = await hashPromise + expect(hash).toBe("14bu24ea") + }) +}) diff --git a/serverUtils/hash.ts b/serverUtils/hash.ts new file mode 100644 index 0000000000..a8430c971d --- /dev/null +++ b/serverUtils/hash.ts @@ -0,0 +1,33 @@ +import crypto, { Hash } from "crypto" +import { pipeline } from "node:stream/promises" + +const DEFAULT_HASH_LENGTH = 8 + +const _hashBase36 = (hash: Hash, hashLength = DEFAULT_HASH_LENGTH) => { + const hashHex = hash.digest("hex") + + // Convert hex to base36 to make it contain more information in fewer characters + const hashBase36 = BigInt(`0x${hashHex}`).toString(36) + return hashBase36.substring(0, hashLength) +} + +export const hashBase36 = ( + strOrBuffer: string | Buffer, + hashLength = DEFAULT_HASH_LENGTH +) => { + const hash = crypto.createHash("sha256").update(strOrBuffer) + + return _hashBase36(hash, hashLength) +} + +export const hashBase36FromStream = async ( + stream: NodeJS.ReadableStream, + hashLength = DEFAULT_HASH_LENGTH +) => { + const hash = crypto.createHash("sha256") + + await pipeline(stream, hash) + hash.end() + + return _hashBase36(hash, hashLength) +} diff --git a/site/DataPageV2.tsx b/site/DataPageV2.tsx index 473f3ec52d..93705f824f 100644 --- a/site/DataPageV2.tsx +++ b/site/DataPageV2.tsx @@ -15,6 +15,7 @@ import { GrapherInterface, ImageMetadata, Url, + AssetMap, } from "@ourworldindata/utils" import { MarkdownTextWrap } from "@ourworldindata/components" import urljoin from "url-join" @@ -43,6 +44,9 @@ export const DataPageV2 = (props: { faqEntries?: FaqEntryData imageMetadata: Record tagToSlugMap: Record + staticAssetMap?: AssetMap + runtimeAssetMap?: AssetMap + dataApiUrl?: string }) => { const { grapher, @@ -53,6 +57,8 @@ export const DataPageV2 = (props: { faqEntries, tagToSlugMap, imageMetadata, + staticAssetMap, + runtimeAssetMap, } = props const pageTitle = grapher?.title ?? datapageData.title.title const canonicalUrl = grapher?.slug @@ -107,6 +113,7 @@ export const DataPageV2 = (props: { pageDesc={pageDesc} imageUrl={imageUrl} baseUrl={baseUrl} + staticAssetMap={staticAssetMap} > @@ -114,8 +121,16 @@ export const DataPageV2 = (props: { {variableIds.flatMap((variableId) => [ - getVariableDataRoute(DATA_API_URL, variableId), - getVariableMetadataRoute(DATA_API_URL, variableId), + getVariableDataRoute( + DATA_API_URL, + variableId, + runtimeAssetMap + ), + getVariableMetadataRoute( + DATA_API_URL, + variableId, + runtimeAssetMap + ), ].map((href) => (