diff --git a/packages/devvit/devvit.yaml b/packages/devvit/devvit.yaml index a9b8102..14a20c8 100644 --- a/packages/devvit/devvit.yaml +++ b/packages/devvit/devvit.yaml @@ -1,2 +1,2 @@ -name: test-app-4 -version: 0.0.1.3 +name: mediareliability +version: 0.0.5.27 diff --git a/packages/devvit/eslint.config.js b/packages/devvit/eslint.config.js index 68bcb7f..e4fe794 100644 --- a/packages/devvit/eslint.config.js +++ b/packages/devvit/eslint.config.js @@ -15,7 +15,7 @@ export default [ ...eslintPluginReactConfig, ...eslintPluginStylisticConfig, { - files: ["src/**/*.{ts,tsx}"], + files: ["src/**/*.{ts,tsx}", "test/**/*.{ts,tsx}"], languageOptions: { globals: { ...globals.browser, diff --git a/packages/devvit/package.json b/packages/devvit/package.json index 566222b..e6cbb26 100644 --- a/packages/devvit/package.json +++ b/packages/devvit/package.json @@ -3,14 +3,19 @@ "type": "module", "scripts": { "check": "tsc --noEmit", - "lint": "eslint --max-warnings 0 src" + "lint": "eslint --max-warnings 0 src", + "test": "vitest --reporter=verbose --run" }, "dependencies": { - "@devvit/public-api": "^0.10.22" + "@devvit/public-api": "^0.10.22", + "linkify-it": "^5.0.0", + "zod": "^3.23.8", + "zod-validation-error": "^3.3.0" }, "devDependencies": { "@devvit/protos": "^0.10.22", "@repo/db": "workspace:*", - "vitest": "^1.6.0" + "@types/linkify-it": "^5.0.0", + "vitest": "^2.0.3" } } diff --git a/packages/devvit/src/comment.ts b/packages/devvit/src/comment.ts new file mode 100644 index 0000000..7129457 --- /dev/null +++ b/packages/devvit/src/comment.ts @@ -0,0 +1,105 @@ +import type { SetPostFlairOptions, TriggerContext } from '@devvit/public-api'; +import { getTierDetails } from './helpers.js'; +import type { AppSettings, PostData, Source } from './types.js'; + +/** + * Matched sources are sorted by tier. + * Don't flair the post if: + * - The first source is official or an aggregator. + * - The post is a self-post. + */ +function shouldFlairPost(postData: PostData, sources: Source[]) { + if (postData.url?.hostname && ['reddit.com', 'www.reddit.com'].includes(postData.url.hostname)) { + return false; + } + + const { postFlairText } = getTierDetails(sources[0].tier); + + return postFlairText !== null; +} + +type FlairPostProps = { + postId: string; + sources: Source[]; + subredditName: Exclude; + flairCssClass: Exclude; + flairTemplateId: Exclude; + context: TriggerContext; +}; + +export async function flairPost({ postId, sources, subredditName, flairCssClass, flairTemplateId, context }: FlairPostProps) { + const { postFlairText } = getTierDetails(sources[0].tier); + + if (!postFlairText) { + return; + } + + await context.reddit.setPostFlair({ + postId, + text: postFlairText, + cssClass: flairCssClass, + flairTemplateId: flairTemplateId, + subredditName: subredditName + }); +} + +function getSourceLine(source: Source) { + const { name, handles, tier, domains } = source; + const { commentText, reliabilityText } = getTierDetails(tier); + + const handle = handles?.at(0) ?? null; + const domain = domains?.at(0) ?? null; + + const sourceName = handle + ? `${name} ([@${handle}](https://twitter.com/${handle}))` + : domain + ? `${name} ([${domain}](https://${domain}))` + : name; + + return `**${commentText}**: ${sourceName}${reliabilityText ? ` - ${reliabilityText}` : ''}`; +} + +type SubmitCommentProps = { + postData: PostData; + sources: Source[]; + settings: AppSettings; + context: TriggerContext; +}; + +export async function submitComment({ postData, sources, settings, context }: SubmitCommentProps) { + if (shouldFlairPost(postData, sources)) { + await flairPost({ + postId: postData.id, + sources: sources, + subredditName: postData.subredditName, + flairCssClass: settings.flairCssClass, + flairTemplateId: settings.flairTemplateId, + context: context + }); + } + + const header = `**Media reliability report:**`; + const warningForUnreliable = sources.some(source => getTierDetails(source.tier).unreliable) + ? settings.unreliableSourcesWarning + : null; + const footer = settings.commentFooter; + + const markdown = [ + header, + ...sources.map(source => `- ${getSourceLine(source)}`), + warningForUnreliable, + footer + ] + .filter(Boolean) + .join('\n\n'); + + const comment = await context.reddit.submitComment({ + id: postData.id, + text: markdown + }); + + await Promise.all([ + comment.distinguish(true), + comment.lock() + ]); +} \ No newline at end of file diff --git a/packages/devvit/src/helpers.ts b/packages/devvit/src/helpers.ts new file mode 100644 index 0000000..2bbe592 --- /dev/null +++ b/packages/devvit/src/helpers.ts @@ -0,0 +1,170 @@ +import type { Context, TriggerContext } from '@devvit/public-api'; +import linkifyit from 'linkify-it'; +import { fromZodError } from 'zod-validation-error'; +import { settingsSchema } from './schema.js'; +import type { AppSettings, RedditPostV1, Source, TierDetails } from './types.js'; + +const linkify = new linkifyit(); + +/** + * Remove diacritics and convert to lowercase + * + * @note There is a slight performance penalty when using modern unicode + * property escapes so prefer using the old method with character class + * range for now. + * + * @see https://stackoverflow.com/a/37511463/3258251 + */ +export function normalizeText(text: string) { + return text + .normalize('NFD') + .replace(/[\u0300-\u036f]/g, '') + .toLowerCase(); +} + +export function capitalizeString(string: string) { + return string.charAt(0).toUpperCase() + string.slice(1); +} + +/** + * Details: + * + * order: used for sorting tiers + * commentText: text to display in the comment + * postFlairText: text to display in the post flair + * reliabilityText: text to display in the comment after the source name + * unreliable: whether the source is unreliable + */ +const tierData: Record = { + official: { + order: 0, + commentText: 'Official', + postFlairText: 'Official', + reliabilityText: 'official source', + unreliable: false + }, + 1: { + order: 1, + commentText: 'Tier 1', + postFlairText: 'Tier 1', + reliabilityText: 'very reliable', + unreliable: false + }, + 2: { + order: 2, + commentText: 'Tier 2', + postFlairText: 'Tier 2', + reliabilityText: 'reliable', + unreliable: false + }, + 3: { + order: 3, + commentText: 'Tier 3', + postFlairText: 'Tier 3', + reliabilityText: '❗ unreliable', + unreliable: true + }, + 4: { + order: 4, + commentText: 'Tier 4', + postFlairText: 'Tier 4', + reliabilityText: '❗ very unreliable', + unreliable: true + }, + 5: { + order: 5, + commentText: 'Tier 5', + postFlairText: 'Tier 5', + reliabilityText: '❗ extremely unrialable', + unreliable: true + }, + aggregator: { + order: 6, + commentText: 'Aggregator', + postFlairText: null, + reliabilityText: null, + unreliable: false + }, +}; + +export function getTierDetails(tier: Source['tier']) { + return tierData[tier]; +} + +export function sortTiers(a: Source, b: Source) { + return getTierDetails(a.tier).order - getTierDetails(b.tier).order; +} + +/** + * Devvit onValidate is a bit weird, if you return string it assumes an error, + * if you return undefined it assumes success, so here we return accordingly. + */ +export function validateSetting(key: keyof AppSettings, value: unknown) { + const parsed = settingsSchema.shape[key].safeParse(value); + + return parsed.success + ? undefined + : `Invalid value for "${key}" setting. Error:\n ${fromZodError(parsed.error)}`; +} + +export async function getAllSettings(context: Context | TriggerContext) { + return settingsSchema.parse(await context.settings.getAll()); +} + +export function isIgnoredUser(username: string, settings: AppSettings) { + return settings.ignoredUsers + .some(ignoredUser => ignoredUser.toLowerCase() === username.toLowerCase()); +} + +/** + * This can be removed when the devvit remote tsc compiler is updated to ts 5.5+ + * @see https://devblogs.microsoft.com/typescript/announcing-typescript-5-5/#inferred-type-predicates + */ +function nonNullable(value: T): value is NonNullable { + return value !== null; +} + +/** + * Return all links in the post body, ignoring ones from reddit + */ +function getPostLinks(body: string) { + const links = (linkify.match(body) ?? []) + .map(link => { + try { + const url = new URL(link.url, 'https://www.reddit.com'); + return ['v.redd.it', 'i.redd.it', 'reddit.com', 'www.reddit.com'].includes(url.hostname) ? null : url; + } + catch (error) { + return null; + } + }) + .filter(nonNullable); + + return links.length > 0 ? links : null; +} + +export function processPost(post: RedditPostV1) { + const url = new URL(post.url, 'https://www.reddit.com'); + const links = post.body ? getPostLinks(post.body) : null; + + return ({ + id: post.id, + subredditName: post.subredditName, + titleNormalized: normalizeText(post.title), + bodyNormalized: post.body && post.body.length > 0 ? normalizeText(post.body) : null, + url: !['v.redd.it', 'i.redd.it', 'reddit.com', 'www.reddit.com'].includes(url.hostname) ? url : null, + links: links, + }); +} + +export async function trySendPostErrorModmail(context: TriggerContext, postId: string, error: Error) { + const { errorReportSubredditName } = await getAllSettings(context); + + if (errorReportSubredditName) { + await context.reddit.sendPrivateMessage({ + subject: 'An error occurred with the media reliability app', + text: `An error occurred with this post: https://redd.it/${postId.replace(/^t3_/, '')}\n\n${String(error)}`, + to: errorReportSubredditName + }); + } +} diff --git a/packages/devvit/src/index.ts b/packages/devvit/src/index.ts new file mode 100644 index 0000000..0e9ef52 --- /dev/null +++ b/packages/devvit/src/index.ts @@ -0,0 +1,4 @@ +export * from './comment.js'; +export * from './helpers.js'; +export * from './matcher.js'; +export * from './schema.js'; diff --git a/packages/devvit/src/main.ts b/packages/devvit/src/main.ts index bcfe111..fb9acfe 100644 --- a/packages/devvit/src/main.ts +++ b/packages/devvit/src/main.ts @@ -1,13 +1,170 @@ import { Devvit } from '@devvit/public-api'; +import { findSourcesInPost, getAllSettings, isIgnoredUser, processPost, submitComment, trySendPostErrorModmail, validateSetting } from './index.js'; Devvit.configure({ redditAPI: true }); +Devvit.addSettings([ + { + type: 'paragraph', + name: 'sources', + label: 'List of media reliability sources in JSON format', + helpText: 'Visit the documentation for more information.', + scope: 'installation', + defaultValue: '[]', + placeholder: 'Paste JSON array here', + onValidate: ({ value }) => { + return validateSetting('sources', value); + } + }, + { + type: 'string', + name: 'flairTemplateId', + label: 'Flair template ID', + helpText: 'The flair template ID to apply when assigning the flair.', + scope: 'installation', + defaultValue: '', + placeholder: 'Enter flair template ID here', + onValidate: ({ value }) => { + return validateSetting('flairTemplateId', value); + } + }, + { + type: 'string', + name: 'flairCssClass', + label: 'Flair CSS class', + helpText: 'The CSS class to apply when assigning the flair.', + scope: 'installation', + defaultValue: '', + placeholder: 'Enter flair template ID here', + onValidate: ({ value }) => { + return validateSetting('flairCssClass', value); + } + }, + { + type: 'paragraph', + name: 'unreliableSourcesWarning', + label: 'Unreliable sources warning', + helpText: 'The warning text to append to each media report comment if unreliable sources are found in the post. Markdown is supported.', + defaultValue: '❗ Readers beware: This post contains information from unreliable and/or untrustworthy source(s). As such, we highly encourage our userbase to question the authenticity of any claims or quotes presented by it before jumping into conclusions or taking things as a fact.', + scope: 'installation', + onValidate: ({ value }) => { + return validateSetting('unreliableSourcesWarning', value); + } + }, + { + type: 'paragraph', + name: 'commentFooter', + label: 'Comment footer', + helpText: 'The footer text to append at the end of each media report comment. Markdown is supported.', + defaultValue: '', + scope: 'installation', + onValidate: ({ value }) => { + return validateSetting('commentFooter', value); + } + }, + { + type: 'paragraph', + name: 'ignoredUsers', + label: 'Comma separated list of users to ignore.', + helpText: 'Posts by these users will not trigger any bot actions.', + defaultValue: 'AutoModerator', + onValidate: ({ value }) => { + return validateSetting('ignoredUsers', value); + } + }, + { + type: 'boolean', + name: 'analyzeNamesInBody', + label: 'Analyze post body for source names', + helpText: 'If enabled, the bot will analyze the post body for source names.', + defaultValue: true, + onValidate: ({ value }) => { + return validateSetting('analyzeNamesInBody', value); + } + }, + { + type: 'boolean', + name: 'analyzeHandlesInBody', + label: 'Analyze post body for source handles', + helpText: 'If enabled, the bot will analyze the post body for source handles.', + defaultValue: true, + onValidate: ({ value }) => { + return validateSetting('analyzeHandlesInBody', value); + } + }, + { + type: 'boolean', + name: 'analyzeLinksInBody', + label: 'Analyze post body for domain names', + helpText: 'If enabled, the bot will analyze the post body for links.', + defaultValue: true, + onValidate: ({ value }) => { + return validateSetting('analyzeLinksInBody', value); + } + }, + { + type: 'string', + name: 'errorReportSubredditName', + label: 'Subrreddit name to send modmail for app error reports', + helpText: 'The subreddit name to send modmail to in case of an error. Set it to your own subreddit to receive error reports in modmail or leave it blank. You can also set it to "barcadev".', + defaultValue: '', + scope: 'installation', + onValidate: ({ value }) => { + return validateSetting('errorReportSubredditName', value); + } + } +]); + +Devvit.addTrigger({ + event: 'PostSubmit', + onEvent: async (event, context) => { + try { + if (!event.post?.id || !event.subreddit?.name || !event.author?.name) { + throw new Error('PostSubmit event missing post id, subreddit name or author name.'); + } + + const post = event.post.crosspostParentId + ? await context.reddit.getPostById(event.post.crosspostParentId) + : await context.reddit.getPostById(event.post.id); + + const postData = processPost(post); + const settings = await getAllSettings(context); + + if (isIgnoredUser(event.author.name, settings)) { + return; + } + + const sources = findSourcesInPost(postData, settings); + + if (!sources) { + return; + } + + await submitComment({ postData, sources, settings, context }); + + } + catch (error) { + console.error(error); + + if (error instanceof Error && event.post?.id) { + await trySendPostErrorModmail(context, event.post.id, error); + } + } + } +}); + Devvit.addMenuItem({ - label: 'Testing', - location: 'subreddit', - onPress(event, context) { - console.log(event); - console.log(context); + label: 'Log found sources (mediareliability)', + location: 'post', + async onPress(event, context) { + const post = await context.reddit.getPostById(event.targetId); + const postData = processPost(post); + + const settings = await getAllSettings(context); + const sources = findSourcesInPost(postData, settings); + + context.ui.showToast('Check app logs for debugging information.'); + console.log(JSON.stringify(sources, null, 2)); }, }); diff --git a/packages/devvit/src/matcher.ts b/packages/devvit/src/matcher.ts new file mode 100644 index 0000000..22e88d5 --- /dev/null +++ b/packages/devvit/src/matcher.ts @@ -0,0 +1,369 @@ +import { sortTiers } from './index.js'; +import type { AppSettings, PostData, Source } from './types.js'; + +export function findSourcesInPost(post: PostData, settings: AppSettings) { + const list = new Map(); + + findMatchesInTitle(post.titleNormalized, settings.sources, list); + + if (post.url) { + findMatchesInUrl(post.url, settings.sources, list); + } + + if (post.links && settings.analyzeLinksInBody) { + findMatchesInLinks(post.links, settings.sources, list); + } + + if (post.bodyNormalized && (settings.analyzeNamesInBody || settings.analyzeHandlesInBody)) { + findMatchesInBody(post.bodyNormalized, settings, list); + } + + const result = Array + .from(list.values()) + .sort(sortTiers); + + return result.length > 0 ? result : null; +} + +function findMatchesInTitle(titleNormalized: string, sources: Source[], list: Map) { + for (const source of sources) { + if (isNameInTitle({ titleNormalized, source })) { + list.set(source.id, source); + } + else if (isAliasInTitle({ titleNormalized, source })) { + list.set(source.id, source); + } + else if (isHandleInTitle({ titleNormalized, source })) { + list.set(source.id, source); + } + } +} + +function findMatchesInUrl(url: URL, sources: Source[], list: Map) { + for (const source of sources) { + if (isHandleInUrl({ url, source })) { + list.set(source.id, source); + } + else if (isDomainInUrl({ url, source })) { + list.set(source.id, source); + } + } +} + +function findMatchesInLinks(urls: URL[], sources: Source[], list: Map) { + for (const source of sources) { + if (isHandleInLinks({ urls, source })) { + list.set(source.id, source); + } + else if (isDomainInLinks({ urls, source })) { + list.set(source.id, source); + } + } +} + +function findMatchesInBody(bodyNormalized: string, settings: AppSettings, list: Map) { + for (const source of settings.sources) { + if (settings.analyzeNamesInBody && isNameInBody({ bodyNormalized, source })) { + list.set(source.id, source); + } + else if (settings.analyzeNamesInBody && isAliasInBody({ bodyNormalized, source })) { + list.set(source.id, source); + } + else if (settings.analyzeHandlesInBody && isHandleInBody({ bodyNormalized, source })) { + list.set(source.id, source); + } + } +} + +/** + * Check if `source.nameNormalized` matches against a title. + * + * If `source.nameIsCommon` is `true`, ONLY match these patterns: + * + * @example```txt + * name: rest of the title + * title which includes (name) + * title which includes [name] + * ``` + * + * If `source.nameIsCommon` is `false`, match whole word: + * + * @example```txt + * title which includes name anywhere + * ``` + * + * @devnote Double escape template literal RegExp + */ +function isNameInTitle({ titleNormalized, source }: { titleNormalized: string, source: Source }) { + if (source.nameIsCommon) { + return new RegExp(`^${source.nameNormalized}:|(\\(|\\[)${source.nameNormalized}(\\)|\\])`, 'i').test(titleNormalized); + } + + return new RegExp(`\\b${source.nameNormalized}\\b`, 'i').test(titleNormalized); +} + +/** + * Check if source.aliasesNormalized.[n].aliasNormalized matches against a title. + * + * If `source.aliasesNormalized.[n].aliasIsCommon` is `true`, ONLY match these patterns: + * + * @example```txt + * alias: rest of the title + * title which includes (alias) + * title which includes [alias] + * ``` + * + * If `source.aliasesNormalized.[n].aliasIsCommon` is `false`, match whole word: + * + * @example```txt + * title which includes alias anywhere + * ``` + * + * @devnote Double escape template literal RegExp + */ +function isAliasInTitle({ titleNormalized, source }: { titleNormalized: string, source: Source }) { + if (!source.aliasesNormalized) { + return false; + } + + return source.aliasesNormalized + .some(alias => alias.aliasIsCommon + ? new RegExp(`^${alias.aliasNormalized}:|(\\(|\\[)${alias.aliasNormalized}(\\)|\\])`, 'i').test(titleNormalized) + : new RegExp(`\\b${alias.aliasNormalized}\\b`, 'i').test(titleNormalized)); +} + +/** + * Check if `source.handlesNormalized.[n].handleNormalized` matches against a title. + * Only match the following patterns: + * + * @example```txt + * ‌@handle: rest of the title + * title which includes (handle) + * title which includes [handle] + * title which includes ‌@handle + * ``` + * + * @devnote Example uses zero-width non-joiner character (U+200c) in front of the @ sign to escape it because of a bug with jsdoc. + * @devnote Double escape template literal RegExp + * + * @see https://github.com/jsdoc/jsdoc/issues/1521 + * @see https://unicode-explorer.com/c/200C + */ +function isHandleInTitle({ titleNormalized, source }: { titleNormalized: string, source: Source }) { + if (!source.handlesNormalized) { + return false; + } + + return source.handlesNormalized + .some(handle => new RegExp(`^@?${handle}:|@${handle}\\b|(\\(|\\[)${handle}(\\)|\\])`, 'i').test(titleNormalized)); +} + +/** + * Check if `source.handlesNormalized.[n].handleNormalized` matches against URL pathname. + * + * Only match x.com and twitter.com domains. + * + * Only match the following patterns: + * + * @example```txt + * twiter.com/handle + * twiter.com/handle/ + * twiter.com/handle/status/12345 + * x.com/handle + * x.com/handle/ + * x.com/handle/status/12345 + * ``` + * + * @devnote URL pathname always starts with forward slash + * @devnote Double escape template literal RegExp + */ +function isHandleInUrl({ url, source }: { url: URL, source: Source }) { + if (!source.handlesNormalized) { + return false; + } + + if (!['x.com', 'twitter.com'].includes(url.hostname)) { + return false; + } + + return source.handlesNormalized + .some(handle => new RegExp(`^\\/${handle}(\\/|\\s|$)`, 'i').test(url.pathname)); +} + +/** + * Check if `source.domains.[n]` matches against URL hostname (domain). + * + * Only match the following patterns: + * + * @example```txt + * example.com + * www.example.com + * sub1.example.com + * sub1.sub2.example.com + * ``` + * + * @devnote The URL constructor does NOT strip out www. prefix if present + * @devnote Escape periods in domain names + * @devnote Double escape template literal RegExp + */ +function isDomainInUrl({ url, source }: { url: URL, source: Source }) { + if (!source.domains) { + return false; + } + + return source.domains + .some(domain => new RegExp(`^(.*\\.)?${domain.replace(/\./g, '\\.')}$`).test(url.hostname)); +} + +/** + * Check if `source.handlesNormalized.[n].handleNormalized` matches against URL list pathnames. + * + * Only match x.com and twitter.com domains. + * + * Only match the following patterns: + * + * @example```txt + * twiter.com/handle + * twiter.com/handle/ + * twiter.com/handle/status/12345 + * x.com/handle + * x.com/handle/ + * x.com/handle/status/12345 + * ``` + * + * @devnote Double escape template literal RegExp + */ +function isHandleInLinks({ urls, source }: { urls: URL[], source: Source }) { + if (!source.handlesNormalized) { + return false; + } + + return urls + .some(url => + ['x.com', 'twitter.com'].includes(url.hostname) && + source.handlesNormalized + ?.some(handle => new RegExp(`^\/${handle}(\\/|\\s|$)`, 'i').test(url.pathname))); +} + +/** + * Check if `source.domains.[n]` matches against URL hostnames (domains) list. + * + * Only match the following patterns: + * + * @example```txt + * example.com + * www.example.com + * sub1.example.com + * sub1.sub2.example.com + * ``` + * + * @devnote The URL constructor does NOT strip out www. prefix if present + * @devnote Escape periods in domain names + * @devnote Double escape template literal RegExp + */ +function isDomainInLinks({ urls, source }: { urls: URL[], source: Source }) { + if (!source.domains) { + return false; + } + + return urls + .some(url => source.domains + ?.some(domain => new RegExp(`^(.*\\.)?${domain.replace(/\./g, '\\.')}$`).test(url.hostname))); +} + +/** + * Check if `source.nameNormalized` matches against a body. + * + * If `source.nameIsCommon` is `true`, ONLY match these patterns: + * + * @example```txt + * name: rest of the body + * body which includes (name) + * body which includes [name] + * ``` + * + * If `source.nameIsCommon` is `false`, match whole word: + * + * @example```txt + * body which includes name anywhere + * ``` + * + * @devnote Double escape template literal RegExp + */ +function isNameInBody({ bodyNormalized, source }: { bodyNormalized: string, source: Source }) { + if (source.nameIsCommon) { + return new RegExp(`^${source.nameNormalized}:|(\\[|\\()${source.nameNormalized}(\\]|\\))`, 'im').test(bodyNormalized); + } + + return new RegExp(`\\b${source.nameNormalized}\\b`, 'i').test(bodyNormalized); +} + +/** + * Check if `source.aliasesNormalized.[n].aliasNormalized` matches against a body. + * + * If `source.aliasesNormalized.[n].aliasIsCommon` is `true`, ONLY match these patterns: + * + * @example```txt + * alias: rest of the body + * body which includes (alias) + * body which includes [alias] + * ``` + * + * If `source.aliasesNormalized.[n].aliasIsCommon` is `false`, match whole word: + * + * @example```txt + * body which includes alias anywhere + * ``` + * + * @devnote Double escape template literal RegExp + */ +function isAliasInBody({ bodyNormalized, source }: { bodyNormalized: string, source: Source }) { + if (!source.aliasesNormalized) { + return false; + } + + return source.aliasesNormalized + .some(alias => alias.aliasIsCommon + ? new RegExp(`^${alias.aliasNormalized}:|(\\[|\\()${alias.aliasNormalized}(\\]|\\))`, 'im').test(bodyNormalized) + : new RegExp(`\\b${alias.aliasNormalized}\\b`, 'i').test(bodyNormalized)); +} + +/** + * Check if `source.handlesNormalized.[n].handleNormalized` matches against a body. + * + * Only match the following patterns: + * + * @example```txt + * handle: rest of the body + * ‌@handle: rest of the body + * body which includes (handle) + * body which includes [handle] + * body which includes ‌@handle + * ``` + * @devnote Example uses zero-width non-joiner character (U+200c) in front of the @ sign to escape it because of a bug with jsdoc. + * @devnote Double escape template literal RegExp + * + * @see https://github.com/jsdoc/jsdoc/issues/1521 + * @see https://unicode-explorer.com/c/200C + */ +function isHandleInBody({ bodyNormalized, source }: { bodyNormalized: string, source: Source }) { + if (!source.handlesNormalized) { + return false; + } + + return source.handlesNormalized + .some(handle => new RegExp(`^@?${handle}:|@${handle}\\b|(\\(|\\[)${handle}(\\)|\\])`, 'im').test(bodyNormalized)); +} + +export const __test__ = { + isNameInTitle, + isAliasInTitle, + isHandleInTitle, + isHandleInUrl, + isHandleInLinks, + isDomainInUrl, + isDomainInLinks, + isNameInBody, + isAliasInBody, + isHandleInBody +}; \ No newline at end of file diff --git a/packages/devvit/src/schema.ts b/packages/devvit/src/schema.ts new file mode 100644 index 0000000..310a9ef --- /dev/null +++ b/packages/devvit/src/schema.ts @@ -0,0 +1,112 @@ +import type { RefinementCtx } from 'zod'; +import { z } from 'zod'; +import { normalizeText } from './index.js'; + +function preprocessCommaSeparated(value: unknown, ctx: RefinementCtx) { + if (typeof value !== 'string') { + ctx.addIssue({ + code: 'invalid_type', + expected: 'string', + received: typeof value, + }); + + return z.NEVER; + } + + try { + const usernames = value.split(',').map(item => item.trim()); + + // reddit usernames can only contain alphanumeric characters, underscores and hyphens + if (usernames.some(username => /[^a-zA-Z0-9_-]/.test(username))) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Invalid input. Reddit usernames can only contain alphanumeric characters, underscores and hyphens.', + fatal: true, + }); + + return z.NEVER; + } + + return usernames; + } + + catch (error) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Invalid input. Expected comma-separated string with values.', + fatal: true, + }); + + return z.NEVER; + } +} + +/** + * @see https://zod.dev/ERROR_HANDLING + */ +function preprocessJSON(value: unknown, ctx: RefinementCtx) { + if (typeof value !== 'string') { + ctx.addIssue({ + code: 'invalid_type', + expected: 'string', + received: typeof value, + }); + + return z.NEVER; + } + + try { + return JSON.parse(value); + } + + catch (error) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Invalid JSON input', + fatal: true, + }); + + return z.NEVER; + } +} + +const aliasSchema = z.object({ + alias: z.string(), + aliasIsCommon: z.boolean(), +}); + +export const sourceSchema = z.object({ + id: z.string(), + name: z.string(), + nameIsCommon: z.boolean(), + tier: z.union([ + z.literal('official'), + z.literal('1'), + z.literal('2'), + z.literal('3'), + z.literal('4'), + z.literal('5'), + z.literal('aggregator'), + ]), + handles: z.array(z.string()).nullable(), + domains: z.array(z.string()).nullable(), + aliases: z.array(aliasSchema).nullable(), +}).transform(data => ({ + ...data, + nameNormalized: normalizeText(data.name), + handlesNormalized: data.handles ? data.handles.map(handle => normalizeText(handle)) : null, + aliasesNormalized: data.aliases ? data.aliases.map(alias => ({ ...alias, aliasNormalized: normalizeText(alias.alias) })) : null, +})); + +export const settingsSchema = z.object({ + sources: z.preprocess((data, ctx) => preprocessJSON(data, ctx), z.array(sourceSchema)), + flairTemplateId: z.string(), + flairCssClass: z.string(), + commentFooter: z.string(), + analyzeNamesInBody: z.boolean(), + analyzeHandlesInBody: z.boolean(), + analyzeLinksInBody: z.boolean(), + unreliableSourcesWarning: z.string(), + ignoredUsers: z.preprocess((data, ctx) => preprocessCommaSeparated(data, ctx), z.array(z.string())), + errorReportSubredditName: z.string(), +}); \ No newline at end of file diff --git a/packages/devvit/src/types.ts b/packages/devvit/src/types.ts new file mode 100644 index 0000000..595f57c --- /dev/null +++ b/packages/devvit/src/types.ts @@ -0,0 +1,25 @@ +import type { PostCreate } from '@devvit/protos'; +import type { Post } from '@devvit/public-api'; +import type { z } from 'zod'; +import type { processPost } from './index.js'; +import type { settingsSchema, sourceSchema } from './schema.js'; + +export type AppSettings = z.infer; +export type Source = z.infer; + +export type RedditPostV1 = Post; +export type RedditPostV2 = Exclude; + +export type PostData = ReturnType; + +export type ValidationResult = + { success: true } | + { success: false, message: string }; + +export type TierDetails = { + order: number; + postFlairText: string | null; + commentText: string; + reliabilityText: string | null; + unreliable: boolean; +}; \ No newline at end of file diff --git a/packages/devvit/test/helpers.ts b/packages/devvit/test/helpers.ts new file mode 100644 index 0000000..610df40 --- /dev/null +++ b/packages/devvit/test/helpers.ts @@ -0,0 +1,35 @@ +import { expect } from 'vitest'; +import { sourceSchema } from '../src/index.js'; +import type { Source } from '../src/types.js'; + +/** + * Disallow passing normalized values directly to the function. + */ +type CreateSourceParams = { + [key in keyof Omit]+?: Source[key] +}; + +export function createSource(params: CreateSourceParams) { + const source: Omit = { + id: params.id ?? Math.random().toString(), + name: params.name ?? 'name', + nameIsCommon: params.nameIsCommon ?? false, + tier: params.tier ?? '1', + handles: params.handles ?? null, + domains: params.domains ?? null, + aliases: params.aliases ?? null, + }; + + return sourceSchema.parse(source); +} + +type Entry = ReadonlyArray; +type MatcherFunction = (params: { source: ReturnType, content: T }) => boolean; + +export function expectEntries(entries: Entry, source: ReturnType, matcher: MatcherFunction) { + return expect( + entries.map(([_, content]) => matcher({ source, content })) + ).toEqual( + entries.map(([expected, _]) => expected) + ); +} \ No newline at end of file diff --git a/packages/devvit/test/matcher.test.ts b/packages/devvit/test/matcher.test.ts new file mode 100644 index 0000000..11a8089 --- /dev/null +++ b/packages/devvit/test/matcher.test.ts @@ -0,0 +1,323 @@ +import { describe, expect, test } from 'vitest'; +import { normalizeText } from '../src/index.js'; +import { __test__ } from '../src/matcher.js'; +import { createSource, expectEntries } from './helpers.js'; + +const { + isNameInTitle, + isNameInBody, + isAliasInTitle, + isAliasInBody, + isHandleInTitle, + isHandleInUrl, + isHandleInLinks, + isHandleInBody, + isDomainInUrl, + isDomainInLinks, +} = __test__; + +describe('normalization', () => { + test('normalize name', () => { + const source = createSource({ name: 'fÓÓNamE' }); + + expect(source.nameNormalized).toEqual('fooname'); + }); + + test('normalize aliases', () => { + const source = createSource({ + aliases: [ + { alias: 'fÓÓNamE', aliasIsCommon: true, }, + { alias: 'bäRNäMê', aliasIsCommon: true, }, + ] + }); + + expect(source.aliasesNormalized).toEqual([ + { alias: 'fÓÓNamE', aliasNormalized: 'fooname', aliasIsCommon: true }, + { alias: 'bäRNäMê', aliasNormalized: 'barname', aliasIsCommon: true }, + ]); + }); +}); + +describe('names', () => { + describe('isNameInTitle', () => { + test('nameIsCommon: true', () => { + expectEntries( + [ + [true, 'name: rest of the title'], + [true, 'title which includes (name)'], + [true, 'title which includes [name]'], + [false, 'title which includes name anywhere'], + ], + createSource({ name: 'NäMê', nameIsCommon: true }), + ({ source, content }) => isNameInTitle({ source, titleNormalized: normalizeText(content) }) + ); + }); + + test('nameIsCommon: false', () => { + expectEntries( + [ + [true, 'name: rest of the title'], + [true, 'title which includes (name)'], + [true, 'title which includes [name]'], + [true, 'title which includes name anywhere'], + [false, 'title which includes name_ anywhere'], + [false, 'title which includes _name anywhere'], + [false, 'nameX: rest of the title'], + ], + createSource({ name: 'NäMê', nameIsCommon: false }), + ({ source, content }) => isNameInTitle({ source, titleNormalized: normalizeText(content) }) + ); + }); + }); + + describe('isNameInBody', () => { + test('nameIsCommon: true', () => { + expectEntries( + [ + [true, 'name: rest of the body'], + [true, 'post body with\nname: in it'], + [true, 'body which includes\n (name)'], + [true, 'body which includes\n [name]'], + [false, 'body which includes name anywhere'], + [false, 'body which includes\nname anywhere'] + ], + createSource({ name: 'name', nameIsCommon: true }), + ({ source, content }) => isNameInBody({ source, bodyNormalized: normalizeText(content) }) + ); + }); + + test('nameIsCommon: false', () => { + expectEntries( + [ + [true, 'body which includes name anywhere'], + [true, 'body which includes\nname\n anywhere'], + [false, 'body which includes name_ anywhere'], + [false, 'body which includes _name anywhere'], + [false, '\nnameX: rest of the body'], + ], + createSource({ name: 'name', nameIsCommon: false }), + ({ source, content }) => isNameInBody({ source, bodyNormalized: normalizeText(content) }) + ); + }); + }); +}); + +describe('aliases', () => { + describe('isAliasInTitle', () => { + test('aliasIsCommon: true', () => { + expectEntries( + [ + [true, 'alias1: rest of the title'], + [true, 'title which includes (alias1)'], + [true, 'title which includes [alias1]'], + [true, 'alias2: rest of the title'], + [true, 'title which includes (alias2)'], + [true, 'title which includes [alias2]'], + [false, 'title which includes alias1 anywhere'], + [false, 'title which includes alias2 anywhere'], + ], + createSource({ + aliases: [ + { alias: 'alias1', aliasIsCommon: true, }, + { alias: 'alias2', aliasIsCommon: true, }, + ] + }), + ({ source, content }) => isAliasInTitle({ source, titleNormalized: normalizeText(content) }) + ); + }); + + test('aliasIsCommon: false', () => { + expectEntries( + [ + [true, 'title which includes alias1 anywhere'], + [true, 'title which includes alias2 anywhere'], + ], + createSource({ + aliases: [ + { alias: 'alias1', aliasIsCommon: false, }, + { alias: 'alias2', aliasIsCommon: false, }, + ] + }), + ({ source, content }) => isAliasInTitle({ source, titleNormalized: normalizeText(content) }) + ); + }); + }); + + describe('isAliasInBody', () => { + test('aliasIsCommon: true', () => { + expectEntries( + [ + [true, 'alias1: rest of the body'], + [true, 'body which includes (alias1)'], + [true, 'body which includes [alias1]'], + [true, 'post body with\nalias1: in it'], + [true, 'alias2: rest of the body'], + [true, 'body which includes (alias2)'], + [true, 'body which includes [alias2]'], + [true, 'post body with\nalias2: in it'], + [false, 'post body with\nalias1 in it'], + [false, 'body which includes alias1 anywhere'], + [false, 'body which includes alias2 anywhere'], + ], + createSource({ + aliases: [ + { alias: 'alias1', aliasIsCommon: true, }, + { alias: 'alias2', aliasIsCommon: true, }, + ] + }), + ({ source, content }) => isAliasInBody({ source, bodyNormalized: normalizeText(content) }) + ); + }); + + test('aliasIsCommon: false', () => { + expectEntries( + [ + [true, 'body which includes\nalias1\nanywhere'], + [true, 'body which includes\nalias2\nanywhere'], + [false, 'body which includes _alias1_ anywhere'], + [false, 'body which includes _alias2_ anywhere'], + ], + createSource({ + aliases: [ + { alias: 'alias1', aliasIsCommon: false, }, + { alias: 'alias2', aliasIsCommon: false, }, + ] + }), + ({ source, content }) => isAliasInBody({ source, bodyNormalized: normalizeText(content) }) + ); + }); + }); +}); + +describe('handles', () => { + test('isHandleInTitle', () => { + expectEntries( + [ + [true, '@handle1: rest of the title'], + [true, 'title which includes (handle1)'], + [true, 'title which includes [handle1]'], + [true, 'title which includes @handle1'], + [true, '@handle2: rest of the title'], + [true, 'title which includes (handle2)'], + [true, 'title which includes [handle2]'], + [true, 'title which includes @handle2'], + ], + createSource({ handles: ['handle1', 'handle2'] }), + ({ source, content }) => isHandleInTitle({ source, titleNormalized: normalizeText(content) }) + ); + }); + + test('isHandleInUrl', () => { + expectEntries( + [ + [true, new URL('https://twitter.com/handle1')], + [true, new URL('https://twitter.com/handle1/')], + [true, new URL('https://twitter.com/handle1/status/12345')], + [true, new URL('https://twitter.com/handle2')], + [true, new URL('https://twitter.com/handle2/')], + [true, new URL('https://twitter.com/handle2/status/12345')], + [true, new URL('https://x.com/handle1')], + [true, new URL('https://x.com/handle1/')], + [true, new URL('https://x.com/handle1/status/12345')], + [true, new URL('https://x.com/handle2')], + [true, new URL('https://x.com/handle2/')], + [true, new URL('https://x.com/handle2/status/12345')], + [false, new URL('https://example.com/handle1')], + [false, new URL('https://example.com/handle1/')], + [false, new URL('https://example.com/handle1/status/12345')], + [false, new URL('https://example.com/handle2')], + [false, new URL('https://example.com/handle2/')], + [false, new URL('https://example.com/handle2/status/12345')], + ], + createSource({ handles: ['handle1', 'handle2'] }), + ({ source, content }) => isHandleInUrl({ source, url: content }) + ); + }); + + test('isHandleInLinks', () => { + expectEntries( + [ + [true, [new URL('https://x.com/handle1')]], + [true, [new URL('https://x.com/handle1/')]], + [true, [new URL('https://x.com/handle1/path')]], + [true, [new URL('https://x.com/handle2')]], + [true, [new URL('https://x.com/handle2/')]], + [true, [new URL('https://x.com/handle2/path')]], + [true, [new URL('https://ignored.com/handle1'), new URL('https://x.com/handle1')]], + [true, [new URL('https://ignored.com/handle1'), new URL('https://x.com/handle1/')]], + [true, [new URL('https://ignored.com/handle1'), new URL('https://x.com/handle1/path')]], + [true, [new URL('https://ignored.com/handle1'), new URL('https://x.com/handle2')]], + [true, [new URL('https://ignored.com/handle1'), new URL('https://x.com/handle2/')]], + [true, [new URL('https://ignored.com/handle1'), new URL('https://x.com/handle2/path')]], + [false, [new URL('https://x.com/status/handle1')]], + [false, [new URL('https://twitter.com/handle123')]], + ], + createSource({ handles: ['handle1', 'handle2'] }), + ({ source, content }) => isHandleInLinks({ source, urls: content }) + ); + }); + + test('isHandleInBody', () => { + expectEntries( + [ + + [true, 'handle1: rest of the body'], + [true, '@handle1: rest of the body'], + [true, 'body which includes (handle1)'], + [true, 'body which includes [handle1]'], + [true, 'body which includes @handle1'], + [true, 'body which includes\nhandle1:'], + [true, 'handle2: rest of the body'], + [true, '@handle2: rest of the body'], + [true, 'body which includes (handle2)'], + [true, 'body which includes [handle2]'], + [true, 'body which includes @handle2'], + [true, 'body which includes\nhandle2:'], + [false, 'body which includes handle1'], + [false, 'body which includes handle2'], + ], + createSource({ handles: ['handle1', 'handle2'] }), + ({ source, content }) => isHandleInBody({ source, bodyNormalized: normalizeText(content) }) + ); + }); +}); + +describe('domains', () => { + test('isDomainInUrl', () => { + expectEntries( + [ + [true, new URL('https://foo.com')], + [true, new URL('https://www.foo.com')], + [true, new URL('https://sub1.foo.com')], + [true, new URL('https://sub1.sub2.foo.com')], + [true, new URL('https://bar.com')], + [true, new URL('https://www.bar.com')], + [true, new URL('https://sub1.bar.com')], + [true, new URL('https://sub1.sub2.bar.com')], + [false, new URL('https://foo.com.au')], + [false, new URL('https://bar.com.au')], + ], + createSource({ domains: ['foo.com', 'bar.com'] }), + ({ source, content }) => isDomainInUrl({ source, url: content }) + ); + }); + + test('isDomainInLinks', () => { + expectEntries( + [ + [true, [new URL('https://foo.com')]], + [true, [new URL('https://www.foo.com')]], + [true, [new URL('https://sub1.foo.com')]], + [true, [new URL('https://sub1.sub2.foo.com')]], + [true, [new URL('https://bar.com')]], + [true, [new URL('https://www.bar.com')]], + [true, [new URL('https://sub1.bar.com')]], + [true, [new URL('https://sub1.sub2.bar.com')]], + [true, [new URL('https://ignored.com'), new URL('https://foo.com')]], + [true, [new URL('https://ignored.com'), new URL('https://bar.com')]], + ], + createSource({ domains: ['foo.com', 'bar.com'] }), + ({ source, content }) => isDomainInLinks({ source, urls: content }) + ); + }); +}); \ No newline at end of file diff --git a/packages/devvit/test/sources.test.ts b/packages/devvit/test/sources.test.ts new file mode 100644 index 0000000..756148e --- /dev/null +++ b/packages/devvit/test/sources.test.ts @@ -0,0 +1,20 @@ +import sourceList from '@repo/db/data/devvit-sources.json'; +import { describe, expect, test } from 'vitest'; +import { settingsSchema } from '../src/index.js'; + +describe('zod schema safeParse', () => { + test('sources list json', () => { + /** + * Converting the source list to string is kinda pointless because + * because the import converts it to an object anyway. + * + * But this is only done for test purposes here so it's fine. + */ + const result = settingsSchema + .shape + .sources + .safeParse(JSON.stringify(sourceList)); + + expect(result.error).toBe(undefined); + }); +}); \ No newline at end of file diff --git a/packages/devvit/tsconfig.json b/packages/devvit/tsconfig.json index 3d16d86..ffe79a8 100644 --- a/packages/devvit/tsconfig.json +++ b/packages/devvit/tsconfig.json @@ -1,4 +1,4 @@ { "extends": "@devvit/public-api/devvit.tsconfig.json", - "include": ["src"] + "include": ["src", "test"] } diff --git a/packages/devvit/vitest.config.ts b/packages/devvit/vitest.config.ts new file mode 100644 index 0000000..fed6507 --- /dev/null +++ b/packages/devvit/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ['./test/*.test.ts'] + } +}); \ No newline at end of file