From c40248aac163875b72322de844474ff10bc0dab0 Mon Sep 17 00:00:00 2001 From: journey-ad Date: Mon, 29 Jul 2024 20:50:36 +0800 Subject: [PATCH] WIP: Support meilisearch --- app/api/graphql/service.ts | 362 ++++++++++++++++++++++++++++--------- lib/meilisearch.ts | 21 +++ package.json | 1 + tsconfig.json | 2 +- utils/index.ts | 6 + 5 files changed, 303 insertions(+), 89 deletions(-) create mode 100644 lib/meilisearch.ts diff --git a/app/api/graphql/service.ts b/app/api/graphql/service.ts index 4f08250..5f6f4e5 100644 --- a/app/api/graphql/service.ts +++ b/app/api/graphql/service.ts @@ -1,6 +1,8 @@ import { query } from "@/lib/pgdb"; import { jiebaCut } from "@/lib/jieba"; +import meiliClient from "@/lib/meilisearch"; import { SEARCH_KEYWORD_SPLIT_REGEX } from "@/config/constant"; +import { getTimestamp } from "@/utils/index"; type Torrent = { info_hash: Buffer; // The hash info of the torrent @@ -73,7 +75,7 @@ export function formatTorrent(row: Torrent) { } // Utility functions for query building -const buildOrderBy = (sortType: keyof typeof orderByMap) => { +const buildOrderBy = (sortType: "size" | "count" | "date") => { const orderByMap = { size: "torrents.size DESC", count: "COALESCE(torrents.files_count, 0) DESC", @@ -83,7 +85,9 @@ const buildOrderBy = (sortType: keyof typeof orderByMap) => { return orderByMap[sortType] || "torrents.created_at DESC"; }; -const buildTimeFilter = (filterTime: keyof typeof timeFilterMap) => { +const buildTimeFilter = ( + filterTime: "gt-1day" | "gt-7day" | "gt-31day" | "gt-365day", +) => { const timeFilterMap = { "gt-1day": "AND torrents.created_at > now() - interval '1 day'", "gt-7day": "AND torrents.created_at > now() - interval '1 week'", @@ -94,7 +98,14 @@ const buildTimeFilter = (filterTime: keyof typeof timeFilterMap) => { return timeFilterMap[filterTime] || ""; }; -const buildSizeFilter = (filterSize: keyof typeof sizeFilterMap) => { +const buildSizeFilter = ( + filterSize: + | "lt100mb" + | "gt100mb-lt500mb" + | "gt500mb-lt1gb" + | "gt1gb-lt5gb" + | "gt5gb", +) => { const sizeFilterMap = { lt100mb: "AND torrents.size < 100 * 1024 * 1024::bigint", "gt100mb-lt500mb": @@ -109,6 +120,74 @@ const buildSizeFilter = (filterSize: keyof typeof sizeFilterMap) => { return sizeFilterMap[filterSize] || ""; }; +// Build Meili Sort +const buildMeiliSort = (sortType: "size" | "count" | "date") => { + const sortMap = { + size: "size:desc", + count: "files_count:desc", + date: "created_at:asc", + }; + + const sort = [sortMap[sortType] || "created_at:desc"]; + + console.log(sort); + + return sort; +}; +// Build Meili Filter +const buildMeiliFilter = (queryInput: any) => { + const { sortType, filterTime, filterSize } = queryInput; + + let filterList = []; + + switch (filterTime) { + case "gt-1day": + filterList.push(`created_at > ${getTimestamp(-1, "day")}`); + break; + case "gt-7day": + filterList.push(`created_at > ${getTimestamp(-7, "day")}`); + break; + case "gt-31day": + filterList.push(`created_at > ${getTimestamp(-1, "month")}`); + break; + case "gt-365day": + filterList.push(`created_at > ${getTimestamp(-1, "year")}`); + break; + } + + switch (filterSize) { + case "lt100mb": + filterList.push(`size < ${100 * 1024 * 1024}`); + break; + case "gt100mb-lt500mb": + filterList.push( + `size >= ${100 * 1024 * 1024} AND size < ${500 * 1024 * 1024}`, + ); + break; + case "gt500mb-lt1gb": + filterList.push( + `size >= ${500 * 1024 * 1024} AND size < ${1 * 1024 * 1024 * 1024}`, + ); + break; + case "gt1gb-lt5gb": + filterList.push( + `size >= ${1 * 1024 * 1024 * 1024} AND size < ${5 * 1024 * 1024 * 1024}`, + ); + break; + case "gt5gb": + filterList.push(`size >= ${5 * 1024 * 1024 * 1024}`); + break; + } + + if (!filterList.length) return []; + + const filter = [filterList.join(" AND ")]; + + console.log(filter); + + return filter; +}; + const QUOTED_KEYWORD_REGEX = /"([^"]+)"/g; const extractKeywords = ( keyword: string, @@ -159,78 +238,42 @@ const extractKeywords = ( return keywords; }; -export async function search(_: any, { queryInput }: any) { - try { - console.info("-".repeat(50)); - console.info("search params", queryInput); - - // trim keyword - queryInput.keyword = queryInput.keyword.trim(); - - const no_result = { - keywords: [queryInput.keyword], - torrents: [], - total_count: 0, - has_more: false, - }; - - // Return an empty result if no keywords are provided - if (queryInput.keyword.length < 2) { - return no_result; - } +const dbsearch = async ({ queryInput }: any) => { + // Build SQL conditions and parameters + const orderBy = buildOrderBy(queryInput.sortType); + const timeFilter = buildTimeFilter(queryInput.filterTime); + const sizeFilter = buildSizeFilter(queryInput.filterSize); - const REGEX_HASH = /^[a-f0-9]{40}$/; + const keywords = extractKeywords(queryInput.keyword); - if (REGEX_HASH.test(queryInput.keyword)) { - const torrent = await torrentByHash(_, { hash: queryInput.keyword }); + // Construct the keyword filter condition + const requiredKeywords: string[] = []; + const optionalKeywords: string[] = []; - if (torrent) { - return { - keywords: [queryInput.keyword], - torrents: [torrent], - total_count: 1, - has_more: false, - }; - } + keywords.forEach(({ required }, i) => { + const condition = `torrents.name ILIKE $${i + 1}`; - return no_result; + if (required) { + requiredKeywords.push(condition); + } else { + optionalKeywords.push(condition); } + }); - // Build SQL conditions and parameters - const orderBy = buildOrderBy(queryInput.sortType); - const timeFilter = buildTimeFilter(queryInput.filterTime); - const sizeFilter = buildSizeFilter(queryInput.filterSize); - - const keywords = extractKeywords(queryInput.keyword); - - // Construct the keyword filter condition - const requiredKeywords: string[] = []; - const optionalKeywords: string[] = []; + const fullConditions = [...requiredKeywords]; - keywords.forEach(({ required }, i) => { - const condition = `torrents.name ILIKE $${i + 1}`; - - if (required) { - requiredKeywords.push(condition); - } else { - optionalKeywords.push(condition); - } - }); - - const fullConditions = [...requiredKeywords]; - - if (optionalKeywords.length > 0) { - optionalKeywords.push("TRUE"); - fullConditions.push(`(${optionalKeywords.join(" OR ")})`); - } + if (optionalKeywords.length > 0) { + optionalKeywords.push("TRUE"); + fullConditions.push(`(${optionalKeywords.join(" OR ")})`); + } - const keywordFilter = fullConditions.join(" AND "); + const keywordFilter = fullConditions.join(" AND "); - const keywordsParams = keywords.map(({ keyword }) => `%${keyword}%`); - const keywordsPlain = keywords.map(({ keyword }) => keyword); + const keywordsParams = keywords.map(({ keyword }) => `%${keyword}%`); + const keywordsPlain = keywords.map(({ keyword }) => keyword); - // SQL query to fetch filtered torrent data and files information - const sql = ` + // SQL query to fetch filtered torrent data and files information + const sql = ` -- 先查到符合过滤条件的数据 WITH filtered AS ( SELECT @@ -277,19 +320,19 @@ FROM filtered; -- 从过滤后的数据中查询 `; - const params = [...keywordsParams, queryInput.limit, queryInput.offset]; + const params = [...keywordsParams, queryInput.limit, queryInput.offset]; - console.debug("SQL:", sql, params); - console.debug( - "keywords:", - keywords.map((item, i) => ({ _: `$${i + 1}`, ...item })), - ); + console.debug("SQL:", sql, params); + console.debug( + "keywords:", + keywords.map((item, i) => ({ _: `$${i + 1}`, ...item })), + ); - const queryArr = [query(sql, params)]; + const queryArr = [query(sql, params)]; - // SQL query to get the total count if requested - if (queryInput.withTotalCount) { - const countSql = ` + // SQL query to get the total count if requested + if (queryInput.withTotalCount) { + const countSql = ` SELECT COUNT(*) AS total FROM ( SELECT 1 @@ -300,25 +343,125 @@ FROM ( ${sizeFilter} ) AS limited_total; `; - const countParams = [...keywordsParams]; + const countParams = [...keywordsParams]; - queryArr.push(query(countSql, countParams)); - } else { - queryArr.push(Promise.resolve({ rows: [{ total: 0 }] }) as any); + queryArr.push(query(countSql, countParams)); + } else { + queryArr.push(Promise.resolve({ rows: [{ total: 0 }] }) as any); + } + + // Execute queries and process results + const [{ rows: torrentsResp }, { rows: countResp }] = + await Promise.all(queryArr); + + const torrents = torrentsResp.map(formatTorrent); + const total_count = countResp[0].total; + + const has_more = + queryInput.withTotalCount && + queryInput.offset + queryInput.limit < total_count; + + return { keywords: keywordsPlain, torrents, total_count, has_more }; +}; + +// invisible characters for highlighting +const _MARK_TAG = ["\u200b\u200c", "\u200b\u200d"]; +// regex to extract keywords from name +const _MARK_TAG_RE = new RegExp(`${_MARK_TAG[0]}(.*?)${_MARK_TAG[1]}`, "g"); +const meilisearch = async ({ queryInput }: any) => { + const { keyword, limit, offset, sortType } = queryInput; + + const search = await meiliClient.torrents.search(keyword, { + offset, + limit, + sort: buildMeiliSort(sortType), + filter: buildMeiliFilter(queryInput), + attributesToSearchOn: ["name"], + attributesToRetrieve: ["info_hash", "name"], + attributesToHighlight: ["name"], + highlightPreTag: _MARK_TAG[0], + highlightPostTag: _MARK_TAG[1], + }); + + const { hits, estimatedTotalHits: total_count } = search; + + const hashes = hits.map((item: any) => item.info_hash as string); + const torrents = await torrentByHashBatch(null, { hashes }); + + const has_more = + queryInput.withTotalCount && + queryInput.offset + queryInput.limit < total_count; + + const keywordsSet = hits.reduce( + (acc: Set, item) => { + const { name } = item._formatted || {}; + + // extract keywords from name + if (name) { + [...name.matchAll(_MARK_TAG_RE)].forEach((match) => { + const [_, keyword] = match; + + if (keyword.length > 1 || !/[a-zA-Z0-9]/.test(keyword)) { + acc.add(keyword); + } + }); + } + + return acc; + }, + new Set([keyword]), + ); + + return { keywords: [...keywordsSet], torrents, total_count, has_more }; +}; + +const searchResolver = async ({ queryInput }: any) => { + // meilisearch resolver + if (meiliClient.enabled) { + return meilisearch({ queryInput }); + } else { + return dbsearch({ queryInput }); + } +}; + +export async function search(_: any, { queryInput }: any) { + try { + console.info("-".repeat(50)); + console.info("search params", queryInput); + + // trim keyword + queryInput.keyword = queryInput.keyword.trim(); + + const no_result = { + keywords: [queryInput.keyword], + torrents: [], + total_count: 0, + has_more: false, + }; + + // Return an empty result if no keywords are provided + if (queryInput.keyword.length < 2) { + return no_result; } - // Execute queries and process results - const [{ rows: torrentsResp }, { rows: countResp }] = - await Promise.all(queryArr); + const REGEX_HASH = /^[a-f0-9]{40}$/; - const torrents = torrentsResp.map(formatTorrent); - const total_count = countResp[0].total; + if (REGEX_HASH.test(queryInput.keyword)) { + const torrent = await torrentByHash(_, { hash: queryInput.keyword }); - const has_more = - queryInput.withTotalCount && - queryInput.offset + queryInput.limit < total_count; + if (torrent) { + return { + keywords: [queryInput.keyword], + torrents: [torrent], + total_count: 1, + has_more: false, + }; + } + + return no_result; + } - return { keywords: keywordsPlain, torrents, total_count, has_more }; + return searchResolver({ queryInput }); } catch (error) { console.error("Error in search resolver:", error); throw new Error("Failed to execute search query"); @@ -364,6 +507,49 @@ GROUP BY t.info_hash, t.name, t.size, t.created_at, t.updated_at, t.files_count; } } +export async function torrentByHashBatch( + _: any, + { hashes }: { hashes: string[] }, +) { + try { + const byteaHashes = hashes.map((hash) => Buffer.from(hash, "hex")); + + // SQL query to fetch torrent data and files information by hash + const sql = ` +SELECT + t.info_hash, + t.name, + t.size, + t.created_at, + t.updated_at, + t.files_count, + ( + SELECT json_agg(json_build_object( + 'index', f.index, + 'path', f.path, + 'size', f.size, + 'extension', f.extension + )) + FROM torrent_files f + WHERE f.info_hash = t.info_hash + ) AS files +FROM torrents t +WHERE t.info_hash = ANY($1) +GROUP BY t.info_hash; + `; + + const params = [byteaHashes]; + + const { rows } = await query(sql, params); + const torrents = rows.map(formatTorrent); + + return torrents; + } catch (error) { + console.error("Error in torrentByHashBatch resolver:", error); + throw new Error("Failed to fetch torrent by hash"); + } +} + export async function statsInfo() { try { const sql = ` diff --git a/lib/meilisearch.ts b/lib/meilisearch.ts new file mode 100644 index 0000000..56bf56a --- /dev/null +++ b/lib/meilisearch.ts @@ -0,0 +1,21 @@ +import { Index, MeiliSearch } from "meilisearch"; + +const MEILISEARCH_API_URL = process.env.MEILISEARCH_API_URL, + MEILISEARCH_API_KEY = process.env.MEILISEARCH_API_KEY; + +const meiliClient = { + enabled: MEILISEARCH_API_URL !== undefined, + client: null as unknown as MeiliSearch, + torrents: null as unknown as Index, +}; + +if (meiliClient.enabled) { + meiliClient.client = new MeiliSearch({ + host: MEILISEARCH_API_URL as string, + apiKey: MEILISEARCH_API_KEY, + }); + + meiliClient.torrents = meiliClient.client.index("torrents"); +} + +export default meiliClient; diff --git a/package.json b/package.json index b843a36..e9c3f33 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "graphql-tag": "^2.12.6", "intl-messageformat": "^10.5.0", "js-cookie": "^3.0.5", + "meilisearch": "^0.41.0", "next": "14.2.3", "next-intl": "^3.14.1", "next-themes": "^0.2.1", diff --git a/tsconfig.json b/tsconfig.json index e06a445..83fba45 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "target": "es5", + "target": "es2015", "lib": ["dom", "dom.iterable", "esnext"], "allowJs": true, "skipLibCheck": true, diff --git a/utils/index.ts b/utils/index.ts index bf7ee8b..a2d40fb 100644 --- a/utils/index.ts +++ b/utils/index.ts @@ -1,3 +1,5 @@ +import type { ManipulateType } from "dayjs"; + import dayjs from "dayjs"; import Cookie from "js-cookie"; @@ -47,6 +49,10 @@ export function formatDate( return dateStr; } +export function getTimestamp(diff = 0, unit = "second" as ManipulateType) { + return dayjs().add(diff, unit).unix(); +} + export function getSizeColor(size: number | string) { size = Number(size);