From 7e72827e85b77e2d8335907f005081884f462e2a Mon Sep 17 00:00:00 2001 From: shadrach Date: Mon, 27 Jan 2025 11:40:22 +0100 Subject: [PATCH 1/3] feat: cache external pub, send email notifications and return default --- desci-server/package.json | 1 + .../migration.sql | 2 + .../migration.sql | 2 + desci-server/prisma/schema.prisma | 10 +- .../controllers/nodes/externalPublications.ts | 201 +++++++----------- desci-server/src/controllers/nodes/publish.ts | 4 + desci-server/src/routes/v1/nodes.ts | 7 + .../src/scripts/pruneExternalPublications.ts | 13 ++ .../src/services/crossRef/definitions.ts | 6 + .../services/crossRef/externalPublication.ts | 196 +++++++++++++++++ .../templates/emails/ExternalPublications.tsx | 87 ++++++++ .../templates/emails/utils/emailRenderer.tsx | 4 + sync-server/src/index.ts | 8 +- 13 files changed, 403 insertions(+), 138 deletions(-) create mode 100644 desci-server/prisma/migrations/20250124124412_is_verified_publication/migration.sql create mode 100644 desci-server/prisma/migrations/20250127072323_externalpub_verified_at/migration.sql create mode 100644 desci-server/src/scripts/pruneExternalPublications.ts create mode 100644 desci-server/src/services/crossRef/externalPublication.ts create mode 100644 desci-server/src/templates/emails/ExternalPublications.tsx diff --git a/desci-server/package.json b/desci-server/package.json index d0e870248..704f18861 100755 --- a/desci-server/package.json +++ b/desci-server/package.json @@ -29,6 +29,7 @@ "script:backfill-annotations": "debug=* node --no-warnings --enable-source-maps --loader ts-node/esm ./src/scripts/backfill-annotations.ts", "script:prune-auth-tokens": "debug=* node --no-warnings --enable-source-maps --loader ts-node/esm ./src/scripts/prune-auth-tokens.ts", "script:backfill-radar": "debug=* node --no-warnings --enable-source-maps --loader ts-node/esm ./src/scripts/backfill-radar.ts", + "script:pruneExternalPublications": "debug=* node --no-warnings --enable-source-maps --loader ts-node/esm ./src/scripts/pruneExternalPublications.ts", "build": "rimraf dist && tsc && yarn copy-files; if [ \"$SENTRY_AUTH_TOKEN\" ]; then yarn sentry:sourcemaps; else echo 'SENTRY_AUTH_TOKEN not set, sourcemaps will not upload'; fi", "build:worker": "cd ../sync-server && ./scripts/build.sh test", "copy-files": "copyfiles -u 1 src/**/*.cjs dist/", diff --git a/desci-server/prisma/migrations/20250124124412_is_verified_publication/migration.sql b/desci-server/prisma/migrations/20250124124412_is_verified_publication/migration.sql new file mode 100644 index 000000000..55016f6c3 --- /dev/null +++ b/desci-server/prisma/migrations/20250124124412_is_verified_publication/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "ExternalPublications" ADD COLUMN "isVerified" BOOLEAN NOT NULL DEFAULT true; diff --git a/desci-server/prisma/migrations/20250127072323_externalpub_verified_at/migration.sql b/desci-server/prisma/migrations/20250127072323_externalpub_verified_at/migration.sql new file mode 100644 index 000000000..9b7099c95 --- /dev/null +++ b/desci-server/prisma/migrations/20250127072323_externalpub_verified_at/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "ExternalPublications" ADD COLUMN "verifiedAt" TIMESTAMP(3); diff --git a/desci-server/prisma/schema.prisma b/desci-server/prisma/schema.prisma index 29cba145c..4668647ac 100755 --- a/desci-server/prisma/schema.prisma +++ b/desci-server/prisma/schema.prisma @@ -986,16 +986,18 @@ model PublishStatus { } model ExternalPublications { - id Int @id @default(autoincrement()) + id Int @id @default(autoincrement()) uuid String - node Node @relation(fields: [uuid], references: [uuid]) + node Node @relation(fields: [uuid], references: [uuid]) score Float doi String publisher String publishYear String sourceUrl String - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt + isVerified Boolean @default(true) + verifiedAt DateTime? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt // @@unique([uuid, publisher]) } diff --git a/desci-server/src/controllers/nodes/externalPublications.ts b/desci-server/src/controllers/nodes/externalPublications.ts index 4984cd6da..6b3d195c1 100644 --- a/desci-server/src/controllers/nodes/externalPublications.ts +++ b/desci-server/src/controllers/nodes/externalPublications.ts @@ -8,9 +8,11 @@ import { NotFoundError } from '../../core/ApiError.js'; import { SuccessMessageResponse, SuccessResponse } from '../../core/ApiResponse.js'; import { logger as parentLogger } from '../../logger.js'; import { RequestWithNode } from '../../middleware/authorisation.js'; -import { crossRefClient } from '../../services/index.js'; -import { NodeUuid } from '../../services/manifestRepo.js'; -import repoService from '../../services/repoService.js'; +import { redisClient } from '../../redisClient.js'; +import { + getExternalPublications, + sendExternalPublicationsNotification, +} from '../../services/crossRef/externalPublication.js'; import { ensureUuidEndsWithDot } from '../../utils.js'; const logger = parentLogger.child({ module: 'ExternalPublications' }); @@ -36,140 +38,64 @@ export const addExternalPublicationsSchema = z.object({ }), }); +export const verifyExternalPublicationSchema = z.object({ + params: z.object({ + // quickly disqualify false uuid strings + uuid: z.string().min(10), + }), + body: z.object({ + verify: z.boolean(), + id: z.coerce.number(), + }), +}); + export const externalPublications = async (req: RequestWithNode, res: Response, _next: NextFunction) => { const { uuid } = req.params as z.infer['params']; const node = await prisma.node.findFirst({ where: { uuid: ensureUuidEndsWithDot(uuid) } }); if (!node) throw new NotFoundError(`Node ${uuid} not found`); - const userIsNodeOwner = req.user?.id === node?.ownerId; - - logger.trace({ uuid, userIsNodeOwner }); - - const externalPublication = await prisma.externalPublications.findMany({ + const userIsOwner = node.ownerId === req?.user?.id; + const externalPublications = await prisma.externalPublications.findMany({ where: { uuid: ensureUuidEndsWithDot(uuid) }, }); - if (externalPublication.length > 0) return new SuccessResponse(externalPublication).send(res); - - // return empty list if user is not node owner - if (!userIsNodeOwner) return new SuccessResponse([]).send(res); - - const manifest = await repoService.getDraftManifest({ uuid: uuid as NodeUuid, documentId: node.manifestDocumentId }); - const data = await crossRefClient.searchWorks({ queryTitle: manifest?.title }); - - if (data.length > 0) { - const titleSearcher = new Searcher(data, { keySelector: (entry) => entry.title }); - const titleResult = titleSearcher.search(manifest.title, { returnMatchData: true }); - logger.trace( - { - data: titleResult.map((data) => ({ - title: data.item.title, - publisher: data.item.publisher, - source_url: data.item?.resource?.primary?.URL || data.item.URL || '', - doi: data.item.DOI, - key: data.key, - match: data.match, - score: data.score, - })), - }, - 'Title search result', - ); - - const descSearcher = new Searcher(data, { keySelector: (entry) => entry?.abstract ?? '' }); - const descResult = descSearcher.search(manifest.description ?? '', { returnMatchData: true }); - logger.trace( - { - data: descResult.map((data) => ({ - title: data.item.title, - key: data.key, - match: data.match, - score: data.score, - })), - }, - 'Abstract search result', - ); - - const authorsSearchScores = data.map((work) => { - const authorSearcher = new Searcher(work.author, { keySelector: (entry) => `${entry.given} ${entry.family}` }); - - const nodeAuthorsMatch = manifest.authors.map((author) => - authorSearcher.search(author.name, { returnMatchData: true }), - ); - return { - publisher: work.publisher, - score: nodeAuthorsMatch.flat().reduce((total, match) => (total += match.score), 0) / manifest.authors.length, - match: nodeAuthorsMatch.flat().map((data) => ({ - key: data.key, - match: data.match, - score: data.score, - author: data.item, - publisher: work.publisher, - doi: work.DOI, - })), - }; - }); - - logger.trace( - { - data: descResult.map((data) => ({ - title: data.item.title, - key: data.key, - match: data.match, - score: data.score, - })), - }, - 'Authors search result', - ); - - const publications = data - .map((data) => ({ - publisher: data.publisher, - sourceUrl: data?.resource?.primary?.URL || data.URL || '', - doi: data.DOI, - 'is-referenced-by-count': data['is-referenced-by-count'] ?? 0, - publishYear: - data.published['date-parts']?.[0]?.[0].toString() ?? - data.license - .map((licence) => licence.start['date-parts']?.[0]?.[0]) - .filter(Boolean)?.[0] - .toString(), - title: titleResult - .filter((res) => res.item.publisher === data.publisher) - .map((data) => ({ - title: data.item.title, - key: data.key, - match: data.match, - score: data.score, - }))?.[0], - abstract: descResult - .filter((res) => res.item.publisher === data.publisher) - .map((data) => ({ - key: data.key, - match: data.match, - score: data.score, - abstract: data.item?.abstract ?? '', - }))?.[0], - authors: authorsSearchScores - .filter((res) => res.publisher === data.publisher) - .map((data) => ({ - score: data.score, - authors: data.match, - }))?.[0], - })) - .map((publication) => ({ - ...publication, - score: - ((publication.title?.score ?? 0) + (publication.abstract?.score ?? 0) + (publication.authors?.score ?? 0)) / - 3, - })) - .filter((entry) => entry.score >= 0.8); - - logger.trace({ publications, uuid }, 'externalPublications'); - - if (publications.length > 0) return new SuccessResponse(publications).send(res); - } - - return new SuccessResponse([]).send(res); + logger.trace({ externalPublications }, 'externalPublications'); + if (externalPublications.length == 1 && !externalPublications[0].verifiedAt) + return new SuccessResponse(externalPublications).send(res); + + const nonVerified = externalPublications.every((pub) => !pub.isVerified); + if (nonVerified && !userIsOwner) return new SuccessResponse(externalPublications).send(res); + + if (externalPublications.length > 1) + return new SuccessResponse(externalPublications.filter((pub) => pub.isVerified)).send(res); + + const isChecked = await redisClient.get(`external-pub-checked-${ensureUuidEndsWithDot(uuid)}`); + if (isChecked === 'true') return new SuccessResponse(externalPublications).send(res); + + const publications = await getExternalPublications(node); + + await redisClient.set(`external-pub-checked-${ensureUuidEndsWithDot(uuid)}`, 'true'); + + const entries = await prisma.$transaction( + publications.map((pub) => + prisma.externalPublications.upsert({ + update: {}, + where: {}, + create: { + doi: pub.doi, + score: pub.score, + sourceUrl: pub.sourceUrl, + publisher: pub.publisher, + publishYear: pub.publishYear, + uuid: ensureUuidEndsWithDot(node.uuid), + isVerified: false, + }, + }), + ), + ); + + sendExternalPublicationsNotification(node); + return new SuccessResponse(entries).send(res); }; export const addExternalPublication = async (req: RequestWithNode, res: Response, _next: NextFunction) => { @@ -186,8 +112,23 @@ export const addExternalPublication = async (req: RequestWithNode, res: Response if (exists) return new SuccessMessageResponse().send(res); const entry = await prisma.externalPublications.create({ - data: { doi, score, sourceUrl, publisher, publishYear, uuid: ensureUuidEndsWithDot(uuid) }, + data: { doi, score, sourceUrl, publisher, publishYear, uuid: ensureUuidEndsWithDot(uuid), isVerified: true }, }); return new SuccessResponse(entry).send(res); }; + +export const verifyExternalPublication = async (req: RequestWithNode, res: Response, _next: NextFunction) => { + const { uuid } = req.params as z.infer['params']; + const { verify, id } = req.body as z.infer['body']; + + const node = await prisma.node.findFirst({ where: { uuid: ensureUuidEndsWithDot(uuid) } }); + if (!node) throw new NotFoundError(`Node ${uuid} not found`); + + await prisma.externalPublications.update({ + where: { id: parseInt(id.toString()) }, + data: { isVerified: verify, verifiedAt: new Date() }, + }); + + return new SuccessMessageResponse().send(res); +}; diff --git a/desci-server/src/controllers/nodes/publish.ts b/desci-server/src/controllers/nodes/publish.ts index d09817834..d104f78e9 100644 --- a/desci-server/src/controllers/nodes/publish.ts +++ b/desci-server/src/controllers/nodes/publish.ts @@ -8,6 +8,7 @@ import { logger as parentLogger } from '../../logger.js'; import { delFromCache } from '../../redisClient.js'; import { attestationService } from '../../services/Attestation.js'; import { directStreamLookup } from '../../services/ceramic.js'; +import { sendExternalPublicationsNotification } from '../../services/crossRef/externalPublication.js'; import { getManifestByCid } from '../../services/data/processing.js'; import { getTargetDpidUrl } from '../../services/fixDpid.js'; import { doiService } from '../../services/index.js'; @@ -183,6 +184,9 @@ export const publish = async (req: PublishRequest, res: Response } } + // trigger external publications email if any + sendExternalPublicationsNotification(node); + return res.send({ ok: true, dpid: dpidAlias ?? parseInt(manifest.dpid?.id), diff --git a/desci-server/src/routes/v1/nodes.ts b/desci-server/src/routes/v1/nodes.ts index be7445ca4..28f61e006 100755 --- a/desci-server/src/routes/v1/nodes.ts +++ b/desci-server/src/routes/v1/nodes.ts @@ -24,6 +24,8 @@ import { addExternalPublicationsSchema, externalPublications, externalPublicationsSchema, + verifyExternalPublication, + verifyExternalPublicationSchema, } from '../../controllers/nodes/externalPublications.js'; import { feed } from '../../controllers/nodes/feed.js'; import { frontmatterPreview } from '../../controllers/nodes/frontmatterPreview.js'; @@ -173,6 +175,11 @@ router.post( [validate(addExternalPublicationsSchema), ensureUser, ensureNodeAccess], asyncHandler(addExternalPublication), ); +router.post( + '/:uuid/verify-publication', + [validate(verifyExternalPublicationSchema), ensureUser, ensureNodeAccess], + asyncHandler(verifyExternalPublication), +); router.get('/:uuid/comments', [ensureUser, validate(getCommentsSchema)], asyncHandler(getGeneralComments)); router.get('/:uuid/comments/:commentId/vote', [ensureUser, validate(postCommentVoteSchema)], asyncHandler(getUserVote)); diff --git a/desci-server/src/scripts/pruneExternalPublications.ts b/desci-server/src/scripts/pruneExternalPublications.ts new file mode 100644 index 000000000..2cb8d497e --- /dev/null +++ b/desci-server/src/scripts/pruneExternalPublications.ts @@ -0,0 +1,13 @@ +import { prisma } from '../client.js'; + +const main = async () => { + const rows = await prisma.externalPublications.deleteMany({ + // where: { verifiedAt: null }, + }); + + return rows; +}; + +main() + .then((result) => console.log('ExternalPublications Pruned', result)) + .catch((err) => console.log('Error running script ', err)); diff --git a/desci-server/src/services/crossRef/definitions.ts b/desci-server/src/services/crossRef/definitions.ts index 318237a06..461c801ac 100644 --- a/desci-server/src/services/crossRef/definitions.ts +++ b/desci-server/src/services/crossRef/definitions.ts @@ -51,6 +51,12 @@ export interface Work { URL: string; // 'http://onlinelibrary.wiley.com/termsAndConditions#vor'; }, ]; + type: 'journal-article' | 'posted-content'; + 'short-container-title'?: string[]; + 'container-title'?: string[]; + institution?: Array<{ + name: string; + }>; } export interface Author { diff --git a/desci-server/src/services/crossRef/externalPublication.ts b/desci-server/src/services/crossRef/externalPublication.ts new file mode 100644 index 000000000..e1fef8517 --- /dev/null +++ b/desci-server/src/services/crossRef/externalPublication.ts @@ -0,0 +1,196 @@ +import { ResearchObjectV1 } from '@desci-labs/desci-models'; +import { Node } from '@prisma/client'; +import sgMail from '@sendgrid/mail'; +import { Searcher } from 'fast-fuzzy'; + +import { prisma } from '../../client.js'; +import { logger } from '../../logger.js'; +import { ExternalPublicationsEmailHtml } from '../../templates/emails/utils/emailRenderer.js'; +import { ensureUuidEndsWithDot } from '../../utils.js'; +import { crossRefClient } from '../index.js'; +import { NodeUuid } from '../manifestRepo.js'; +import repoService from '../repoService.js'; + +import { Work } from './definitions.js'; + +sgMail.setApiKey(process.env.SENDGRID_API_KEY); + +const getPublisherTitle = (data: Work): string => + data?.['container-title']?.[0] || + data?.['short-container-title']?.[0] || + data?.institution?.[0]?.name || + data?.publisher; + +export const getExternalPublications = async (node: Node) => { + const manifest = await repoService.getDraftManifest({ + uuid: node.uuid as NodeUuid, + documentId: node.manifestDocumentId, + }); + let data = await crossRefClient.searchWorks({ queryTitle: manifest?.title }); + + data = data?.filter((works) => works.type === 'journal-article'); + + if (!data || data.length === 0) return []; + + const titleSearcher = new Searcher(data, { keySelector: (entry) => entry.title }); + const titleResult = titleSearcher.search(manifest.title, { returnMatchData: true }); + // logger.trace( + // { + // data: titleResult.map((data) => ({ + // title: data.item.title, + // publisher: data.item.publisher, + // source_url: data.item?.resource?.primary?.URL || data.item.URL || '', + // doi: data.item.DOI, + // key: data.key, + // match: data.match, + // score: data.score, + // })), + // }, + // 'Title search result', + // ); + + const descSearcher = new Searcher(data, { keySelector: (entry) => entry?.abstract ?? '' }); + const descResult = descSearcher.search(manifest.description ?? '', { returnMatchData: true }); + // logger.trace( + // { + // data: descResult.map((data) => ({ + // title: data.item.title, + // key: data.key, + // match: data.match, + // score: data.score, + // publisher: data.item.publisher, + // })), + // }, + // 'Abstract search result', + // ); + + const authorsSearchScores = data.map((work) => { + const authorSearcher = new Searcher(work.author, { keySelector: (entry) => `${entry.given} ${entry.family}` }); + + const nodeAuthorsMatch = manifest.authors.map((author) => + authorSearcher.search(author.name, { returnMatchData: true }), + ); + return { + publisher: getPublisherTitle(work), + score: nodeAuthorsMatch.flat().reduce((total, match) => (total += match.score), 0) / manifest.authors.length, + match: nodeAuthorsMatch.flat().map((data) => ({ + key: data.key, + match: data.match, + score: data.score, + author: data.item, + publisher: getPublisherTitle(work), + doi: work.DOI, + })), + }; + }); + + // logger.trace( + // { + // data: descResult.map((data) => ({ + // title: data.item.title, + // key: data.key, + // match: data.match, + // score: data.score, + // publisher: data.item.publisher, + // })), + // }, + // 'Authors search result', + // ); + + const publications = data + .map((data) => ({ + publisher: getPublisherTitle(data), + sourceUrl: data?.resource?.primary?.URL || data.URL || '', + doi: data.DOI, + isVerified: false, + 'is-referenced-by-count': data['is-referenced-by-count'] ?? 0, + publishYear: + data.published['date-parts']?.[0]?.[0].toString() ?? + data.license + .map((licence) => licence.start['date-parts']?.[0]?.[0]) + .filter(Boolean)?.[0] + .toString(), + title: titleResult + .filter((res) => getPublisherTitle(res.item) === getPublisherTitle(data)) + .map((data) => ({ + title: data.item.title, + key: data.key, + match: data.match, + score: data.score, + }))?.[0], + abstract: descResult + .filter((res) => getPublisherTitle(res.item) === getPublisherTitle(data)) + .map((data) => ({ + key: data.key, + match: data.match, + score: data.score, + abstract: data.item?.abstract ?? '', + }))?.[0], + authors: authorsSearchScores + .filter((res) => res.publisher === getPublisherTitle(data)) + .map((data) => ({ + score: data.score, + authors: data.match, + }))?.[0], + })) + .map((publication) => ({ + ...publication, + score: + ((publication.title?.score ?? 0) + (publication.abstract?.score ?? 0) + (publication.authors?.score ?? 0)) / 3, + })) + .filter((entry) => entry.score >= 0.8); + + logger.trace({ publications }, 'externalPublications'); + + return publications; +}; + +export const sendExternalPublicationsNotification = async (node: Node) => { + const publications = await getExternalPublications(node); + + if (!publications.length) return; + + await prisma.externalPublications.createMany({ + skipDuplicates: true, + data: publications.map((pub) => ({ + doi: pub.doi, + score: pub.score, + sourceUrl: pub.sourceUrl, + publisher: pub.publisher, + publishYear: pub.publishYear, + uuid: ensureUuidEndsWithDot(node.uuid), + isVerified: false, + })), + }); + + // send email to node owner about potential publications + const user = await prisma.user.findFirst({ where: { id: node.ownerId } }); + const message = { + to: user.email, + from: 'no-reply@desci.com', + subject: `[nodes.desci.com] Verify your external publications`, + text: `${ + publications.length > 1 + ? `We found a similar publications to ${node.title}, View your publication to verify external publications` + : `We linked ${publications.length} external publications from publishers like ${publications[0].publisher} to your node, open your node to verify the external publication.` + }`, + html: ExternalPublicationsEmailHtml({ + dpid: node.dpidAlias.toString(), + dpidPath: `${process.env.DAPP_URL}/dpid/${node.dpidAlias}`, + publisherName: publications[0].publisher, + multiple: publications.length > 1, + }), + }; + + try { + logger.info({ message, NODE_ENV: process.env.NODE_ENV }, '[EMAIL]:: ExternalPublications EMAIL'); + if (process.env.SHOULD_SEND_EMAIL) { + const response = await sgMail.send(message); + logger.info(response, '[EMAIL]:: Response'); + } else { + logger.info({ nodeEnv: process.env.NODE_ENV }, message.subject); + } + } catch (err) { + logger.info({ err }, '[ExternalPublications EMAIL]::ERROR'); + } +}; diff --git a/desci-server/src/templates/emails/ExternalPublications.tsx b/desci-server/src/templates/emails/ExternalPublications.tsx new file mode 100644 index 000000000..9620c51ed --- /dev/null +++ b/desci-server/src/templates/emails/ExternalPublications.tsx @@ -0,0 +1,87 @@ +import { + Body, + Container, + Column, + Head, + Heading, + Html, + Preview, + Row, + Text, + Button, + Section, + render, +} from '@react-email/components'; +import * as React from 'react'; + +import MainLayout from './MainLayout.js'; + +export interface ExternalPublicationsEmailProps { + dpid: string; + dpidPath: string; + publisherName: string; + multiple: boolean; +} + +export const ExternalPublicationsEmail = ({ + dpid, + dpidPath, + publisherName, + multiple, +}: ExternalPublicationsEmailProps) => ( + + + + + External publication{multiple ? `s` : ``} found DPID://{dpid} + + + + + {multiple + ? `View your publication to verify external publications` + : `We linked an external publication from ${publisherName} to your node`} + +
+ +
+ + + + +); + +export default ExternalPublicationsEmail; + +const main = { + backgroundColor: '#ffffff', + margin: '0 auto', + fontFamily: 'Inter, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif', +}; + +const container = { + margin: '0 auto', + padding: '0px 20px', +}; + +const h1 = { + // color: '#000000', + fontSize: '30px', + fontWeight: '700', + margin: '30px 0', + padding: '0', + lineHeight: '42px', +}; diff --git a/desci-server/src/templates/emails/utils/emailRenderer.tsx b/desci-server/src/templates/emails/utils/emailRenderer.tsx index 19ab238a4..57cf626b9 100644 --- a/desci-server/src/templates/emails/utils/emailRenderer.tsx +++ b/desci-server/src/templates/emails/utils/emailRenderer.tsx @@ -3,6 +3,7 @@ import { render } from '@react-email/components'; import AttestationClaimedEmail, { AttestationClaimedEmailProps } from '../AttestationClaimed.js'; import ContributorInvite, { ContributorInviteEmailProps } from '../ContributorInvite.js'; import DoiMintedEmail, { DoiMintedEmailProps } from '../DoiMinted.js'; +import ExternalPublications, { ExternalPublicationsEmailProps } from '../ExternalPublications.js'; import MagicCodeEmail, { MagicCodeEmailProps } from '../MagicCode.js'; import NodeUpdated, { NodeUpdatedEmailProps } from '../NodeUpdated.js'; import SubmissionPackage, { SubmissionPackageEmailProps } from '../SubmissionPackage.js'; @@ -26,4 +27,7 @@ export const NodeUpdatedEmailHtml = (props: NodeUpdatedEmailProps) => render(Nod export const SubmissionPackageEmailHtml = (props: SubmissionPackageEmailProps) => render(SubmissionPackage(props)); +export const ExternalPublicationsEmailHtml = (props: ExternalPublicationsEmailProps) => + render(ExternalPublications(props)); + export const DoiMintedEmailHtml = (props: DoiMintedEmailProps) => render(DoiMintedEmail(props)); diff --git a/sync-server/src/index.ts b/sync-server/src/index.ts index 2ec089785..93942bab1 100644 --- a/sync-server/src/index.ts +++ b/sync-server/src/index.ts @@ -141,7 +141,7 @@ export class AutomergeServer extends PartyServer { } async onRequest(request: Request) { - console.log('Incoming Request', request.url); + console.log('Incoming Request', request.method, request.url); if (request.headers.get('x-api-key') != this.env.API_TOKEN) { console.log('[Error]::Api key error', { api: this.env.API_TOKEN, key: request.headers.get('x-api-key') }); @@ -157,7 +157,7 @@ export class AutomergeServer extends PartyServer { return this.getLatestDocument(request); } - return new Response('Method not allowed', { status: 405 }); + return new Response('Method not allowed', { status: 404 }); } onMessage(connection: Connection, message: WSMessage): void | Promise { @@ -179,7 +179,7 @@ export class AutomergeServer extends PartyServer { async getLatestDocument(request) { const documentId = request.url.split('/').pop() as DocumentId; - console.log('getLatestDocument: ', { documentId }); + console.log(`getLatestDocument: ${documentId}`, { documentId }); if (!documentId) { console.error('No DocumentID found'); return new Response(JSON.stringify({ ok: false, message: 'Invalid body' }), { status: 400 }); @@ -263,7 +263,7 @@ async function handleCreateDocument(request: Request, env: Env) { export default { fetch(request: Request, env) { - console.log('Fetch handler: ', env, 'request api key: ', request.headers.get('x-api')); + console.log('Fetch handler: ', request.url, env); if (request.url.includes('/api/documents') && request.method.toLowerCase() === 'post') return handleCreateDocument(request, env); From 7c74e4a821676191218bcc9a97babe453cdb91e8 Mon Sep 17 00:00:00 2001 From: shadrach Date: Mon, 27 Jan 2025 13:05:37 +0100 Subject: [PATCH 2/3] fix: duplicate entries bug --- .../controllers/nodes/externalPublications.ts | 73 +++++++++++++------ desci-server/src/controllers/nodes/publish.ts | 7 +- .../services/crossRef/externalPublication.ts | 63 ++-------------- 3 files changed, 64 insertions(+), 79 deletions(-) diff --git a/desci-server/src/controllers/nodes/externalPublications.ts b/desci-server/src/controllers/nodes/externalPublications.ts index 6b3d195c1..3561722ad 100644 --- a/desci-server/src/controllers/nodes/externalPublications.ts +++ b/desci-server/src/controllers/nodes/externalPublications.ts @@ -13,7 +13,7 @@ import { getExternalPublications, sendExternalPublicationsNotification, } from '../../services/crossRef/externalPublication.js'; -import { ensureUuidEndsWithDot } from '../../utils.js'; +import { asyncMap, ensureUuidEndsWithDot } from '../../utils.js'; const logger = parentLogger.child({ module: 'ExternalPublications' }); export const externalPublicationsSchema = z.object({ @@ -59,12 +59,13 @@ export const externalPublications = async (req: RequestWithNode, res: Response, where: { uuid: ensureUuidEndsWithDot(uuid) }, }); - logger.trace({ externalPublications }, 'externalPublications'); - if (externalPublications.length == 1 && !externalPublications[0].verifiedAt) - return new SuccessResponse(externalPublications).send(res); + // logger.trace({ externalPublications }, 'externalPublications'); + // if (externalPublications.length == 1 && !externalPublications[0].verifiedAt) + // return new SuccessResponse(externalPublications).send(res); const nonVerified = externalPublications.every((pub) => !pub.isVerified); - if (nonVerified && !userIsOwner) return new SuccessResponse(externalPublications).send(res); + if (nonVerified && !userIsOwner) + return new SuccessResponse(externalPublications.sort((a, b) => b.score - a.score).slice(0, 1)).send(res); if (externalPublications.length > 1) return new SuccessResponse(externalPublications.filter((pub) => pub.isVerified)).send(res); @@ -76,23 +77,51 @@ export const externalPublications = async (req: RequestWithNode, res: Response, await redisClient.set(`external-pub-checked-${ensureUuidEndsWithDot(uuid)}`, 'true'); - const entries = await prisma.$transaction( - publications.map((pub) => - prisma.externalPublications.upsert({ - update: {}, - where: {}, - create: { - doi: pub.doi, - score: pub.score, - sourceUrl: pub.sourceUrl, - publisher: pub.publisher, - publishYear: pub.publishYear, - uuid: ensureUuidEndsWithDot(node.uuid), - isVerified: false, - }, - }), - ), - ); + // const entries = await prisma.$transaction( + // publications.map((pub) => + // prisma.externalPublications.upsert({ + // update: { + // doi: pub.doi, + // score: pub.score, + // sourceUrl: pub.sourceUrl, + // publisher: pub.publisher, + // publishYear: pub.publishYear, + // uuid: ensureUuidEndsWithDot(node.uuid), + // isVerified: false, + // }, + // where: { id: -1 }, + // create: { + // doi: pub.doi, + // score: pub.score, + // sourceUrl: pub.sourceUrl, + // publisher: pub.publisher, + // publishYear: pub.publishYear, + // uuid: ensureUuidEndsWithDot(node.uuid), + // isVerified: false, + // }, + // }), + // ), + // ); + + const entries = await asyncMap(publications, async (pub) => { + const exists = await prisma.externalPublications.findFirst({ + where: { AND: { uuid: ensureUuidEndsWithDot(uuid), doi: pub.doi } }, + }); + logger.trace({ pub: { publisher: pub.publisher, doi: pub.doi }, exists }, '[pub exists]'); + if (exists) return exists; + + return prisma.externalPublications.create({ + data: { + doi: pub.doi, + score: pub.score, + sourceUrl: pub.sourceUrl, + publisher: pub.publisher, + publishYear: pub.publishYear, + uuid: ensureUuidEndsWithDot(node.uuid), + isVerified: false, + }, + }); + }); sendExternalPublicationsNotification(node); return new SuccessResponse(entries).send(res); diff --git a/desci-server/src/controllers/nodes/publish.ts b/desci-server/src/controllers/nodes/publish.ts index d104f78e9..e2e0136e3 100644 --- a/desci-server/src/controllers/nodes/publish.ts +++ b/desci-server/src/controllers/nodes/publish.ts @@ -8,7 +8,10 @@ import { logger as parentLogger } from '../../logger.js'; import { delFromCache } from '../../redisClient.js'; import { attestationService } from '../../services/Attestation.js'; import { directStreamLookup } from '../../services/ceramic.js'; -import { sendExternalPublicationsNotification } from '../../services/crossRef/externalPublication.js'; +import { + checkExternalPublications, + sendExternalPublicationsNotification, +} from '../../services/crossRef/externalPublication.js'; import { getManifestByCid } from '../../services/data/processing.js'; import { getTargetDpidUrl } from '../../services/fixDpid.js'; import { doiService } from '../../services/index.js'; @@ -185,7 +188,7 @@ export const publish = async (req: PublishRequest, res: Response } // trigger external publications email if any - sendExternalPublicationsNotification(node); + checkExternalPublications(node).then((_) => sendExternalPublicationsNotification(node)); return res.send({ ok: true, diff --git a/desci-server/src/services/crossRef/externalPublication.ts b/desci-server/src/services/crossRef/externalPublication.ts index e1fef8517..4610dd5e2 100644 --- a/desci-server/src/services/crossRef/externalPublication.ts +++ b/desci-server/src/services/crossRef/externalPublication.ts @@ -1,5 +1,5 @@ import { ResearchObjectV1 } from '@desci-labs/desci-models'; -import { Node } from '@prisma/client'; +import { ExternalPublications, Node } from '@prisma/client'; import sgMail from '@sendgrid/mail'; import { Searcher } from 'fast-fuzzy'; @@ -34,35 +34,9 @@ export const getExternalPublications = async (node: Node) => { const titleSearcher = new Searcher(data, { keySelector: (entry) => entry.title }); const titleResult = titleSearcher.search(manifest.title, { returnMatchData: true }); - // logger.trace( - // { - // data: titleResult.map((data) => ({ - // title: data.item.title, - // publisher: data.item.publisher, - // source_url: data.item?.resource?.primary?.URL || data.item.URL || '', - // doi: data.item.DOI, - // key: data.key, - // match: data.match, - // score: data.score, - // })), - // }, - // 'Title search result', - // ); const descSearcher = new Searcher(data, { keySelector: (entry) => entry?.abstract ?? '' }); const descResult = descSearcher.search(manifest.description ?? '', { returnMatchData: true }); - // logger.trace( - // { - // data: descResult.map((data) => ({ - // title: data.item.title, - // key: data.key, - // match: data.match, - // score: data.score, - // publisher: data.item.publisher, - // })), - // }, - // 'Abstract search result', - // ); const authorsSearchScores = data.map((work) => { const authorSearcher = new Searcher(work.author, { keySelector: (entry) => `${entry.given} ${entry.family}` }); @@ -84,19 +58,6 @@ export const getExternalPublications = async (node: Node) => { }; }); - // logger.trace( - // { - // data: descResult.map((data) => ({ - // title: data.item.title, - // key: data.key, - // match: data.match, - // score: data.score, - // publisher: data.item.publisher, - // })), - // }, - // 'Authors search result', - // ); - const publications = data .map((data) => ({ publisher: getPublisherTitle(data), @@ -146,21 +107,9 @@ export const getExternalPublications = async (node: Node) => { }; export const sendExternalPublicationsNotification = async (node: Node) => { - const publications = await getExternalPublications(node); - - if (!publications.length) return; - - await prisma.externalPublications.createMany({ - skipDuplicates: true, - data: publications.map((pub) => ({ - doi: pub.doi, - score: pub.score, - sourceUrl: pub.sourceUrl, - publisher: pub.publisher, - publishYear: pub.publishYear, - uuid: ensureUuidEndsWithDot(node.uuid), - isVerified: false, - })), + // const publications = await getExternalPublications(node); + const publications = await prisma.externalPublications.findMany({ + where: { uuid: node.uuid }, }); // send email to node owner about potential publications @@ -194,3 +143,7 @@ export const sendExternalPublicationsNotification = async (node: Node) => { logger.info({ err }, '[ExternalPublications EMAIL]::ERROR'); } }; + +export const checkExternalPublications = async (node: Node) => { + return await getExternalPublications(node); +}; From 8a854de2df1bb849c0b472033b2f8627b3d091fd Mon Sep 17 00:00:00 2001 From: shadrach Date: Mon, 27 Jan 2025 14:25:57 +0100 Subject: [PATCH 3/3] fix: typescript import issues --- .../PartykitWsServerAdapter.ts | 12 ++- desci-repo/src/repo.ts | 81 ++++++++++--------- desci-repo/src/server.ts | 77 ++++++++---------- 3 files changed, 82 insertions(+), 88 deletions(-) diff --git a/desci-repo/src/lib/automerge-repo-network-websocket/PartykitWsServerAdapter.ts b/desci-repo/src/lib/automerge-repo-network-websocket/PartykitWsServerAdapter.ts index 19451eb5a..edb56c04b 100644 --- a/desci-repo/src/lib/automerge-repo-network-websocket/PartykitWsServerAdapter.ts +++ b/desci-repo/src/lib/automerge-repo-network-websocket/PartykitWsServerAdapter.ts @@ -1,15 +1,13 @@ -// import WebSocket from 'isomorphic-ws'; -// import { type WebSocketServer } from 'isomorphic-ws'; - +import { cbor as cborHelpers, NetworkAdapter, type PeerMetadata, type PeerId } from '@automerge/automerge-repo/slim'; import debug from 'debug'; -const log = debug('WebsocketServer'); +import { Connection as WebSocket } from 'partyserver'; -import { cbor as cborHelpers, NetworkAdapter, type PeerMetadata, type PeerId } from '@automerge/automerge-repo/slim'; +import { assert } from './assert.js'; import { FromClientMessage, FromServerMessage, isJoinMessage, isLeaveMessage } from './messages.js'; import { ProtocolV1, ProtocolVersion } from './protocolVersion.js'; -import { assert } from './assert.js'; import { toArrayBuffer } from './toArrayBuffer.js'; -import { Connection as WebSocket } from 'partyserver'; + +const log = debug('WebsocketServer'); // import { handleChunked, sendChunked } from './chunking.js'; const { encode, decode } = cborHelpers; diff --git a/desci-repo/src/repo.ts b/desci-repo/src/repo.ts index 45ccec325..c00419dbe 100644 --- a/desci-repo/src/repo.ts +++ b/desci-repo/src/repo.ts @@ -34,8 +34,16 @@ const hostname = os.hostname(); logger.trace({ partyServerHost, partyServerToken, serverName: os.hostname() ?? 'no-hostname' }, 'Env checked'); -let config: RepoConfig; -let socket: WebSocketServer; +let config: RepoConfig = { + peerId: `repo-server-${hostname}` as PeerId, + // Since this is a server, we don't share generously — meaning we only sync documents they already + // know about and can ask for by ID. + sharePolicy: async (peerId, documentId) => { + // logger.trace({ peerId, documentId }, 'SharePolicy called'); + return true; + }, +}; +// let socket: WebSocketServer; if (ENABLE_PARTYKIT_FEATURE) { config = { @@ -48,44 +56,41 @@ if (ENABLE_PARTYKIT_FEATURE) { }, }; } else { - socket = new WebSocketServer({ - port: process.env.WS_PORT ? parseInt(process.env.WS_PORT) : 5445, - path: '/sync', - }); - - const adapter = new NodeWSServerAdapter(socket); - - config = { - network: [adapter], - storage: new PostgresStorageAdapter(), - peerId: `repo-server-${hostname}` as PeerId, - // Since this is a server, we don't share generously — meaning we only sync documents they already - // know about and can ask for by ID. - sharePolicy: async (peerId, documentId) => { - try { - if (!documentId) { - logger.trace({ peerId }, 'SharePolicy: Document ID NOT found'); - return false; - } - // peer format: `peer-[user#id]:[unique string combination] - if (peerId.toString().length < 8) { - logger.error({ peerId }, 'SharePolicy: Peer ID invalid'); - return false; - } - - const userId = peerId.split(':')?.[0]?.split('-')?.[1]; - const isAuthorised = await verifyNodeDocumentAccess(Number(userId), documentId); - logger.trace({ peerId, userId, documentId, isAuthorised }, '[SHARE POLICY CALLED]::'); - return isAuthorised; - } catch (err) { - logger.error({ err }, 'Error in share policy'); - return false; - } - }, - }; + // socket = new WebSocketServer({ + // port: process.env.WS_PORT ? parseInt(process.env.WS_PORT) : 5445, + // path: '/sync', + // }); + // const adapter = new NodeWSServerAdapter(socket); + // config = { + // network: [adapter], + // storage: new PostgresStorageAdapter(), + // peerId: `repo-server-${hostname}` as PeerId, + // // Since this is a server, we don't share generously — meaning we only sync documents they already + // // know about and can ask for by ID. + // sharePolicy: async (peerId, documentId) => { + // try { + // if (!documentId) { + // logger.trace({ peerId }, 'SharePolicy: Document ID NOT found'); + // return false; + // } + // // peer format: `peer-[user#id]:[unique string combination] + // if (peerId.toString().length < 8) { + // logger.error({ peerId }, 'SharePolicy: Peer ID invalid'); + // return false; + // } + // const userId = peerId.split(':')?.[0]?.split('-')?.[1]; + // const isAuthorised = await verifyNodeDocumentAccess(Number(userId), documentId); + // logger.trace({ peerId, userId, documentId, isAuthorised }, '[SHARE POLICY CALLED]::'); + // return isAuthorised; + // } catch (err) { + // logger.error({ err }, 'Error in share policy'); + // return false; + // } + // }, + // }; } -export { socket }; +// export { socket }; export const backendRepo = new Repo(config); diff --git a/desci-repo/src/server.ts b/desci-repo/src/server.ts index 0934bab2c..601f41128 100644 --- a/desci-repo/src/server.ts +++ b/desci-repo/src/server.ts @@ -1,17 +1,10 @@ import 'reflect-metadata'; +import type { Server as HttpServer } from 'http'; import path from 'path'; +import { fileURLToPath } from 'url'; import * as Sentry from '@sentry/node'; import { nodeProfilingIntegration } from '@sentry/profiling-node'; - -const ENABLE_TELEMETRY = process.env.NODE_ENV === 'production'; -const IS_DEV = !ENABLE_TELEMETRY; - -// @ts-check - -import type { Server as HttpServer } from 'http'; -import { fileURLToPath } from 'url'; - import 'dotenv/config'; import 'reflect-metadata'; import bodyParser from 'body-parser'; @@ -23,10 +16,10 @@ import { v4 } from 'uuid'; import { als, logger } from './logger.js'; import { RequestWithUser } from './middleware/guard.js'; -import { extractAuthToken, extractUserFromToken } from './middleware/permissions.js'; import routes from './routes/index.js'; -import { ENABLE_PARTYKIT_FEATURE } from './config.js'; +const ENABLE_TELEMETRY = process.env.NODE_ENV === 'production'; +const IS_DEV = !ENABLE_TELEMETRY; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -136,41 +129,39 @@ class AppServer { logger.info(`Server running on port ${this.port}`); }); - if (!ENABLE_PARTYKIT_FEATURE) { - this.acceptWebsocketConnections(); - } + // if (!ENABLE_PARTYKIT_FEATURE) { + // this.acceptWebsocketConnections(); + // } } async acceptWebsocketConnections() { - const wsSocket = await import('./repo.js').then((x) => x.socket); - - wsSocket.on('listening', () => { - logger.info({ module: 'WebSocket SERVER', port: wsSocket.address() }, 'WebSocket Server Listening'); - }); - - wsSocket.on('connection', async (socket, request) => { - try { - logger.info({ module: 'WebSocket' }, 'WebSocket Connection Attempt'); - const token = await extractAuthToken(request as Request); - const authUser = await extractUserFromToken(token!); - if (!authUser) { - socket.close(); // Close connection if user is not authorized - return; - } - logger.info( - { module: 'WebSocket SERVER', id: authUser.id, name: authUser.name }, - 'WebSocket Connection Authorised', - ); - socket.on('message', (message) => { - // Handle incoming messages - // console.log(`Received message: ${message}`); - }); - // Additional event listeners (e.g., 'close', 'error') can be set up here - } catch (error) { - socket.close(); // Close the connection in case of an error - logger.error(error, 'Error during WebSocket connection'); - } - }); + // const wsSocket = await import('./repo.js').then((x) => x.socket); + // wsSocket.on('listening', () => { + // logger.info({ module: 'WebSocket SERVER', port: wsSocket.address() }, 'WebSocket Server Listening'); + // }); + // wsSocket.on('connection', async (socket, request) => { + // try { + // logger.info({ module: 'WebSocket' }, 'WebSocket Connection Attempt'); + // const token = await extractAuthToken(request as Request); + // const authUser = await extractUserFromToken(token!); + // if (!authUser) { + // socket.close(); // Close connection if user is not authorized + // return; + // } + // logger.info( + // { module: 'WebSocket SERVER', id: authUser.id, name: authUser.name }, + // 'WebSocket Connection Authorised', + // ); + // socket.on('message', (message) => { + // // Handle incoming messages + // // console.log(`Received message: ${message}`); + // }); + // // Additional event listeners (e.g., 'close', 'error') can be set up here + // } catch (error) { + // socket.close(); // Close the connection in case of an error + // logger.error(error, 'Error during WebSocket connection'); + // } + // }); } #initSerialiser() {