Skip to content

Commit

Permalink
Merge pull request #246 from coderwassananmol/PR240
Browse files Browse the repository at this point in the history 
Google Books x Wikidata
  • Loading branch information
@coderwassananmol
coderwassananmol authored Jun 8, 2024
2 parents 39e708b + a4b3598 commit 908fb55
Show file tree
Hide file tree
Showing 12 changed files with 372 additions and 106 deletions.
1 change: 1 addition & 0 deletions .env.example
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,5 +12,6 @@ EMAIL_BOT_USERNAME = -- mediawiki bot username --
EMAIL_BOT_PASSWORD = -- mediawiki bot password --
NEXT_PUBLIC_WIKIMEDIA_URL = -- https://meta.wikimedia.beta.wmflabs.org for local setup --
NEXT_PUBLIC_COMMONS_URL = -- https://commons.wikimedia.beta.wmflabs.org for local setup --
NEXT_PUBLIC_WIKIDATA_URL = -- https://commons.wikimedia.beta.wmflabs.org for local setup --
NEXT_PUBLIC_IA_USER = -- bub_wikimedia --
IA_EMAIL = -- IA email --
4 changes: 3 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,6 @@ node_modules
.env
package-lock.json
out/
*.log
*.log
.env.stage
.env.local
153 changes: 90 additions & 63 deletions bull/commons-queue/consumer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,83 +4,110 @@ const winston = require("winston");
const {
downloadFile,
uploadToCommons,
uploadToWikiData,
convertZipToPdf,
} = require("../../utils/helper");
const JSZip = require("jszip");
const logger = winston.loggers.get("defaultLogger");

CommonsQueue.process(async (job, done) => {
if (job.data.type === "pdlZip") {
const zipBuffer = Buffer.from(job.data.downloadFileURL, "base64");
const zip = await JSZip.loadAsync(zipBuffer);
const convertZipToPdfRes = await convertZipToPdf(
zip,
"commonsFilePayload.pdf"
);
if (convertZipToPdfRes.status !== 200) {
logger.log({
level: "error",
message: `convertZipToPdfRes: ${JSON.stringify(convertZipToPdfRes)}`,
});
process.emit("commonsJobComplete", {
status: false,
value: null,
});
return done(null, true);
}
const commonsResponse = await uploadToCommons(job.data.metadata);
try {
if (job.data.type === "pdlZip") {
const zipBuffer = Buffer.from(job.data.downloadFileURL, "base64");
const zip = await JSZip.loadAsync(zipBuffer);
const convertZipToPdfRes = await convertZipToPdf(
zip,
"commonsFilePayload.pdf"
);
if (convertZipToPdfRes.status !== 200) {
logger.log({
level: "error",
message: `convertZipToPdfRes: ${JSON.stringify(convertZipToPdfRes)}`,
});
process.emit("commonsJobComplete", {
status: false,
value: null,
});
return done(null, true);
}
const commonsResponse = await uploadToCommons(job.data.metadata);

if (commonsResponse.fileUploadStatus !== 200) {
logger.log({
level: "error",
message: `uploadToCommons: ${commonsResponse}`,
});
if (commonsResponse.fileUploadStatus !== 200) {
logger.log({
level: "error",
message: `uploadToCommons: ${commonsResponse}`,
});
process.emit(`commonsJobComplete:${job.id}`, {
status: false,
value: null,
});
return done(null, true);
}
process.emit(`commonsJobComplete:${job.id}`, {
status: false,
value: null,
status: true,
value: commonsResponse,
});
return done(null, true);
}
process.emit(`commonsJobComplete:${job.id}`, {
status: true,
value: commonsResponse,
});
return done(null, true);
} else {
const url =
job.data?.metadata?.uri ||
job.data?.downloadFileURL?.uri ||
job.data?.metadata?.pdfUrl;
const downloadFileRes = await downloadFile(url, "commonsFilePayload.pdf");
} else {
const url =
job.data?.metadata?.uri ||
job.data?.downloadFileURL?.uri ||
job.data?.metadata?.pdfUrl;
const downloadFileRes = await downloadFile(url, "commonsFilePayload.pdf");

if (downloadFileRes.writeFileStatus !== 200) {
logger.log({
level: "error",
message: `downloadFile: ${downloadFileRes}`,
});
process.emit(`commonsJobComplete:${job.id}`, {
status: false,
value: null,
});
return done(null, true);
}
const commonsResponse = await uploadToCommons(job.data.metadata);
if (downloadFileRes.writeFileStatus !== 200) {
logger.log({
level: "error",
message: `downloadFile: ${downloadFileRes}`,
});
process.emit(`commonsJobComplete:${job.id}`, {
status: false,
value: null,
});
return done(null, true);
}
const commonsResponse = await uploadToCommons(job.data.metadata);

if (commonsResponse.fileUploadStatus !== 200) {
logger.log({
level: "error",
message: `uploadToCommons: ${commonsResponse}`,
});
process.emit(`commonsJobComplete:${job.id}`, {
status: false,
value: null,
});
if (commonsResponse.fileUploadStatus !== 200) {
logger.log({
level: "error",
message: `uploadToCommons: ${commonsResponse}`,
});
process.emit(`commonsJobComplete:${job.id}`, {
status: false,
value: null,
});
return done(new Error(commonsResponse));
}
const wikiDataResponse = await uploadToWikiData(
job.data.metadata,
commonsResponse.filename,
job.data.libraryName
);
if (wikiDataResponse !== 404) {
process.emit(`commonsJobComplete:${job.id}`, {
status: true,
value: {
commons: commonsResponse,
wikidata: wikiDataResponse,
},
});
} else {
process.emit(`commonsJobComplete:${job.id}`, {
status: true,
value: {
commons: commonsResponse,
wikidata: 404,
},
});
}
return done(null, true);
}
process.emit(`commonsJobComplete:${job.id}`, {
status: true,
value: commonsResponse,
} catch (error) {
logger.log({
level: "error",
message: err,
});
return done(null, true);
console.log(error, "::errorCommonsQueue");
}
});
2 changes: 2 additions & 0 deletions bull/commons-queue/producer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,15 @@ module.exports = async (
type = "other",
downloadFileURL = null,
metadata,
libraryName,
callback
) => {
const job = await CommonsQueue.add({
type,
downloadFileURL,
metadata,
callback,
libraryName,
});
process.on(`commonsJobComplete:${job.id}`, (commonsResponse) => {
callback(commonsResponse);
Expand Down
83 changes: 50 additions & 33 deletions bull/google-books-queue/consumer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ GoogleBooksQueue.process((job, done) => {
"X-Archive-Meta-Collection": "opensource",
"X-Archive-Ignore-Preexisting-Bucket": 1,
"X-archive-meta-title": `uri(${encodeURI(title.trim())})`,
"X-archive-meta-date": publishedDate.trim(),
"X-archive-meta-date": publishedDate ? publishedDate.trim() : "",
"X-archive-meta-language": language.trim(),
"X-archive-meta-mediatype": "texts",
"X-archive-meta-licenseurl":
Expand Down Expand Up @@ -120,41 +120,58 @@ GoogleBooksQueue.process((job, done) => {
step: "Uploading to Wikimedia Commons",
value: `(${50}%)`,
});
CommonsProducer(null, null, job.data, async (commonsResponse) => {
if (commonsResponse.status === true) {
job.progress({
step: "Upload to Wikimedia Commons",
value: `(${100}%)`,
wikiLinks: {
commons: await commonsResponse.value.filename,
},
});
if (job.data.isEmailNotification === "true") {
const commonsLink =
process.env.NEXT_PUBLIC_COMMONS_URL +
`/wiki/File:${commonsResponse.value.filename}`;
EmailProducer(
job.data.userName,
title,
{ archiveLink: trueURI, commonsLink: commonsLink },
{ archive: true, commons: true }
);
}
return done(null, true);
} else {
job.progress({
step: "Upload To IA (100%), Upload To Commons",
value: `(Failed)`,
});
if (job.data.isEmailNotification === "true") {
EmailProducer(job.data.userName, title, trueURI, {
archive: true,
commons: false,
CommonsProducer(
null,
null,
job.data,
"gb",
async (commonsResponse) => {
if (commonsResponse.status === true) {
job.progress({
step: "Upload to Wikimedia Commons",
value: `(${100}%)`,
wikiLinks: {
commons:
(await commonsResponse.value.filename) ||
commonsResponse.filename ||
commonsResponse.value.commons.filename,
wikidata:
(await commonsResponse.value.wikidata) !== 404
? await commonsResponse.value.wikidata
: 404,
},
});
if (job.data.isEmailNotification === "true") {
const commonsLink =
process.env.NEXT_PUBLIC_COMMONS_URL +
`/wiki/File:${
commonsResponse.value.filename ||
commonsResponse.filename ||
commonsResponse.value.commons.filename
}`;
EmailProducer(
job.data.userName,
title,
{ archiveLink: trueURI, commonsLink: commonsLink },
{ archive: true, commons: true }
);
}
return done(null, true);
} else {
job.progress({
step: "Upload To IA (100%), Upload To Commons",
value: `(Failed)`,
});
if (job.data.isEmailNotification === "true") {
EmailProducer(job.data.userName, title, trueURI, {
archive: true,
commons: false,
});
}
return done(null, true);
}
return done(null, true);
}
});
);
}
}
}
Expand Down
1 change: 1 addition & 0 deletions bull/pdl-queue/consumer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,7 @@ function uploadPdfToIA(pdfUrl, job, metadata, trueURI, done) {
null,
null,
job.data.details,
"pdl",
async (commonsResponse) => {
if (commonsResponse.status === true) {
job.progress({
Expand Down
18 changes: 16 additions & 2 deletions bull/trove-queue/consumer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,12 @@ TroveQueue.process((job, done) => {
step: "Upload To IA",
value: `(${100}%)`,
});
if (
isEmailNotification !== "true" &&
job.data.details.isUploadCommons !== "true"
) {
done(null, true);
}
if (
isEmailNotification === "true" &&
job.data.details.isUploadCommons !== "true"
Expand All @@ -125,13 +131,18 @@ TroveQueue.process((job, done) => {
null,
downloadFileUrl,
job.data.details,
"trove",
async (commonsResponse) => {
if (commonsResponse.status === true) {
job.progress({
step: "Upload to Wikimedia Commons",
value: `(100%)`,
value: `(${100}%)`,
wikiLinks: {
commons: await commonsResponse.value.filename,
wikidata:
(await commonsResponse.value.wikidata) !== 404
? await commonsResponse.value.wikidata
: 404,
},
});
if (job.data.isEmailNotification === "true") {
Expand All @@ -141,7 +152,10 @@ TroveQueue.process((job, done) => {
EmailProducer(
userName,
name,
{ archiveLink: trueURI, commonsLink: commonsLink },
{
archiveLink: trueURI,
commonsLink: commonsLink,
},
{ archive: true, commons: true }
);
}
Expand Down
13 changes: 13 additions & 0 deletions components/Books.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -448,6 +448,19 @@ const Books = () => {
</div>
</span>
</div>
{isUploadCommons ? (
<div
class="cdx-message cdx-message--block cdx-message--success"
aria-live="polite"
style={{ marginTop: "10px" }}
>
<span class="cdx-message__icon"></span>
<div class="cdx-message__content">
[NEW] BUB2 will create Wikidata entries for the file
metadata. Only applicable for <b>Google Books.</b>
</div>
</div>
) : null}
<div style={{ marginTop: "10px" }} className="section">
<span className="cdx-checkbox">
<input
Expand Down
1 change: 0 additions & 1 deletion components/Footer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ export default () => (
background-color: #f5f5f5;
margin-top: auto;
padding: 1em;
position: fixed;
left: 0;
bottom: 0;
}
Expand Down
Loading

0 comments on commit 908fb55

Please sign in to comment.