Smarter delay between requests

Wait a randomized and slowly growing amount of time between requests to the same domain.

src/App.vue

@@ -135,7 +135,7 @@ async function updateWall() {
       if (progress.errors.length) {
         setStatus(progress.errors.slice(-1)[0].message, true)
       } else if (progress.finished < progress.total) {
-        setStatus(`Loading [${progress.finished}/${progress.total}] sources ...`)
+        setStatus(`Loading ${progress.finished}/${progress.total} sources ...`)
       } else {
         setStatus(false)
       }

src/main.ts

@@ -1,5 +1,5 @@
 /**
-    Copyright (C) 2023  Marcel Hellkamp
+    Copyright (C) 2024  Marcel Hellkamp
     Copyright (C) 2023  Gesellschaft für wissenschaftliche Datenverarbeitung mbH Göttingen
 
     This program is free software: you can redistribute it and/or modify

src/sources.ts

@@ -102,11 +102,13 @@ export async function fetchPosts(cfg: Config, onProgress: (progress: Progress) =
     }
 
     // Be nice and not overwhelm servers with parallel requests.
-    // Run tasks for the same domain in sequence instead.
+    // Run tasks for the same domain in sequence instead, and wait between
+    // requests for a small random amount of time.
     const groupedTasks = Object.entries(domainTasks)
         .map(([domain, tasks]) => {
             return async () => {
-                for (const task of tasks) {
+                for (const [taskIndex, task] of tasks.entries()) {
+                    await sleep(Math.min(Math.random() * 100 * taskIndex, 500))
                     progress.started += 1;
                     try {
                         (await task())
@@ -168,8 +170,6 @@ async function fetchJson(domain: string, path: string, query?: Record<string, an
     let rs: Response;
     let json: any;
 
-    await new Promise(resolve => setTimeout(resolve, 1000));
-
     try {
         rs = await fetch(url)
 

src/utils.ts

@@ -27,6 +27,10 @@ export function deepClone(obj: any) {
     return JSON.parse(JSON.stringify(obj))
 }
 
+export function sleep(ms:number) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+
 
 /**
  * Find all text nodes and replace each occurrences of a pattern with either