add web download refreshing

assyst-core/src/command/services/mod.rs

@@ -2,7 +2,6 @@ use std::str::FromStr;
 use std::time::Duration;
 
 use anyhow::bail;
-use assyst_common::markdown::{self, Markdown};
 use assyst_common::util::format_duration;
 use assyst_proc_macro::command;
 

assyst-core/src/main.rs

@@ -23,6 +23,7 @@ use command::registry::register_interaction_commands;
 use gateway_handler::handle_raw_event;
 use gateway_handler::incoming_event::IncomingEvent;
 use rest::web_media_download::get_web_download_api_urls;
+use task::tasks::refresh_web_download_urls::refresh_web_download_urls;
 use task::tasks::reminders::handle_reminders;
 use tokio::spawn;
 use tracing::{debug, info, trace};
@@ -124,6 +125,13 @@ async fn main() {
         info!("Reminder processing disabled in config.dev.disable_reminder_check: not registering task");
     }
 
+    assyst.register_task(Task::new_delayed(
+        assyst.clone(),
+        Duration::from_secs(60 * 10),
+        Duration::from_secs(60 * 10),
+        function_task_callback!(refresh_web_download_urls),
+    ));
+
     info!("Starting assyst-webserver");
     assyst_webserver::run(
         assyst.database_handler.clone(),

assyst-core/src/rest/bad_translation.rs

@@ -122,6 +122,8 @@ pub async fn get_languages(client: &Client) -> Result<Vec<(Box<str>, Box<str>)>,
         .map_err(TranslateError::Reqwest)
 }
 
+// used in btchannel later
+#[allow(unused)]
 pub async fn validate_language(client: &Client, provided_language: &str) -> Result<bool, TranslateError> {
     let languages = get_languages(client).await?;
     Ok(languages.iter().any(|(language, _)| &**language == provided_language))

assyst-core/src/rest/web_media_download.rs

@@ -1,4 +1,3 @@
-use std::cell::LazyCell;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
@@ -20,9 +19,10 @@ use crate::downloader::{download_content, ABSOLUTE_INPUT_FILE_SIZE_LIMIT_BYTES};
 pub const INSTANCES_ROUTE: &str = "https://instances.hyper.lol/instances.json";
 
 pub const TEST_URL: &str = "https://www.youtube.com/watch?v=sbvp3kuU2ak";
-pub const TEST_URL_TIMEOUT: LazyCell<Duration> = LazyCell::new(|| Duration::from_secs(5));
 pub const TEST_SCORE_THRESHOLD: f32 = 90.0;
 
+pub static TEST_URL_TIMEOUT: Duration = Duration::from_secs(10);
+
 #[derive(Default)]
 pub struct WebDownloadOpts {
     pub audio_only: Option<bool>,
@@ -77,20 +77,20 @@ async fn test_route(assyst: ThreadSafeAssyst, url: &str) -> bool {
 
     let elapsed = start.elapsed();
 
-    if success && elapsed < *TEST_URL_TIMEOUT {
+    if success && elapsed < TEST_URL_TIMEOUT {
         debug!(
             "Web download URL {url} took {} to download test media",
             format_duration(&elapsed)
         );
-    } else if elapsed < *TEST_URL_TIMEOUT {
+    } else if elapsed < TEST_URL_TIMEOUT {
         let err = res.unwrap_err();
         debug!(
             "Web download URL {url} failed to download test media ({})",
             err.to_string()
         );
     }
 
-    success && (elapsed < *TEST_URL_TIMEOUT)
+    success && (elapsed < TEST_URL_TIMEOUT)
 }
 
 /// Always returns the main API instance (api.cobalt.tools) at the minimum. \
@@ -109,20 +109,19 @@ pub async fn get_web_download_api_urls(assyst: ThreadSafeAssyst) -> anyhow::Resu
 
     let test_urls = json
         .iter()
-        .map(|entry: &InstancesQueryResult| {
+        .filter_map(|entry: &InstancesQueryResult| {
             if (entry.protocol == "https" && entry.score >= TEST_SCORE_THRESHOLD) || (entry.api == "api.cobalt.tools") {
-                format!("https://{}/api/json", entry.api)
+                Some(format!("https://{}/api/json", entry.api))
             } else {
-                String::new()
+                None
             }
         })
-        .filter(|x| !x.is_empty())
         .map(|url| {
             debug!("Testing web download API URL {}", url);
 
             let a = assyst.clone();
             timeout(
-                *TEST_URL_TIMEOUT,
+                TEST_URL_TIMEOUT,
                 tokio::spawn(async move {
                     if url != "https://api.cobalt.tools/api/json" {
                         let res = test_route(a, &url).await;
@@ -147,7 +146,7 @@ pub async fn get_web_download_api_urls(assyst: ThreadSafeAssyst) -> anyhow::Resu
     Ok(valid_urls)
 }
 
-/// Attempts to download web media. Will try all APIs in the event of valiure, unless
+/// Attempts to download web media. Will try all APIs in the event of faliure, unless
 /// `opts.api_url_override` is set.
 pub async fn download_web_media(assyst: ThreadSafeAssyst, url: &str, opts: WebDownloadOpts) -> anyhow::Result<Vec<u8>> {
     let encoded_url = urlencoding::encode(url).to_string();
@@ -200,10 +199,6 @@ pub async fn download_web_media(assyst: ThreadSafeAssyst, url: &str, opts: WebDo
                             let try_json = from_str::<WebDownloadError>(&e);
                             match try_json {
                                 Ok(j) => {
-                                    // some errors we can break on because they probably affect all instances (i.e.,
-                                    // could not find info about link)
-                                    let mut breakable = false;
-
                                     let mut e = j.text;
                                     if e.contains("i couldn't process your request :(") {
                                         e = "The web downloader could not process your request. Please try again later."
@@ -213,14 +208,9 @@ pub async fn download_web_media(assyst: ThreadSafeAssyst, url: &str, opts: WebDo
                                     } else if e.contains("couldn't get this youtube video because it requires sign in")
                                     {
                                         e = "YouTube has blocked video downloading. Please try again later.".to_owned()
-                                    } else if e.contains("i couldn't find anything about this link") {
-                                        breakable = true;
                                     }
 
                                     err = format!("Download request failed: {}", e);
-                                    if breakable {
-                                        break;
-                                    }
                                 },
                                 Err(d_e) => err = format!("Download request failed: {} (raw error: {})", d_e, e),
                             }

assyst-core/src/task/mod.rs

@@ -38,7 +38,7 @@ impl Task {
         Task { _thread: thread }
     }
 
-    pub fn _new_delayed(assyst: ThreadSafeAssyst, interval: Duration, delay: Duration, callback: TaskRun) -> Task {
+    pub fn new_delayed(assyst: ThreadSafeAssyst, interval: Duration, delay: Duration, callback: TaskRun) -> Task {
         let thread = spawn(async move {
             sleep(delay).await;
             loop {

assyst-core/src/task/tasks/mod.rs

@@ -1,3 +1,4 @@
 pub mod get_premium_users;
+pub mod refresh_web_download_urls;
 pub mod reminders;
 pub mod top_gg_stats;

assyst-core/src/task/tasks/refresh_web_download_urls.rs

@@ -0,0 +1,45 @@
+use crate::assyst::ThreadSafeAssyst;
+use crate::rest::web_media_download::get_web_download_api_urls;
+use assyst_common::err;
+use tracing::info;
+
+pub async fn refresh_web_download_urls(assyst: ThreadSafeAssyst) {
+    info!("Updating web download source URLs");
+
+    let old = assyst
+        .rest_cache_handler
+        .get_web_download_urls()
+        .iter()
+        .map(|x| (*(x.clone())).clone())
+        .collect::<Vec<_>>();
+
+    let urls = get_web_download_api_urls(assyst.clone()).await;
+
+    if let Ok(ref new) = urls {
+        let mut removed = 0;
+        let mut added = 0;
+
+        println!("{:?}", old);
+        println!("{:?}", new);
+
+        for url in new {
+            if !old.contains(url) {
+                added += 1;
+            }
+        }
+
+        for url in old {
+            if !new.contains(&url) {
+                removed += 1;
+            }
+        }
+
+        info!(
+            "Updated web download source URLs: got {} urls (removed={removed}, added={added})",
+            new.len()
+        );
+        assyst.rest_cache_handler.set_web_download_urls(new.clone());
+    } else if let Err(e) = urls {
+        err!("Error updating web download source URLs: {}", e.to_string())
+    };
+}