diff --git a/fastside-actualizer/src/main.rs b/fastside-actualizer/src/main.rs index 09d2ed35..b5c7766c 100644 --- a/fastside-actualizer/src/main.rs +++ b/fastside-actualizer/src/main.rs @@ -54,6 +54,10 @@ enum Commands { /// Amount of maximum parallel requests. #[arg(long, default_value = None)] max_parallel: Option, + /// List of service names to actualize. + /// If not provided, all services will be actualized. + #[arg(short = 'u', long = "update", default_value = None)] + update_service_names: Option>, }, } @@ -189,8 +193,7 @@ async fn update_service( Some(updater) => { let updated_instances_result = updater .update(client, &service.instances, changes_summary.clone()) - .await - .context("failed to update service"); + .await; match updated_instances_result { Ok(updated_instances) => { debug!("Updated instances: {:?}", updated_instances); @@ -321,6 +324,7 @@ async fn main() -> Result<()> { output, data, max_parallel, + update_service_names, }) => { let config = load_config(&cli.config).context("failed to load config")?; @@ -348,12 +352,21 @@ async fn main() -> Result<()> { std::fs::read_to_string(services).context("failed to read services file")?; serde_json::from_str(&data_content).context("failed to parse services file")? }; - let mut services_data = stored_data + let mut services_data: HashMap = stored_data .services .into_iter() .map(|service| (service.name.clone(), service)) .collect(); + // Check if all services from update_service_names exist + if let Some(update_service_names) = update_service_names { + for name in update_service_names.iter() { + if !services_data.contains_key(name) { + return Err(anyhow!("service {name:?} does not exist")); + } + } + } + let changes_summary = ChangesSummary::new(); let start = std::time::Instant::now(); @@ -362,8 +375,21 @@ async fn main() -> Result<()> { actualizer_data.remove_removed_instances(&services_data); let update_service_client = reqwest::Client::new(); - let length = services_data.len(); - for (i, (name, service)) in services_data.iter_mut().enumerate() { + + // Filter services data + let mut filtered_services_data = services_data + .iter_mut() + .filter(|(name, _)| { + if let Some(update_service_names) = update_service_names { + update_service_names.contains(name) + } else { + true + } + }) + .collect::>(); + let length = filtered_services_data.len(); + + for (i, (name, service)) in filtered_services_data.iter_mut().enumerate() { info!( "Actualizing service {name} ({i}/{length})", name = name, diff --git a/fastside-actualizer/src/services/breezewiki.rs b/fastside-actualizer/src/services/breezewiki.rs new file mode 100644 index 00000000..1fafc9e8 --- /dev/null +++ b/fastside-actualizer/src/services/breezewiki.rs @@ -0,0 +1,67 @@ +use crate::{types::ServiceUpdater, ChangesSummary}; +use async_trait::async_trait; +use fastside_shared::serde_types::Instance; +use serde::Deserialize; +use url::Url; + +pub struct BreezewikiUpdater { + pub instances_url: String, +} + +impl BreezewikiUpdater { + pub fn new() -> Self { + Self { + instances_url: "https://docs.breezewiki.com/files/instances.json".to_string(), + } + } +} + +impl Default for BreezewikiUpdater { + fn default() -> Self { + Self::new() + } +} + +#[derive(Debug, Deserialize)] +struct BreezewikiInstance { + instance: Url, +} + +#[derive(Debug, Deserialize)] +struct InstancesResponse(Vec); + +#[async_trait] +impl ServiceUpdater for BreezewikiUpdater { + async fn update( + &self, + client: reqwest::Client, + current_instances: &[Instance], + changes_summary: ChangesSummary, + ) -> anyhow::Result> { + let response = client.get(&self.instances_url).send().await?; + let response_str = response.text().await?; + let parsed: InstancesResponse = serde_json::from_str(&response_str)?; + + let mut instances = current_instances.to_vec(); + let mut new_instances = Vec::new(); + + for instance in parsed.0 { + let url = instance.instance; + if current_instances.iter().any(|i| i.url == url) { + continue; + } + new_instances.push(Instance::from(url.clone())); + } + + changes_summary + .set_new_instances_added( + "breezewiki", + new_instances.iter().map(|i| i.url.clone()).collect(), + ) + .await; + + instances.extend(new_instances); + + Ok(instances) + } +} diff --git a/fastside-actualizer/src/services/gothub.rs b/fastside-actualizer/src/services/gothub.rs new file mode 100644 index 00000000..e98cbf3d --- /dev/null +++ b/fastside-actualizer/src/services/gothub.rs @@ -0,0 +1,69 @@ +use crate::{types::ServiceUpdater, ChangesSummary}; +use async_trait::async_trait; +use fastside_shared::serde_types::Instance; +use serde::Deserialize; +use url::Url; + +pub struct GothubUpdater { + pub instances_url: String, +} + +impl GothubUpdater { + pub fn new() -> Self { + Self { + instances_url: + "https://codeberg.org/gothub/gothub-instances/raw/branch/master/instances.json" + .to_string(), + } + } +} + +impl Default for GothubUpdater { + fn default() -> Self { + Self::new() + } +} + +#[derive(Debug, Deserialize)] +struct GothubInstance { + link: Url, +} + +#[derive(Debug, Deserialize)] +struct InstancesResponse(Vec); + +#[async_trait] +impl ServiceUpdater for GothubUpdater { + async fn update( + &self, + client: reqwest::Client, + current_instances: &[Instance], + changes_summary: ChangesSummary, + ) -> anyhow::Result> { + let response = client.get(&self.instances_url).send().await?; + let response_str = response.text().await?; + let parsed: InstancesResponse = serde_json::from_str(&response_str)?; + + let mut instances = current_instances.to_vec(); + let mut new_instances = Vec::new(); + + for instance in parsed.0 { + let url = instance.link; + if current_instances.iter().any(|i| i.url == url) { + continue; + } + new_instances.push(Instance::from(url.clone())); + } + + changes_summary + .set_new_instances_added( + "gothub", + new_instances.iter().map(|i| i.url.clone()).collect(), + ) + .await; + + instances.extend(new_instances); + + Ok(instances) + } +} diff --git a/fastside-actualizer/src/services/invidious.rs b/fastside-actualizer/src/services/invidious.rs new file mode 100644 index 00000000..88d5ba80 --- /dev/null +++ b/fastside-actualizer/src/services/invidious.rs @@ -0,0 +1,64 @@ +use crate::{types::ServiceUpdater, utils::url::default_domain_scheme, ChangesSummary}; +use async_trait::async_trait; +use fastside_shared::serde_types::Instance; +use serde::Deserialize; +use url::Url; + +pub struct InvidiousUpdater { + pub instances_url: String, +} + +impl InvidiousUpdater { + pub fn new() -> Self { + Self { + instances_url: "https://api.invidious.io/instances.json".to_string(), + } + } +} + +impl Default for InvidiousUpdater { + fn default() -> Self { + Self::new() + } +} + +#[derive(Debug, Deserialize)] +struct InstancesResponse(Vec<(String, serde_json::Value)>); + +#[async_trait] +impl ServiceUpdater for InvidiousUpdater { + async fn update( + &self, + client: reqwest::Client, + current_instances: &[Instance], + changes_summary: ChangesSummary, + ) -> anyhow::Result> { + let response = client.get(&self.instances_url).send().await?; + let response_str = response.text().await?; + let parsed: InstancesResponse = serde_json::from_str(&response_str)?; + + let mut instances = current_instances.to_vec(); + let mut new_instances = Vec::new(); + + for instance in parsed.0 { + let domain = instance.0; + let scheme = default_domain_scheme(domain.as_str()); + let url = Url::parse(&format!("{}{}", scheme, domain))?; + if current_instances.iter().any(|i| i.url == url) { + continue; + } + new_instances.push(Instance::from(url.clone())); + } + + changes_summary + .set_new_instances_added( + "invidious", + new_instances.iter().map(|i| i.url.clone()).collect(), + ) + .await; + + instances.extend(new_instances); + + Ok(instances) + } +} diff --git a/fastside-actualizer/src/services/libreddit.rs b/fastside-actualizer/src/services/libreddit.rs new file mode 100644 index 00000000..98f6bd33 --- /dev/null +++ b/fastside-actualizer/src/services/libreddit.rs @@ -0,0 +1,71 @@ +use crate::{types::ServiceUpdater, ChangesSummary}; +use async_trait::async_trait; +use fastside_shared::serde_types::Instance; +use serde::Deserialize; +use url::Url; + +pub struct LibredditUpdater { + pub instances_url: String, +} + +impl LibredditUpdater { + pub fn new() -> Self { + Self { + instances_url: + "https://raw.githubusercontent.com/redlib-org/redlib-instances/main/instances.json" + .to_string(), + } + } +} + +impl Default for LibredditUpdater { + fn default() -> Self { + Self::new() + } +} + +#[derive(Debug, Deserialize)] +struct LibredditInstance { + url: Url, +} + +#[derive(Debug, Deserialize)] +struct InstancesResponse { + instances: Vec, +} + +#[async_trait] +impl ServiceUpdater for LibredditUpdater { + async fn update( + &self, + client: reqwest::Client, + current_instances: &[Instance], + changes_summary: ChangesSummary, + ) -> anyhow::Result> { + let response = client.get(&self.instances_url).send().await?; + let response_str = response.text().await?; + let parsed: InstancesResponse = serde_json::from_str(&response_str)?; + + let mut instances = current_instances.to_vec(); + let mut new_instances = Vec::new(); + + for instance in parsed.instances { + let url = instance.url; + if current_instances.iter().any(|i| i.url == url) { + continue; + } + new_instances.push(Instance::from(url.clone())); + } + + changes_summary + .set_new_instances_added( + "libreddit", + new_instances.iter().map(|i| i.url.clone()).collect(), + ) + .await; + + instances.extend(new_instances); + + Ok(instances) + } +} diff --git a/fastside-actualizer/src/services/librex.rs b/fastside-actualizer/src/services/librex.rs new file mode 100644 index 00000000..040cca5c --- /dev/null +++ b/fastside-actualizer/src/services/librex.rs @@ -0,0 +1,87 @@ +use crate::{types::ServiceUpdater, ChangesSummary}; +use async_trait::async_trait; +use fastside_shared::serde_types::Instance; +use serde::Deserialize; +use url::Url; + +pub struct LibrexUpdater { + pub instances_url: String, +} + +impl LibrexUpdater { + pub fn new() -> Self { + Self { + instances_url: "https://raw.githubusercontent.com/Ahwxorg/LibreY/main/instances.json" + .to_string(), + } + } +} + +impl Default for LibrexUpdater { + fn default() -> Self { + Self::new() + } +} + +#[derive(Debug, Deserialize)] +struct LibrexInstance { + #[serde(default)] + clearnet: Option, + #[serde(default)] + tor: Option, + #[serde(default)] + i2p: Option, +} + +#[derive(Debug, Deserialize)] +struct InstancesResponse { + instances: Vec, +} + +#[async_trait] +impl ServiceUpdater for LibrexUpdater { + async fn update( + &self, + client: reqwest::Client, + current_instances: &[Instance], + changes_summary: ChangesSummary, + ) -> anyhow::Result> { + let response = client.get(&self.instances_url).send().await?; + let response_str = response.text().await?; + let parsed: InstancesResponse = serde_json::from_str(&response_str)?; + + let mut instances = current_instances.to_vec(); + let mut new_instances = Vec::new(); + + let mut parsed_urls = Vec::new(); + for instance in parsed.instances { + if let Some(url) = instance.clearnet { + parsed_urls.push(url); + } + if let Some(url) = instance.tor { + parsed_urls.push(url); + } + if let Some(url) = instance.i2p { + parsed_urls.push(url); + } + } + + for url in parsed_urls { + if current_instances.iter().any(|i| i.url == url) { + continue; + } + new_instances.push(Instance::from(url.clone())); + } + + changes_summary + .set_new_instances_added( + "librex", + new_instances.iter().map(|i| i.url.clone()).collect(), + ) + .await; + + instances.extend(new_instances); + + Ok(instances) + } +} diff --git a/fastside-actualizer/src/services/mod.rs b/fastside-actualizer/src/services/mod.rs index 830a138a..88e861bc 100644 --- a/fastside-actualizer/src/services/mod.rs +++ b/fastside-actualizer/src/services/mod.rs @@ -1,5 +1,14 @@ +mod breezewiki; mod default; +mod gothub; +mod invidious; +mod libreddit; +mod librex; +mod scribe; mod searx; +mod searxng; +mod simplytranslate; +mod tent; use crate::types::ServiceUpdater; @@ -9,16 +18,25 @@ pub use default::DefaultInstanceChecker; pub fn get_service_updater(name: &str) -> Option> { match name { "searx" => Some(Box::new(searx::SearxUpdater::new())), + "searxng" => Some(Box::new(searxng::SearxngUpdater::new())), + "simplytranslate" => Some(Box::new(simplytranslate::SimplyTranslateUpdater::new())), + "invidious" => Some(Box::new(invidious::InvidiousUpdater::new())), + "scribe" => Some(Box::new(scribe::ScribeUpdater::new())), + "libreddit" => Some(Box::new(libreddit::LibredditUpdater::new())), + "breezewiki" => Some(Box::new(breezewiki::BreezewikiUpdater::new())), + "librex" => Some(Box::new(librex::LibrexUpdater::new())), + "gothub" => Some(Box::new(gothub::GothubUpdater::new())), + "tent" => Some(Box::new(tent::TentUpdater::new())), _ => None, } } /// Get an instance checker by name. +#[allow(clippy::match_single_binding)] pub fn get_instance_checker( name: &str, ) -> Box<(dyn crate::types::InstanceChecker + Send + Sync + 'static)> { match name { - "searx" => Box::new(searx::SearxUpdater::new()), _ => Box::new(DefaultInstanceChecker::new()), } } diff --git a/fastside-actualizer/src/services/scribe.rs b/fastside-actualizer/src/services/scribe.rs new file mode 100644 index 00000000..f61bb8c3 --- /dev/null +++ b/fastside-actualizer/src/services/scribe.rs @@ -0,0 +1,62 @@ +use crate::{types::ServiceUpdater, ChangesSummary}; +use async_trait::async_trait; +use fastside_shared::serde_types::Instance; +use serde::Deserialize; +use url::Url; + +pub struct ScribeUpdater { + pub instances_url: String, +} + +impl ScribeUpdater { + pub fn new() -> Self { + Self { + instances_url: "https://git.sr.ht/~edwardloveall/scribe/blob/main/docs/instances.json" + .to_string(), + } + } +} + +impl Default for ScribeUpdater { + fn default() -> Self { + Self::new() + } +} + +#[derive(Debug, Deserialize)] +struct InstancesResponse(Vec); + +#[async_trait] +impl ServiceUpdater for ScribeUpdater { + async fn update( + &self, + client: reqwest::Client, + current_instances: &[Instance], + changes_summary: ChangesSummary, + ) -> anyhow::Result> { + let response = client.get(&self.instances_url).send().await?; + let response_str = response.text().await?; + let parsed: InstancesResponse = serde_json::from_str(&response_str)?; + + let mut instances = current_instances.to_vec(); + let mut new_instances = Vec::new(); + + for url in parsed.0 { + if current_instances.iter().any(|i| i.url == url) { + continue; + } + new_instances.push(Instance::from(url.clone())); + } + + changes_summary + .set_new_instances_added( + "scribe", + new_instances.iter().map(|i| i.url.clone()).collect(), + ) + .await; + + instances.extend(new_instances); + + Ok(instances) + } +} diff --git a/fastside-actualizer/src/services/searx.rs b/fastside-actualizer/src/services/searx.rs index a0345ad9..f0f7d3ac 100644 --- a/fastside-actualizer/src/services/searx.rs +++ b/fastside-actualizer/src/services/searx.rs @@ -1,11 +1,8 @@ use std::collections::HashMap; -use crate::{ - types::{InstanceChecker, ServiceUpdater}, - ChangesSummary, -}; +use crate::{types::ServiceUpdater, ChangesSummary}; use async_trait::async_trait; -use fastside_shared::serde_types::{Instance, Service}; +use fastside_shared::serde_types::Instance; use serde::Deserialize; use url::Url; @@ -65,16 +62,3 @@ impl ServiceUpdater for SearxUpdater { Ok(instances) } } - -#[async_trait] -impl InstanceChecker for SearxUpdater { - async fn check( - &self, - client: reqwest::Client, - _service: &Service, - instance: &Instance, - ) -> anyhow::Result { - let response = client.get(instance.url.clone()).send().await?; - Ok(response.status().is_success()) - } -} diff --git a/fastside-actualizer/src/services/searxng.rs b/fastside-actualizer/src/services/searxng.rs new file mode 100644 index 00000000..a5272b5e --- /dev/null +++ b/fastside-actualizer/src/services/searxng.rs @@ -0,0 +1,66 @@ +use std::collections::HashMap; + +use crate::{types::ServiceUpdater, ChangesSummary}; +use async_trait::async_trait; +use fastside_shared::serde_types::Instance; +use serde::Deserialize; +use url::Url; + +pub struct SearxngUpdater { + pub instances_url: String, +} + +impl SearxngUpdater { + pub fn new() -> Self { + Self { + instances_url: "https://searx.space/data/instances.json".to_string(), + } + } +} + +impl Default for SearxngUpdater { + fn default() -> Self { + Self::new() + } +} + +#[derive(Debug, Deserialize)] +struct InstancesResponse { + instances: HashMap, +} + +#[async_trait] +impl ServiceUpdater for SearxngUpdater { + async fn update( + &self, + client: reqwest::Client, + current_instances: &[Instance], + changes_summary: ChangesSummary, + ) -> anyhow::Result> { + let response = client.get(&self.instances_url).send().await?; + let response_str = response.text().await?; + let parsed: InstancesResponse = serde_json::from_str(&response_str)?; + + let mut instances = current_instances.to_vec(); + let mut new_instances = Vec::new(); + + for url in parsed.instances.keys() { + if current_instances.iter().any(|i| &i.url == url) { + continue; + } + + new_instances.push(Instance::from(url.clone())); + } + + changes_summary + .set_new_instances_added( + "searxng", + new_instances.iter().map(|i| i.url.clone()).collect(), + ) + .await; + + instances.extend(new_instances); + + Ok(instances) + } +} diff --git a/fastside-actualizer/src/services/simplytranslate.rs b/fastside-actualizer/src/services/simplytranslate.rs new file mode 100644 index 00000000..b151ff49 --- /dev/null +++ b/fastside-actualizer/src/services/simplytranslate.rs @@ -0,0 +1,95 @@ +use crate::{types::ServiceUpdater, ChangesSummary}; +use async_trait::async_trait; +use fastside_shared::serde_types::Instance; +use serde::Deserialize; +use url::Url; + +pub struct SimplyTranslateUpdater { + pub instances_url: String, +} + +impl SimplyTranslateUpdater { + pub fn new() -> Self { + Self { + instances_url: "https://codeberg.org/SimpleWeb/Website/raw/branch/master/config.json" + .to_string(), + } + } +} + +impl Default for SimplyTranslateUpdater { + fn default() -> Self { + Self::new() + } +} + +#[derive(Debug, Deserialize)] +struct Project { + id: String, + #[serde(default)] + instances: Vec, + #[serde(default)] + onion_instances: Vec, + #[serde(default)] + i2p_instances: Vec, +} + +#[derive(Debug, Deserialize)] +struct InstancesResponse { + projects: Vec, +} + +#[async_trait] +impl ServiceUpdater for SimplyTranslateUpdater { + async fn update( + &self, + client: reqwest::Client, + current_instances: &[Instance], + changes_summary: ChangesSummary, + ) -> anyhow::Result> { + let response = client.get(&self.instances_url).send().await?; + let response_str = response.text().await?; + let parsed: InstancesResponse = serde_json::from_str(&response_str)?; + + let mut instances = current_instances.to_vec(); + let mut new_instances = Vec::new(); + + let st_project = parsed + .projects + .iter() + .find(|p| p.id == "simplytranslate") + .ok_or(anyhow::anyhow!( + "No project with id 'simplytranslate' found" + ))?; + + for domain in st_project + .instances + .iter() + .map(|i| format!("https://{}", i)) + .chain( + st_project + .i2p_instances + .iter() + .chain(st_project.onion_instances.iter()) + .map(|i| format!("http://{}", i)), + ) + { + let url = Url::parse(domain.as_str())?; + if current_instances.iter().any(|i| i.url == url) { + continue; + } + new_instances.push(Instance::from(url.clone())); + } + + changes_summary + .set_new_instances_added( + "simplytranslate", + new_instances.iter().map(|i| i.url.clone()).collect(), + ) + .await; + + instances.extend(new_instances); + + Ok(instances) + } +} diff --git a/fastside-actualizer/src/services/tent.rs b/fastside-actualizer/src/services/tent.rs new file mode 100644 index 00000000..14043984 --- /dev/null +++ b/fastside-actualizer/src/services/tent.rs @@ -0,0 +1,68 @@ +use crate::{types::ServiceUpdater, ChangesSummary}; +use async_trait::async_trait; +use fastside_shared::serde_types::Instance; +use serde::Deserialize; +use url::Url; + +pub struct TentUpdater { + pub instances_url: String, +} + +impl TentUpdater { + pub fn new() -> Self { + Self { + instances_url: "https://forgejo.sny.sh/sun/Tent/raw/branch/main/instances.json" + .to_string(), + } + } +} + +impl Default for TentUpdater { + fn default() -> Self { + Self::new() + } +} + +#[derive(Debug, Deserialize)] +struct TentInstance { + url: Url, +} + +#[derive(Debug, Deserialize)] +struct InstancesResponse(Vec); + +#[async_trait] +impl ServiceUpdater for TentUpdater { + async fn update( + &self, + client: reqwest::Client, + current_instances: &[Instance], + changes_summary: ChangesSummary, + ) -> anyhow::Result> { + let response = client.get(&self.instances_url).send().await?; + let response_str = response.text().await?; + let parsed: InstancesResponse = serde_json::from_str(&response_str)?; + + let mut instances = current_instances.to_vec(); + let mut new_instances = Vec::new(); + + for instance in parsed.0 { + let url = instance.url; + if current_instances.iter().any(|i| i.url == url) { + continue; + } + new_instances.push(Instance::from(url.clone())); + } + + changes_summary + .set_new_instances_added( + "tent", + new_instances.iter().map(|i| i.url.clone()).collect(), + ) + .await; + + instances.extend(new_instances); + + Ok(instances) + } +} diff --git a/fastside-actualizer/src/utils/mod.rs b/fastside-actualizer/src/utils/mod.rs index 5355ca0f..beda2d47 100644 --- a/fastside-actualizer/src/utils/mod.rs +++ b/fastside-actualizer/src/utils/mod.rs @@ -1,3 +1,4 @@ pub mod log_err; pub mod normalize; pub mod tags; +pub mod url; diff --git a/fastside-actualizer/src/utils/url.rs b/fastside-actualizer/src/utils/url.rs new file mode 100644 index 00000000..e4ccadf0 --- /dev/null +++ b/fastside-actualizer/src/utils/url.rs @@ -0,0 +1,7 @@ +pub fn default_domain_scheme(domain: &str) -> String { + match domain { + _ if domain.ends_with(".onion") => "http://".to_string(), + _ if domain.ends_with(".i2p") => "http://".to_string(), + _ => "https://".to_string(), + } +} diff --git a/services.json b/services.json index 91cf70c7..a3853ac8 100644 --- a/services.json +++ b/services.json @@ -2671,7 +2671,7 @@ "allowed_http_codes": "200", "search_string": null, "regexes": [], - "aliases": [], + "aliases": ["librey"], "source_link": null, "deprecated_message": null, "instances": [ @@ -7293,4 +7293,4 @@ "instances": [] } ] -} \ No newline at end of file +}