Add new service actualizers

fastside-actualizer/src/main.rs

@@ -54,6 +54,10 @@ enum Commands {
         /// Amount of maximum parallel requests.
         #[arg(long, default_value = None)]
         max_parallel: Option<usize>,
+        /// List of service names to actualize.
+        /// If not provided, all services will be actualized.
+        #[arg(short = 'u', long = "update", default_value = None)]
+        update_service_names: Option<Vec<String>>,
     },
 }
 
@@ -189,8 +193,7 @@ async fn update_service(
         Some(updater) => {
             let updated_instances_result = updater
                 .update(client, &service.instances, changes_summary.clone())
-                .await
-                .context("failed to update service");
+                .await;
             match updated_instances_result {
                 Ok(updated_instances) => {
                     debug!("Updated instances: {:?}", updated_instances);
@@ -321,6 +324,7 @@ async fn main() -> Result<()> {
             output,
             data,
             max_parallel,
+            update_service_names,
         }) => {
             let config = load_config(&cli.config).context("failed to load config")?;
 
@@ -348,12 +352,21 @@ async fn main() -> Result<()> {
                     std::fs::read_to_string(services).context("failed to read services file")?;
                 serde_json::from_str(&data_content).context("failed to parse services file")?
             };
-            let mut services_data = stored_data
+            let mut services_data: HashMap<String, Service> = stored_data
                 .services
                 .into_iter()
                 .map(|service| (service.name.clone(), service))
                 .collect();
 
+            // Check if all services from update_service_names exist
+            if let Some(update_service_names) = update_service_names {
+                for name in update_service_names.iter() {
+                    if !services_data.contains_key(name) {
+                        return Err(anyhow!("service {name:?} does not exist"));
+                    }
+                }
+            }
+
             let changes_summary = ChangesSummary::new();
 
             let start = std::time::Instant::now();
@@ -362,8 +375,21 @@ async fn main() -> Result<()> {
             actualizer_data.remove_removed_instances(&services_data);
 
             let update_service_client = reqwest::Client::new();
-            let length = services_data.len();
-            for (i, (name, service)) in services_data.iter_mut().enumerate() {
+
+            // Filter services data
+            let mut filtered_services_data = services_data
+                .iter_mut()
+                .filter(|(name, _)| {
+                    if let Some(update_service_names) = update_service_names {
+                        update_service_names.contains(name)
+                    } else {
+                        true
+                    }
+                })
+                .collect::<HashMap<_, _>>();
+            let length = filtered_services_data.len();
+
+            for (i, (name, service)) in filtered_services_data.iter_mut().enumerate() {
                 info!(
                     "Actualizing service {name} ({i}/{length})",
                     name = name,

fastside-actualizer/src/services/breezewiki.rs

@@ -0,0 +1,67 @@
+use crate::{types::ServiceUpdater, ChangesSummary};
+use async_trait::async_trait;
+use fastside_shared::serde_types::Instance;
+use serde::Deserialize;
+use url::Url;
+
+pub struct BreezewikiUpdater {
+    pub instances_url: String,
+}
+
+impl BreezewikiUpdater {
+    pub fn new() -> Self {
+        Self {
+            instances_url: "https://docs.breezewiki.com/files/instances.json".to_string(),
+        }
+    }
+}
+
+impl Default for BreezewikiUpdater {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[derive(Debug, Deserialize)]
+struct BreezewikiInstance {
+    instance: Url,
+}
+
+#[derive(Debug, Deserialize)]
+struct InstancesResponse(Vec<BreezewikiInstance>);
+
+#[async_trait]
+impl ServiceUpdater for BreezewikiUpdater {
+    async fn update(
+        &self,
+        client: reqwest::Client,
+        current_instances: &[Instance],
+        changes_summary: ChangesSummary,
+    ) -> anyhow::Result<Vec<Instance>> {
+        let response = client.get(&self.instances_url).send().await?;
+        let response_str = response.text().await?;
+        let parsed: InstancesResponse = serde_json::from_str(&response_str)?;
+
+        let mut instances = current_instances.to_vec();
+        let mut new_instances = Vec::new();
+
+        for instance in parsed.0 {
+            let url = instance.instance;
+            if current_instances.iter().any(|i| i.url == url) {
+                continue;
+            }
+            new_instances.push(Instance::from(url.clone()));
+        }
+
+        changes_summary
+            .set_new_instances_added(
+                "breezewiki",
+                new_instances.iter().map(|i| i.url.clone()).collect(),
+            )
+            .await;
+
+        instances.extend(new_instances);
+
+        Ok(instances)
+    }
+}

fastside-actualizer/src/services/gothub.rs

@@ -0,0 +1,69 @@
+use crate::{types::ServiceUpdater, ChangesSummary};
+use async_trait::async_trait;
+use fastside_shared::serde_types::Instance;
+use serde::Deserialize;
+use url::Url;
+
+pub struct GothubUpdater {
+    pub instances_url: String,
+}
+
+impl GothubUpdater {
+    pub fn new() -> Self {
+        Self {
+            instances_url:
+                "https://codeberg.org/gothub/gothub-instances/raw/branch/master/instances.json"
+                    .to_string(),
+        }
+    }
+}
+
+impl Default for GothubUpdater {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[derive(Debug, Deserialize)]
+struct GothubInstance {
+    link: Url,
+}
+
+#[derive(Debug, Deserialize)]
+struct InstancesResponse(Vec<GothubInstance>);
+
+#[async_trait]
+impl ServiceUpdater for GothubUpdater {
+    async fn update(
+        &self,
+        client: reqwest::Client,
+        current_instances: &[Instance],
+        changes_summary: ChangesSummary,
+    ) -> anyhow::Result<Vec<Instance>> {
+        let response = client.get(&self.instances_url).send().await?;
+        let response_str = response.text().await?;
+        let parsed: InstancesResponse = serde_json::from_str(&response_str)?;
+
+        let mut instances = current_instances.to_vec();
+        let mut new_instances = Vec::new();
+
+        for instance in parsed.0 {
+            let url = instance.link;
+            if current_instances.iter().any(|i| i.url == url) {
+                continue;
+            }
+            new_instances.push(Instance::from(url.clone()));
+        }
+
+        changes_summary
+            .set_new_instances_added(
+                "gothub",
+                new_instances.iter().map(|i| i.url.clone()).collect(),
+            )
+            .await;
+
+        instances.extend(new_instances);
+
+        Ok(instances)
+    }
+}

fastside-actualizer/src/services/invidious.rs

@@ -0,0 +1,64 @@
+use crate::{types::ServiceUpdater, utils::url::default_domain_scheme, ChangesSummary};
+use async_trait::async_trait;
+use fastside_shared::serde_types::Instance;
+use serde::Deserialize;
+use url::Url;
+
+pub struct InvidiousUpdater {
+    pub instances_url: String,
+}
+
+impl InvidiousUpdater {
+    pub fn new() -> Self {
+        Self {
+            instances_url: "https://api.invidious.io/instances.json".to_string(),
+        }
+    }
+}
+
+impl Default for InvidiousUpdater {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[derive(Debug, Deserialize)]
+struct InstancesResponse(Vec<(String, serde_json::Value)>);
+
+#[async_trait]
+impl ServiceUpdater for InvidiousUpdater {
+    async fn update(
+        &self,
+        client: reqwest::Client,
+        current_instances: &[Instance],
+        changes_summary: ChangesSummary,
+    ) -> anyhow::Result<Vec<Instance>> {
+        let response = client.get(&self.instances_url).send().await?;
+        let response_str = response.text().await?;
+        let parsed: InstancesResponse = serde_json::from_str(&response_str)?;
+
+        let mut instances = current_instances.to_vec();
+        let mut new_instances = Vec::new();
+
+        for instance in parsed.0 {
+            let domain = instance.0;
+            let scheme = default_domain_scheme(domain.as_str());
+            let url = Url::parse(&format!("{}{}", scheme, domain))?;
+            if current_instances.iter().any(|i| i.url == url) {
+                continue;
+            }
+            new_instances.push(Instance::from(url.clone()));
+        }
+
+        changes_summary
+            .set_new_instances_added(
+                "invidious",
+                new_instances.iter().map(|i| i.url.clone()).collect(),
+            )
+            .await;
+
+        instances.extend(new_instances);
+
+        Ok(instances)
+    }
+}

fastside-actualizer/src/services/libreddit.rs

@@ -0,0 +1,71 @@
+use crate::{types::ServiceUpdater, ChangesSummary};
+use async_trait::async_trait;
+use fastside_shared::serde_types::Instance;
+use serde::Deserialize;
+use url::Url;
+
+pub struct LibredditUpdater {
+    pub instances_url: String,
+}
+
+impl LibredditUpdater {
+    pub fn new() -> Self {
+        Self {
+            instances_url:
+                "https://raw.githubusercontent.com/redlib-org/redlib-instances/main/instances.json"
+                    .to_string(),
+        }
+    }
+}
+
+impl Default for LibredditUpdater {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[derive(Debug, Deserialize)]
+struct LibredditInstance {
+    url: Url,
+}
+
+#[derive(Debug, Deserialize)]
+struct InstancesResponse {
+    instances: Vec<LibredditInstance>,
+}
+
+#[async_trait]
+impl ServiceUpdater for LibredditUpdater {
+    async fn update(
+        &self,
+        client: reqwest::Client,
+        current_instances: &[Instance],
+        changes_summary: ChangesSummary,
+    ) -> anyhow::Result<Vec<Instance>> {
+        let response = client.get(&self.instances_url).send().await?;
+        let response_str = response.text().await?;
+        let parsed: InstancesResponse = serde_json::from_str(&response_str)?;
+
+        let mut instances = current_instances.to_vec();
+        let mut new_instances = Vec::new();
+
+        for instance in parsed.instances {
+            let url = instance.url;
+            if current_instances.iter().any(|i| i.url == url) {
+                continue;
+            }
+            new_instances.push(Instance::from(url.clone()));
+        }
+
+        changes_summary
+            .set_new_instances_added(
+                "libreddit",
+                new_instances.iter().map(|i| i.url.clone()).collect(),
+            )
+            .await;
+
+        instances.extend(new_instances);
+
+        Ok(instances)
+    }
+}