chore(dev): rm bg qc; add fs template config

README.md

@@ -50,7 +50,7 @@ Support
 
 ## Getting started
 
-Let's step through some common tasks and core functions of `Cerebro` and its data application and reporting stack. This section provides some examples of how to get started quickly with `Cerebro`. For more details and how to deploy and operate the full application in production please see the [documentation](). 
+Let's step through some common tasks and core functions of `Cerebro` and the application and reporting stack. This section provides some examples of how to get started quickly with `Cerebro`. For more details and how to deploy and operate the full application in production please see the [documentation](). 
 
 Minimum requirements:
 
@@ -98,7 +98,15 @@ You do not need the `Docker` stack for core metagenome diagnostic pipelines and
 ```
 
 
-## Cerebro API and FS
+## Cerebro API
+
+### Quick start
+
+```
+
+```
+
+## Cerebro FS
 
 ### Quick start
 

cerebro/cerebro/src/cli.rs

@@ -66,6 +66,10 @@ fn main() -> anyhow::Result<(), StackConfigError> {
                                     log::warn!("Insecure deployment on localhost using standard root and admin user credentials for MongoDB (root:root, admin:admin) and Cerebro (admin@cerebro:admin)");
                                     cerebro::stack::deploy::StackConfig::default_localhost_insecure_from_args(args, args.interactive)?
                                 },
+                                StackConfigTemplate::LocalhostFs => {
+                                    log::warn!("Deployment of CerebroFS for proxied configuration only");  // uses insecure localhost stack config, since we do not launch the full stack
+                                    cerebro::stack::deploy::StackConfig::default_localhost_fs_from_args(args, args.interactive)?
+                                },
                                 StackConfigTemplate::Web => {
                                     cerebro::stack::deploy::StackConfig::default_web_from_args(args, args.interactive)?
                                 }

cerebro/cerebro/src/stack/deploy.rs

@@ -620,6 +620,7 @@ impl DataCenterConfig {
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct FileSystemConfig {
     pub enabled: bool,
+    pub fs_only: bool,
     pub replication: String,
     pub primary: Option<DataCenterConfig>,
     pub secondary: Option<DataCenterConfig>,
@@ -628,16 +629,18 @@ impl Default for FileSystemConfig {
     fn default() -> Self {
         Self {
             enabled: false,
+            fs_only: false,
             replication: String::from("100"),
             primary: None,
             secondary: None
         }
     }
 }
 impl FileSystemConfig {
-    pub fn default_localhost(fs_primary: &PathBuf, fs_secondary: &PathBuf) -> Self {
+    pub fn default_localhost(fs_primary: &PathBuf, fs_secondary: &PathBuf, fs_only: bool) -> Self {
         Self {
             enabled: true,
+            fs_only,
             replication: String::from("100"),
             primary: Some(DataCenterConfig::default_primary(fs_primary)),
             secondary: Some(DataCenterConfig::default_secondary(fs_secondary)),
@@ -646,6 +649,7 @@ impl FileSystemConfig {
     pub fn default_web() -> Self {
         Self {
             enabled: false,
+            fs_only: false,
             replication: String::from("100"),
             primary: None,
             secondary: None,
@@ -678,6 +682,7 @@ pub fn write_rendered_template(buf: &[u8], path: &PathBuf) -> Result<(), StackCo
 pub enum StackConfigTemplate {
     Localhost,
     LocalhostInsecure,
+    LocalhostFs,
     Web
 }
 
@@ -804,6 +809,7 @@ impl StackConfig {
             &fs_primary,
             &fs_secondary,
             false,
+            false
         ))
     }
     pub fn default_localhost_insecure_from_args(
@@ -825,9 +831,31 @@ impl StackConfig {
             &fs_primary,
             &fs_secondary,
             false,
+            false
         ))
     }
+    pub fn default_localhost_fs_from_args(
+        args: &StackDeployArgs,
+        interactive: bool,
+    ) -> Result<Self, StackConfigError> {
+
+        let fs_primary = Self::get_path_or_prompt(args.fs_primary.as_ref(), "--fs-primary", interactive)?;
+        let fs_secondary = Self::get_path_or_prompt(args.fs_secondary.as_ref(), "--fs-secondary", interactive)?;
 
+        Ok(Self::default_localhost(
+            "root",
+            "root",
+            "admin",
+            "admin",
+            "admin@cerebro",
+            "Administrator",
+            "admin",
+            &fs_primary,
+            &fs_secondary,
+            false,
+            true
+        ))
+    }
     pub fn default_web_from_args(
         args: &StackDeployArgs,
         interactive: bool,
@@ -870,6 +898,7 @@ impl StackConfig {
         fs_primary: &PathBuf,
         fs_secondary: &PathBuf,
         traefik_launch: bool,
+        fs_only: bool
     ) -> Self {
 
         log::info!("Creating default localhost deployment template");
@@ -898,7 +927,8 @@ impl StackConfig {
             ),
             fs: FileSystemConfig::default_localhost(
                 fs_primary, 
-                fs_secondary
+                fs_secondary,
+                fs_only
             )
         }
     }

cerebro/stack/pipe/src/cli.rs

@@ -15,7 +15,8 @@ fn main() -> anyhow::Result<()> {
 
                 ProcessCommands::Panviral(args) => {
                     let output = PanviralOutput::from(
-                        &args.input, args.id.clone(), args.background
+                        &args.input, 
+                        args.id.clone()
                     )?;
                     let quality_control = QualityControl::from_panviral(&output);
 
@@ -38,8 +39,7 @@ fn main() -> anyhow::Result<()> {
 
                     let output = PathogenOutput::from(
                         &args.input, 
-                        args.id.clone(), 
-                        args.background
+                        args.id.clone()
                     )?;
 
                     let quality_control = QualityControl::from_pathogen(&output);
@@ -78,7 +78,7 @@ fn main() -> anyhow::Result<()> {
 
                 ProcessCommands::Quality(args) => {
                     let output = QualityControlOutput::from(
-                        &args.input, args.id.clone(), args.background
+                        &args.input, args.id.clone()
                     )?;
 
                     let quality_control = QualityControl::from_quality(&output);

cerebro/stack/pipe/src/modules/mod.rs

@@ -1,4 +1,4 @@
 pub mod quality;
 pub mod pathogen;
 pub mod panviral;
-pub mod assembly;
+pub mod mag;

cerebro/stack/pipe/src/nextflow/mag.rs

@@ -0,0 +1,56 @@
+use std::path::PathBuf;
+use crate::utils::{
+    get_file_component, 
+    get_file_by_name, 
+    FileComponent
+};
+
+use vircov::vircov::VircovSummary;
+use crate::error::WorkflowError;
+use super::quality::{QualityControlFiles, QualityControlOutput};
+
+
+pub struct MagFiles {
+    pub qc: QualityControlFiles,
+    pub ncbi_blast: Option<PathBuf>,
+}
+impl MagFiles {
+    pub fn from(path: &PathBuf, id: &str) -> Result<Self, WorkflowError> {
+
+        Ok(Self{
+            qc: QualityControlFiles::from(path, id)?,
+            ncbi_blast: get_file_by_name(&path, &id, ".vircov.tsv")?,
+        })
+    }
+}
+
+pub struct MagOutput {
+    pub id: String,
+    pub qc: QualityControlOutput,
+    pub ncbi_blast: String,
+
+}
+impl MagOutput {
+
+    pub fn from(path: &PathBuf, id: Option<String>) -> Result<Self, WorkflowError> {
+
+        let id = match id {
+            Some(id) => id,
+            None => get_file_component(&path, FileComponent::FileName)?
+        };
+
+        let files = MagFiles::from(&path, &id)?; 
+
+        Ok(Self{
+            id: id.to_string(),
+            qc: QualityControlOutput::from_files(&id, &files.qc)? ,
+            ncbi_blast: match files.ncbi_blast { 
+                Some(ref path) => String::new(), 
+                None => {
+                    log::error!("No NCBI BLAST file detected for: {id}");
+                    return Err(WorkflowError::PipelineOutputNotFound)
+                }
+            },
+        })
+    }
+}

cerebro/stack/pipe/src/nextflow/mod.rs

@@ -1,4 +1,5 @@
 pub mod panviral;
 pub mod pathogen;
 pub mod quality;
+pub mod mag;
 pub mod sheet;

cerebro/stack/pipe/src/nextflow/panviral.rs

@@ -32,7 +32,7 @@ pub struct PanviralOutput {
 }
 impl PanviralOutput {
 
-    pub fn from(path: &PathBuf, id: Option<String>, background: bool) -> Result<Self, WorkflowError> {
+    pub fn from(path: &PathBuf, id: Option<String>) -> Result<Self, WorkflowError> {
 
         let id = match id {
             Some(id) => id,
@@ -43,7 +43,7 @@ impl PanviralOutput {
 
         Ok(Self{
             id: id.to_string(),
-            qc: QualityControlOutput::from_files(&id, &files.qc, background)? ,
+            qc: QualityControlOutput::from_files(&id, &files.qc)? ,
             vircov: match files.vircov { 
                 Some(ref path) => VircovSummary::from_tsv(path, true)?, 
                 None => {

cerebro/stack/pipe/src/nextflow/pathogen.rs

@@ -3,12 +3,11 @@ use std::path::PathBuf;
 
 use vircov::vircov::VircovSummary;
 
-use core::fmt;
 use std::path::Path;
 
 use serde::{Deserialize, Serialize};
 
-use crate::utils::{is_file_empty, read_tsv, read_tsv_skip};
+use crate::utils::{is_file_empty, read_tsv};
 use crate::error::WorkflowError;
 use crate::utils::{get_file_by_name, get_file_component, FileComponent};
 use super::quality::{QualityControlFiles, QualityControlOutput};
@@ -149,7 +148,7 @@ pub struct PathogenOutput {
 }
 impl PathogenOutput {
 
-    pub fn from(path: &PathBuf, id: Option<String>, background: bool) -> Result<Self, WorkflowError> {
+    pub fn from(path: &PathBuf, id: Option<String>) -> Result<Self, WorkflowError> {
 
         let id = match id {
             Some(id) => id,
@@ -160,7 +159,7 @@ impl PathogenOutput {
 
         Ok(Self{
             id: id.to_string(),
-            qc: QualityControlOutput::from_files(&id, &files.qc, background)?,
+            qc: QualityControlOutput::from_files(&id, &files.qc)?,
             profile: PathogenProfileOutput::from_files(&id, &files.profile)?,
             assembly: PathogenAssemblyOutput::from_files(&id, &files.profile)?
         })

cerebro/stack/pipe/src/nextflow/quality.rs

@@ -55,7 +55,7 @@ pub struct QualityControlOutput {
     pub output_scan: ScanReport,
 }
 impl QualityControlOutput {
-    pub fn from(path: &PathBuf, id: Option<String>, background: bool) -> Result<Self, WorkflowError> {
+    pub fn from(path: &PathBuf, id: Option<String>) -> Result<Self, WorkflowError> {
 
         let id = match id {
             Some(id) => id,
@@ -64,21 +64,11 @@ impl QualityControlOutput {
 
         let qc_files =  QualityControlFiles::from(path, &id)?;
 
-        if background {
-            Self::get_background_qc(&id, &qc_files)
-        } else {
-
-            Self::get_modular_qc(&id, &qc_files)
-        }
+        Self::get_modular_qc(&id, &qc_files)
     }
-    pub fn from_files(id: &str, qc_files: &QualityControlFiles, background: bool) -> Result<Self, WorkflowError> {
+    pub fn from_files(id: &str, qc_files: &QualityControlFiles) -> Result<Self, WorkflowError> {
 
-        if background {
-            Self::get_background_qc(&id, &qc_files)
-        } else {
-
-            Self::get_modular_qc(&id, &qc_files)
-        }
+        Self::get_modular_qc(&id, &qc_files)
 
     }
     pub fn get_modular_qc(id: &str, qc_files: &QualityControlFiles) -> Result<Self, WorkflowError> {
@@ -128,39 +118,4 @@ impl QualityControlOutput {
             },
         })
     }
-    pub fn get_background_qc(id: &str, qc_files: &QualityControlFiles) -> Result<Self, WorkflowError> {
-        Ok(QualityControlOutput { 
-            id: id.to_string(),
-            input_scan: match qc_files.input_scan { 
-                Some(ref path) => ScanReport::from_json(path)?, 
-                None => {
-                    log::error!("No required read scanning file (input) detected for {id}");
-                    return Err(WorkflowError::PipelineOutputNotFound)
-                }
-            },
-            reads_qc_fastp: match qc_files.reads_qc_fastp { 
-                Some(ref path) => Some(FastpReport::from_json(path)?), 
-                None => None
-            },
-            reads_qc_nanoq: match qc_files.reads_qc_nanoq { 
-                Some(ref path) => Some(NanoqReport::from_json(path)?), 
-                None => None
-            },
-            deduplication: match qc_files.deduplication { 
-                Some(ref path) => Some(DeduplicationReport::from_json(path)?), 
-                None => None
-            },
-            internal_controls: None,
-            synthetic_controls: None,
-            host_depletion: None,
-            background_alignment: None,
-            output_scan: match qc_files.output_scan { 
-                Some(ref path) => ScanReport::from_json(path)?, 
-                None => {
-                    log::error!("No required read scanning file (output) detected for {id}");
-                    return Err(WorkflowError::PipelineOutputNotFound)
-                }
-            }
-        })
-    }
 }

cerebro/stack/pipe/src/taxa/taxon.rs

@@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize};
 use taxonomy::{Taxonomy, GeneralTaxonomy, TaxRank};
 use vircov::vircov::VircovRecord;
 use std::{path::PathBuf, fs::File, io::BufReader, collections::HashMap};
-use crate::modules::assembly::AssemblyRecord;
+use crate::modules::mag::AssemblyRecord;
 use crate::modules::pathogen::{PathogenDetectionRecord, PathogenDetectionResult};
 use crate::{error::WorkflowError, utils::get_colored_string};
 

cerebro/stack/pipe/src/terminal.rs

@@ -396,13 +396,6 @@ pub struct ProcessArgs {
     /// Filter the output by taxranks
     #[clap(long, short = 'r')]
     pub filter_ranks: Option<Vec<PathogenDetectionRank>>,
-    /// Parse the background alignment from the quality control module variant
-    /// 
-    /// Combined reference for detecting organism, synthetic, internal controls 
-    /// and other relevant background and deplete in a single alignment before
-    /// other quality control steps
-    #[clap(long, short = 'a')]
-    pub background: bool,
     /// Paired-end reads (same identifier for each mate) used as input
     /// 
     /// Classifiers like Kraken2 or Metabuli output reads as unique reads

lib/processes/panviral.nf

@@ -91,8 +91,7 @@ process UploadOutput {
 
     script:
 
-    // We do not need the global --team authentication option for the client,
-    // the upload tasks use the data from stageJson for this
+    // We do not need the global --team authentication option for the client, the upload tasks use the data from stageJson
 
     """
     cerebro-client --url $apiUrl --token $authToken upload-panviral --quality $quality --panviral $panviral --taxonomy $taxonomyDirectory --pipeline-config $pipelineConfig --stage-json $stageJson --model ${sampleID}.cerebro.json