From 62306fdc48d021e3063004d7d6d9b3c42ff94307 Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Tue, 10 Oct 2023 11:01:34 +0800 Subject: [PATCH 01/22] add devtool: text-optimizer. --- Cargo.toml | 1 + devtools/text-optimizer/Cargo.toml | 14 ++ devtools/text-optimizer/src/backfill.rs | 42 +++++ .../src/extractors/clap_extractor.rs | 32 ++++ .../src/extractors/log_extractor.rs | 32 ++++ devtools/text-optimizer/src/extractors/mod.rs | 164 ++++++++++++++++++ .../src/extractors/std_output_extractor.rs | 27 +++ .../src/extractors/thiserror_extractor.rs | 32 ++++ devtools/text-optimizer/src/main.rs | 52 ++++++ devtools/text-optimizer/src/types.rs | 107 ++++++++++++ devtools/text-optimizer/src/yaml_processor.rs | 37 ++++ 11 files changed, 540 insertions(+) create mode 100644 devtools/text-optimizer/Cargo.toml create mode 100644 devtools/text-optimizer/src/backfill.rs create mode 100644 devtools/text-optimizer/src/extractors/clap_extractor.rs create mode 100644 devtools/text-optimizer/src/extractors/log_extractor.rs create mode 100644 devtools/text-optimizer/src/extractors/mod.rs create mode 100644 devtools/text-optimizer/src/extractors/std_output_extractor.rs create mode 100644 devtools/text-optimizer/src/extractors/thiserror_extractor.rs create mode 100644 devtools/text-optimizer/src/main.rs create mode 100644 devtools/text-optimizer/src/types.rs create mode 100644 devtools/text-optimizer/src/yaml_processor.rs diff --git a/Cargo.toml b/Cargo.toml index c7711140c3..8fd0cb3c2b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,6 +92,7 @@ members = [ "util/launcher", "ckb-bin" ] +exclude = ["devtools/text-optimizer"] [workspace.dependencies] tempfile = "3" diff --git a/devtools/text-optimizer/Cargo.toml b/devtools/text-optimizer/Cargo.toml new file mode 100644 index 0000000000..5f88909672 --- /dev/null +++ b/devtools/text-optimizer/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "text-optimizer" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +cargo_metadata = "0.18" +clap = { version = "4", features = ["derive", "string"] } +proc-macro2 = { version = "1", features = ["span-locations"] } +serde = "1.0" +serde_yaml = "0.9" +syn = { version = "2", features = ["visit", "full"] } diff --git a/devtools/text-optimizer/src/backfill.rs b/devtools/text-optimizer/src/backfill.rs new file mode 100644 index 0000000000..7d9cd65534 --- /dev/null +++ b/devtools/text-optimizer/src/backfill.rs @@ -0,0 +1,42 @@ +use crate::yaml_processor::load_yaml; +use crate::LOG_TEXT_FILE; + +use std::path::PathBuf; +use std::{ + fs::File, + io::{BufRead, BufReader}, +}; + +pub fn backfill(input_dir: &PathBuf) { + let log_text_file = input_dir.to_owned().join(LOG_TEXT_FILE); + let log_text_list = load_yaml(&log_text_file).expect("load yaml"); + println!("{:#?}", log_text_list); + + for log_text in log_text_list { + let file_path = log_text.metadata().file(); + let code_line = log_text.metadata().start_line(); + + let file = File::open(file_path).expect("open file"); + let reader = BufReader::new(file); + let lines = reader + .lines() + .map(|line| line.unwrap()) + .collect::>(); + + let _start_line = code_line.saturating_sub(3); // Ensure start_line is at least 0 + let _end_line = (code_line + 3).min(lines.len()); + + // for i in start_line..end_line { + // if lines[i].contains(&log_text.original()) { + // // Replace the line containing the original text + // lines[i] = lines[i].replace(&log_info.original, &log_info.editable); + // } + // } + + // let updated_content = lines.join("\n"); + + // // Write the updated content back to the file + // let mut file = File::create(file_path)?; + // file.write_all(updated_content.as_bytes())?; + } +} diff --git a/devtools/text-optimizer/src/extractors/clap_extractor.rs b/devtools/text-optimizer/src/extractors/clap_extractor.rs new file mode 100644 index 0000000000..1dd49ccdf5 --- /dev/null +++ b/devtools/text-optimizer/src/extractors/clap_extractor.rs @@ -0,0 +1,32 @@ +use super::{extract_contents_in_brackets, ClapExtractor}; +use crate::types::{Category, Meta, TextInfo}; +use std::str::FromStr; +use syn::Expr::{self}; +use syn::Lit::Str; + +impl syn::visit::Visit<'_> for ClapExtractor { + fn visit_expr_method_call(&mut self, expr: &syn::ExprMethodCall) { + let method_ident = &expr.method; + if *method_ident == "about" || *method_ident == "help" { + if let Some(Expr::Lit(lit)) = expr.args.first() { + if let Str(lit_str) = &lit.lit { + let lit = lit_str.token().to_string(); + + if let Some(text) = extract_contents_in_brackets(lit) { + println!("Found format string: {}", text); + + let span = lit_str.span(); + let start_line = span.start().line; + let end_line = span.end().line; + let category = + Category::from_str(method_ident.to_string().as_str()).unwrap(); + let meta = + Meta::new(category, self.file_path.to_owned(), start_line, end_line); + self.add_text_info(TextInfo::new(text, meta)); + } + } + } + } + syn::visit::visit_expr_method_call(self, expr); + } +} diff --git a/devtools/text-optimizer/src/extractors/log_extractor.rs b/devtools/text-optimizer/src/extractors/log_extractor.rs new file mode 100644 index 0000000000..414af42661 --- /dev/null +++ b/devtools/text-optimizer/src/extractors/log_extractor.rs @@ -0,0 +1,32 @@ +use super::{extract_contents_in_brackets, LogExtractor}; +use crate::types::{Category, Meta, TextInfo}; +use std::str::FromStr; +use syn::spanned::Spanned; +use syn::Macro; + +impl syn::visit::Visit<'_> for LogExtractor { + fn visit_macro(&mut self, node: &Macro) { + if let Some(ident) = node.path.get_ident() { + // Determine if the macro is println! + if ident == "error" + || ident == "warn" + || ident == "info" + || ident == "debug" + || ident == "trace" + { + let lit = node.tokens.to_string(); + + if let Some(text) = extract_contents_in_brackets(lit) { + println!("Found format string: {}", text); + + let span = node.span(); + let start_line = span.start().line; + let end_line = span.end().line; + let category = Category::from_str(ident.to_string().as_str()).unwrap(); + let meta = Meta::new(category, self.file_path.to_owned(), start_line, end_line); + self.add_text_info(TextInfo::new(text, meta)); + } + } + } + } +} diff --git a/devtools/text-optimizer/src/extractors/mod.rs b/devtools/text-optimizer/src/extractors/mod.rs new file mode 100644 index 0000000000..63267fba00 --- /dev/null +++ b/devtools/text-optimizer/src/extractors/mod.rs @@ -0,0 +1,164 @@ +pub mod clap_extractor; +pub mod log_extractor; +pub mod std_output_extractor; +pub mod thiserror_extractor; + +use crate::types::TextInfo; +use crate::{ + yaml_processor::save_yaml, CLAP_TEXT_FILE, LOG_TEXT_FILE, STD_OUTPUT_TEXT_FILE, + THISERROR_TEXT_FILE, +}; + +use cargo_metadata::MetadataCommand; +use syn::visit::visit_file; + +use std::{ + fs, + path::{Path, PathBuf}, +}; + +macro_rules! define_extractor { + ($struct_name:ident) => { + #[derive(Default)] + pub struct $struct_name { + list: Vec, + file_path: PathBuf, + } + + impl $struct_name { + pub fn new() -> Self { + $struct_name::default() + } + + pub fn reset_analysis_path(&mut self, file_path: &PathBuf) { + self.file_path = file_path.to_owned(); + } + + pub fn add_text_info(&mut self, text_info: TextInfo) { + self.list.push(text_info) + } + + pub fn get_text_list(&self) -> &[TextInfo] { + &self.list + } + } + }; +} + +define_extractor!(ClapExtractor); +define_extractor!(LogExtractor); +define_extractor!(StdOutputExtractor); +define_extractor!(ThiserrorExtractor); + +pub fn extract(project_root: PathBuf, output_dir: &PathBuf) { + // extractors + let mut clap_extractor = ClapExtractor::new(); + let mut log_extractor = LogExtractor::new(); + let mut std_output_extractor = StdOutputExtractor::new(); + let mut thiserror_extractor = ThiserrorExtractor::new(); + + let project_metadata = MetadataCommand::new() + .manifest_path(project_root) + .exec() + .expect("Failed to get current directory"); + + for package in project_metadata.workspace_packages() { + println!("Crate Name: {}", package.name); + println!("Crate Version: {}", package.version); + + let crate_path = Path::new(&package.manifest_path).parent().unwrap(); + process_rs_files_in_src( + crate_path, + &mut log_extractor, + &mut std_output_extractor, + &mut thiserror_extractor, + &mut clap_extractor, + ); + + println!(); + } + + save_as_file( + output_dir, + clap_extractor, + log_extractor, + std_output_extractor, + thiserror_extractor, + ); +} + +pub fn process_rs_files_in_src( + src_dir: &Path, + log_extractor: &mut LogExtractor, + std_output_extractor: &mut StdOutputExtractor, + thiserror_extractor: &mut ThiserrorExtractor, + clap_extractor: &mut ClapExtractor, +) { + if let Ok(entries) = fs::read_dir(&src_dir.join("src")) { + for entry in entries.flatten() { + if let Some(file_name) = entry.file_name().to_str() { + if file_name.ends_with(".rs") { + let file_path = entry.path(); + println!("Found .rs file: {:?}", file_path); + + // reset file path + clap_extractor.reset_analysis_path(&file_path); + log_extractor.reset_analysis_path(&file_path); + std_output_extractor.reset_analysis_path(&file_path); + thiserror_extractor.reset_analysis_path(&file_path); + + let file_content = fs::read_to_string(&file_path).expect("Failed to read file"); + if let Ok(syntax_tree) = syn::parse_file(&file_content) { + visit_file(clap_extractor, &syntax_tree); + visit_file(log_extractor, &syntax_tree); + visit_file(std_output_extractor, &syntax_tree); + visit_file(thiserror_extractor, &syntax_tree); + } else { + println!("Failed to parse .rs file: {:?}", file_path); + } + } + } + } + } +} + +pub fn extract_contents_in_brackets(lit: String) -> Option { + if let Some(start) = lit.find('"') { + if let Some(end) = lit.rfind('"') { + let format_string = &lit[start + 1..end]; + return Some(format_string.to_string()); + } + } + None +} + +fn save_as_file( + output_dir: &PathBuf, + clap_extractor: ClapExtractor, + log_extractor: LogExtractor, + std_output_extractor: StdOutputExtractor, + thiserror_extractor: ThiserrorExtractor, +) { + fs::create_dir_all(output_dir).expect("create dir all"); + + save_yaml( + &output_dir.join(LOG_TEXT_FILE), + log_extractor.get_text_list(), + ) + .expect("save yaml"); + save_yaml( + &output_dir.join(CLAP_TEXT_FILE), + clap_extractor.get_text_list(), + ) + .expect("save yaml"); + save_yaml( + &output_dir.join(STD_OUTPUT_TEXT_FILE), + std_output_extractor.get_text_list(), + ) + .expect("save yaml"); + save_yaml( + &output_dir.join(THISERROR_TEXT_FILE), + thiserror_extractor.get_text_list(), + ) + .expect("save yaml"); +} diff --git a/devtools/text-optimizer/src/extractors/std_output_extractor.rs b/devtools/text-optimizer/src/extractors/std_output_extractor.rs new file mode 100644 index 0000000000..6469f08f29 --- /dev/null +++ b/devtools/text-optimizer/src/extractors/std_output_extractor.rs @@ -0,0 +1,27 @@ +use super::{extract_contents_in_brackets, StdOutputExtractor}; +use crate::types::{Category, Meta, TextInfo}; +use std::str::FromStr; +use syn::spanned::Spanned; +use syn::Macro; + +impl syn::visit::Visit<'_> for StdOutputExtractor { + fn visit_macro(&mut self, node: &Macro) { + if let Some(ident) = node.path.get_ident() { + // Determine if the macro is println! + if ident == "println" || ident == "eprintln" { + let lit = node.tokens.to_string(); + + if let Some(text) = extract_contents_in_brackets(lit.to_owned()) { + println!("Found format string: {}", text); + + let span = node.span(); + let start_line = span.start().line; + let end_line = span.end().line; + let category = Category::from_str(ident.to_string().as_str()).unwrap(); + let meta = Meta::new(category, self.file_path.to_owned(), start_line, end_line); + self.add_text_info(TextInfo::new(text, meta)); + } + } + } + } +} diff --git a/devtools/text-optimizer/src/extractors/thiserror_extractor.rs b/devtools/text-optimizer/src/extractors/thiserror_extractor.rs new file mode 100644 index 0000000000..85c7c96304 --- /dev/null +++ b/devtools/text-optimizer/src/extractors/thiserror_extractor.rs @@ -0,0 +1,32 @@ +use super::{extract_contents_in_brackets, ThiserrorExtractor}; +use crate::types::{Category, Meta, TextInfo}; +use syn::Expr::{self, Lit}; +use syn::Lit::Str; + +impl syn::visit::Visit<'_> for ThiserrorExtractor { + fn visit_attribute(&mut self, attr: &syn::Attribute) { + if attr.path().is_ident("error") { + let precondition: Expr = attr.parse_args().unwrap(); + if let Lit(lit) = precondition { + if let Str(lit_str) = lit.lit { + let lit = lit_str.token().to_string(); + + if let Some(text) = extract_contents_in_brackets(lit) { + println!("Found format string: {}", text); + + let span = lit_str.span(); + let start_line = span.start().line; + let end_line = span.end().line; + let meta = Meta::new( + Category::ThisError, + self.file_path.to_owned(), + start_line, + end_line, + ); + self.add_text_info(TextInfo::new(text, meta)); + } + } + } + } + } +} diff --git a/devtools/text-optimizer/src/main.rs b/devtools/text-optimizer/src/main.rs new file mode 100644 index 0000000000..fde694c0b6 --- /dev/null +++ b/devtools/text-optimizer/src/main.rs @@ -0,0 +1,52 @@ +mod backfill; +mod extractors; +mod types; +mod yaml_processor; + +use backfill::backfill; +use clap::{Parser, Subcommand}; +use extractors::extract; +use std::path::PathBuf; + +pub const PROJECT_ROOT: &str = "../../Cargo.toml"; +pub const LOG_TEXT_FILE: &str = "log_text_list.yml"; +pub const CLAP_TEXT_FILE: &str = "clap_text_list.yml"; +pub const STD_OUTPUT_TEXT_FILE: &str = "std_output_text_list.yml"; +pub const THISERROR_TEXT_FILE: &str = "thiserror_text_list.yml"; + +#[derive(Parser)] +#[command(author, version, about, long_about = None)] +struct Cli { + #[command(subcommand)] + command: Option, +} + +#[derive(Subcommand)] +enum Commands { + /// extracting text + Extract { + /// specifies a directory path for .yml text files output + #[arg(short, long, default_value = PathBuf::from("./").into_os_string())] + output_dir: PathBuf, + }, + /// backfill text + Backfill { + /// specifies a directory path for .yml text files input + #[arg(short, long, default_value = PathBuf::from("./").into_os_string())] + input_dir: PathBuf, + }, +} + +fn main() { + let cli = Cli::parse(); + + match &cli.command { + Some(Commands::Extract { output_dir }) => { + extract(PathBuf::from(PROJECT_ROOT), output_dir); + } + Some(Commands::Backfill { input_dir }) => { + backfill(input_dir); + } + None => {} + } +} diff --git a/devtools/text-optimizer/src/types.rs b/devtools/text-optimizer/src/types.rs new file mode 100644 index 0000000000..7bbf3bedfb --- /dev/null +++ b/devtools/text-optimizer/src/types.rs @@ -0,0 +1,107 @@ +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; +use std::str::FromStr; + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] +pub struct TextInfo { + original: String, + editable: String, + metadata: Meta, +} + +impl TextInfo { + pub fn new(original: String, metadata: Meta) -> Self { + TextInfo { + original: original.to_owned(), + editable: original, + metadata, + } + } + + #[allow(dead_code)] + pub fn original(&self) -> &str { + &self.original + } + + #[allow(dead_code)] + pub fn editable(&self) -> &str { + &self.editable + } + + pub fn metadata(&self) -> &Meta { + &self.metadata + } +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] +pub struct Meta { + category: Category, + file: PathBuf, + start_line: usize, + end_line: usize, +} + +impl Meta { + pub fn new(category: Category, file: PathBuf, start_line: usize, end_line: usize) -> Self { + Meta { + category, + file, + start_line, + end_line, + } + } + + #[allow(dead_code)] + pub fn category(&self) -> &Category { + &self.category + } + + pub fn file(&self) -> &PathBuf { + &self.file + } + + pub fn start_line(&self) -> usize { + self.start_line + } + + #[allow(dead_code)] + pub fn end_line(&self) -> usize { + self.end_line + } +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] +pub enum Category { + ClapHelp, + ClapAbout, + + LogDebug, + LogWarn, + LogInfo, + LogError, + LogTrace, + + StdOutput, + StdError, + + ThisError, +} + +impl FromStr for Category { + type Err = (); + + fn from_str(s: &str) -> Result { + match s { + "help" => Ok(Category::ClapHelp), + "about" => Ok(Category::ClapAbout), + "debug" => Ok(Category::LogDebug), + "warn" => Ok(Category::LogWarn), + "info" => Ok(Category::LogInfo), + "error" => Ok(Category::LogError), + "trace" => Ok(Category::LogTrace), + "println" => Ok(Category::StdOutput), + "eprintln" => Ok(Category::StdError), + _ => Err(()), + } + } +} diff --git a/devtools/text-optimizer/src/yaml_processor.rs b/devtools/text-optimizer/src/yaml_processor.rs new file mode 100644 index 0000000000..3a730023d7 --- /dev/null +++ b/devtools/text-optimizer/src/yaml_processor.rs @@ -0,0 +1,37 @@ +use super::types::TextInfo; +use std::io::Read; +use std::{fs::File, path::PathBuf}; + +#[derive(Debug)] +pub enum MyError { + IoError(std::io::Error), + SerdeError(serde_yaml::Error), +} + +impl From for MyError { + fn from(error: std::io::Error) -> Self { + MyError::IoError(error) + } +} + +impl From for MyError { + fn from(error: serde_yaml::Error) -> Self { + MyError::SerdeError(error) + } +} + +pub fn save_yaml(file: &PathBuf, data: &[TextInfo]) -> Result<(), MyError> { + let file = File::create(file)?; + serde_yaml::to_writer(file, data)?; + Ok(()) +} + +pub fn load_yaml(filename: &PathBuf) -> Result, MyError> { + let mut file = File::open(filename)?; + let mut yaml_content = String::new(); + file.read_to_string(&mut yaml_content)?; + + let extracted_texts: Vec = serde_yaml::from_str(&yaml_content)?; + + Ok(extracted_texts) +} From aafbdcf513a7f9e944d44b8f3367361acc189af9 Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Tue, 10 Oct 2023 12:19:04 +0800 Subject: [PATCH 02/22] Add item count statistics. --- devtools/text-optimizer/src/yaml_processor.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/devtools/text-optimizer/src/yaml_processor.rs b/devtools/text-optimizer/src/yaml_processor.rs index 3a730023d7..61e6fa2d05 100644 --- a/devtools/text-optimizer/src/yaml_processor.rs +++ b/devtools/text-optimizer/src/yaml_processor.rs @@ -1,5 +1,6 @@ use super::types::TextInfo; use std::io::Read; +use std::io::Write; use std::{fs::File, path::PathBuf}; #[derive(Debug)] @@ -21,7 +22,8 @@ impl From for MyError { } pub fn save_yaml(file: &PathBuf, data: &[TextInfo]) -> Result<(), MyError> { - let file = File::create(file)?; + let mut file = File::create(file)?; + file.write_fmt(format_args!("# Number of TextInfo items: {}\n\n", data.len()))?; serde_yaml::to_writer(file, data)?; Ok(()) } From 51bc3d91c2ee564236354779c71205e637d10b87 Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Tue, 10 Oct 2023 15:24:16 +0800 Subject: [PATCH 03/22] fix log extractor: code line. --- .../src/extractors/log_extractor.rs | 22 +++++++++---------- devtools/text-optimizer/src/yaml_processor.rs | 5 ++++- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/devtools/text-optimizer/src/extractors/log_extractor.rs b/devtools/text-optimizer/src/extractors/log_extractor.rs index 414af42661..0139d15d71 100644 --- a/devtools/text-optimizer/src/extractors/log_extractor.rs +++ b/devtools/text-optimizer/src/extractors/log_extractor.rs @@ -1,7 +1,6 @@ use super::{extract_contents_in_brackets, LogExtractor}; use crate::types::{Category, Meta, TextInfo}; use std::str::FromStr; -use syn::spanned::Spanned; use syn::Macro; impl syn::visit::Visit<'_> for LogExtractor { @@ -14,17 +13,18 @@ impl syn::visit::Visit<'_> for LogExtractor { || ident == "debug" || ident == "trace" { - let lit = node.tokens.to_string(); + if let Some(open_paren) = node.tokens.clone().into_iter().next() { + if let Some(text) = extract_contents_in_brackets(open_paren.to_string()) { + println!("Found format string: {}", text); - if let Some(text) = extract_contents_in_brackets(lit) { - println!("Found format string: {}", text); - - let span = node.span(); - let start_line = span.start().line; - let end_line = span.end().line; - let category = Category::from_str(ident.to_string().as_str()).unwrap(); - let meta = Meta::new(category, self.file_path.to_owned(), start_line, end_line); - self.add_text_info(TextInfo::new(text, meta)); + let span = open_paren.span(); + let start_line = span.start().line; + let end_line = span.end().line; + let category = Category::from_str(ident.to_string().as_str()).unwrap(); + let meta = + Meta::new(category, self.file_path.to_owned(), start_line, end_line); + self.add_text_info(TextInfo::new(text, meta)); + } } } } diff --git a/devtools/text-optimizer/src/yaml_processor.rs b/devtools/text-optimizer/src/yaml_processor.rs index 61e6fa2d05..ebb6a4ed58 100644 --- a/devtools/text-optimizer/src/yaml_processor.rs +++ b/devtools/text-optimizer/src/yaml_processor.rs @@ -23,7 +23,10 @@ impl From for MyError { pub fn save_yaml(file: &PathBuf, data: &[TextInfo]) -> Result<(), MyError> { let mut file = File::create(file)?; - file.write_fmt(format_args!("# Number of TextInfo items: {}\n\n", data.len()))?; + file.write_fmt(format_args!( + "# Number of TextInfo items: {}\n\n", + data.len() + ))?; serde_yaml::to_writer(file, data)?; Ok(()) } From 8b0a0e1901167df0e918cb09dfba5ddb55c0fdcf Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Tue, 10 Oct 2023 15:55:59 +0800 Subject: [PATCH 04/22] fix std output extractor: code line. --- .../src/extractors/log_extractor.rs | 6 ++--- .../src/extractors/std_output_extractor.rs | 22 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/devtools/text-optimizer/src/extractors/log_extractor.rs b/devtools/text-optimizer/src/extractors/log_extractor.rs index 0139d15d71..86ebf20d94 100644 --- a/devtools/text-optimizer/src/extractors/log_extractor.rs +++ b/devtools/text-optimizer/src/extractors/log_extractor.rs @@ -13,11 +13,11 @@ impl syn::visit::Visit<'_> for LogExtractor { || ident == "debug" || ident == "trace" { - if let Some(open_paren) = node.tokens.clone().into_iter().next() { - if let Some(text) = extract_contents_in_brackets(open_paren.to_string()) { + if let Some(lit) = node.tokens.clone().into_iter().next() { + if let Some(text) = extract_contents_in_brackets(lit.to_string()) { println!("Found format string: {}", text); - let span = open_paren.span(); + let span = lit.span(); let start_line = span.start().line; let end_line = span.end().line; let category = Category::from_str(ident.to_string().as_str()).unwrap(); diff --git a/devtools/text-optimizer/src/extractors/std_output_extractor.rs b/devtools/text-optimizer/src/extractors/std_output_extractor.rs index 6469f08f29..ca968b59e4 100644 --- a/devtools/text-optimizer/src/extractors/std_output_extractor.rs +++ b/devtools/text-optimizer/src/extractors/std_output_extractor.rs @@ -1,7 +1,6 @@ use super::{extract_contents_in_brackets, StdOutputExtractor}; use crate::types::{Category, Meta, TextInfo}; use std::str::FromStr; -use syn::spanned::Spanned; use syn::Macro; impl syn::visit::Visit<'_> for StdOutputExtractor { @@ -9,17 +8,18 @@ impl syn::visit::Visit<'_> for StdOutputExtractor { if let Some(ident) = node.path.get_ident() { // Determine if the macro is println! if ident == "println" || ident == "eprintln" { - let lit = node.tokens.to_string(); + if let Some(lit) = node.tokens.clone().into_iter().next() { + if let Some(text) = extract_contents_in_brackets(lit.to_string()) { + println!("Found format string: {}", text); - if let Some(text) = extract_contents_in_brackets(lit.to_owned()) { - println!("Found format string: {}", text); - - let span = node.span(); - let start_line = span.start().line; - let end_line = span.end().line; - let category = Category::from_str(ident.to_string().as_str()).unwrap(); - let meta = Meta::new(category, self.file_path.to_owned(), start_line, end_line); - self.add_text_info(TextInfo::new(text, meta)); + let span = lit.span(); + let start_line = span.start().line; + let end_line = span.end().line; + let category = Category::from_str(ident.to_string().as_str()).unwrap(); + let meta = + Meta::new(category, self.file_path.to_owned(), start_line, end_line); + self.add_text_info(TextInfo::new(text, meta)); + } } } } From 0793c6faf78748cb7e7ca7a1d7ff9c90513a678b Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Tue, 10 Oct 2023 21:54:29 +0800 Subject: [PATCH 05/22] 1 impl recursive search for rs files. 2 optimize log display --- devtools/text-optimizer/Cargo.toml | 2 + devtools/text-optimizer/src/backfill.rs | 2 +- .../src/extractors/clap_extractor.rs | 2 +- .../src/extractors/log_extractor.rs | 3 +- devtools/text-optimizer/src/extractors/mod.rs | 43 +++++++++++-------- .../src/extractors/std_output_extractor.rs | 3 +- .../src/extractors/thiserror_extractor.rs | 17 +++++++- devtools/text-optimizer/src/main.rs | 3 ++ 8 files changed, 50 insertions(+), 25 deletions(-) diff --git a/devtools/text-optimizer/Cargo.toml b/devtools/text-optimizer/Cargo.toml index 5f88909672..9645c767cf 100644 --- a/devtools/text-optimizer/Cargo.toml +++ b/devtools/text-optimizer/Cargo.toml @@ -8,6 +8,8 @@ edition = "2021" [dependencies] cargo_metadata = "0.18" clap = { version = "4", features = ["derive", "string"] } +env_logger = "0.10" +log = "0.4" proc-macro2 = { version = "1", features = ["span-locations"] } serde = "1.0" serde_yaml = "0.9" diff --git a/devtools/text-optimizer/src/backfill.rs b/devtools/text-optimizer/src/backfill.rs index 7d9cd65534..20765b5386 100644 --- a/devtools/text-optimizer/src/backfill.rs +++ b/devtools/text-optimizer/src/backfill.rs @@ -10,7 +10,7 @@ use std::{ pub fn backfill(input_dir: &PathBuf) { let log_text_file = input_dir.to_owned().join(LOG_TEXT_FILE); let log_text_list = load_yaml(&log_text_file).expect("load yaml"); - println!("{:#?}", log_text_list); + log::debug!("{:#?}", log_text_list); for log_text in log_text_list { let file_path = log_text.metadata().file(); diff --git a/devtools/text-optimizer/src/extractors/clap_extractor.rs b/devtools/text-optimizer/src/extractors/clap_extractor.rs index 1dd49ccdf5..9f76c839db 100644 --- a/devtools/text-optimizer/src/extractors/clap_extractor.rs +++ b/devtools/text-optimizer/src/extractors/clap_extractor.rs @@ -13,7 +13,7 @@ impl syn::visit::Visit<'_> for ClapExtractor { let lit = lit_str.token().to_string(); if let Some(text) = extract_contents_in_brackets(lit) { - println!("Found format string: {}", text); + log::trace!("Found target text: {}", text); let span = lit_str.span(); let start_line = span.start().line; diff --git a/devtools/text-optimizer/src/extractors/log_extractor.rs b/devtools/text-optimizer/src/extractors/log_extractor.rs index 86ebf20d94..0c2e20ce09 100644 --- a/devtools/text-optimizer/src/extractors/log_extractor.rs +++ b/devtools/text-optimizer/src/extractors/log_extractor.rs @@ -6,7 +6,6 @@ use syn::Macro; impl syn::visit::Visit<'_> for LogExtractor { fn visit_macro(&mut self, node: &Macro) { if let Some(ident) = node.path.get_ident() { - // Determine if the macro is println! if ident == "error" || ident == "warn" || ident == "info" @@ -15,7 +14,7 @@ impl syn::visit::Visit<'_> for LogExtractor { { if let Some(lit) = node.tokens.clone().into_iter().next() { if let Some(text) = extract_contents_in_brackets(lit.to_string()) { - println!("Found format string: {}", text); + log::trace!("Found target text: {}", text); let span = lit.span(); let start_line = span.start().line; diff --git a/devtools/text-optimizer/src/extractors/mod.rs b/devtools/text-optimizer/src/extractors/mod.rs index 63267fba00..e253027ea1 100644 --- a/devtools/text-optimizer/src/extractors/mod.rs +++ b/devtools/text-optimizer/src/extractors/mod.rs @@ -63,19 +63,19 @@ pub fn extract(project_root: PathBuf, output_dir: &PathBuf) { .expect("Failed to get current directory"); for package in project_metadata.workspace_packages() { - println!("Crate Name: {}", package.name); - println!("Crate Version: {}", package.version); + log::info!("Scanning Crate: {}", package.name); - let crate_path = Path::new(&package.manifest_path).parent().unwrap(); + let crate_src_path = Path::new(&package.manifest_path) + .parent() + .expect("workspace member crate path") + .join("src"); process_rs_files_in_src( - crate_path, + &crate_src_path, &mut log_extractor, &mut std_output_extractor, &mut thiserror_extractor, &mut clap_extractor, ); - - println!(); } save_as_file( @@ -88,33 +88,42 @@ pub fn extract(project_root: PathBuf, output_dir: &PathBuf) { } pub fn process_rs_files_in_src( - src_dir: &Path, + dir_path: &Path, log_extractor: &mut LogExtractor, std_output_extractor: &mut StdOutputExtractor, thiserror_extractor: &mut ThiserrorExtractor, clap_extractor: &mut ClapExtractor, ) { - if let Ok(entries) = fs::read_dir(&src_dir.join("src")) { + if let Ok(entries) = fs::read_dir(dir_path) { for entry in entries.flatten() { - if let Some(file_name) = entry.file_name().to_str() { + let entry_path = entry.path(); + if entry_path.is_dir() { + process_rs_files_in_src( + &entry_path, + log_extractor, + std_output_extractor, + thiserror_extractor, + clap_extractor, + ); + } else if let Some(file_name) = entry.file_name().to_str() { if file_name.ends_with(".rs") { - let file_path = entry.path(); - println!("Found .rs file: {:?}", file_path); + log::trace!("Found .rs file: {:?}", entry_path); // reset file path - clap_extractor.reset_analysis_path(&file_path); - log_extractor.reset_analysis_path(&file_path); - std_output_extractor.reset_analysis_path(&file_path); - thiserror_extractor.reset_analysis_path(&file_path); + clap_extractor.reset_analysis_path(&entry_path); + log_extractor.reset_analysis_path(&entry_path); + std_output_extractor.reset_analysis_path(&entry_path); + thiserror_extractor.reset_analysis_path(&entry_path); - let file_content = fs::read_to_string(&file_path).expect("Failed to read file"); + let file_content = + fs::read_to_string(&entry_path).expect("Failed to read file"); if let Ok(syntax_tree) = syn::parse_file(&file_content) { visit_file(clap_extractor, &syntax_tree); visit_file(log_extractor, &syntax_tree); visit_file(std_output_extractor, &syntax_tree); visit_file(thiserror_extractor, &syntax_tree); } else { - println!("Failed to parse .rs file: {:?}", file_path); + log::error!("Failed to parse .rs file: {:?}", entry_path); } } } diff --git a/devtools/text-optimizer/src/extractors/std_output_extractor.rs b/devtools/text-optimizer/src/extractors/std_output_extractor.rs index ca968b59e4..321f4581bd 100644 --- a/devtools/text-optimizer/src/extractors/std_output_extractor.rs +++ b/devtools/text-optimizer/src/extractors/std_output_extractor.rs @@ -6,11 +6,10 @@ use syn::Macro; impl syn::visit::Visit<'_> for StdOutputExtractor { fn visit_macro(&mut self, node: &Macro) { if let Some(ident) = node.path.get_ident() { - // Determine if the macro is println! if ident == "println" || ident == "eprintln" { if let Some(lit) = node.tokens.clone().into_iter().next() { if let Some(text) = extract_contents_in_brackets(lit.to_string()) { - println!("Found format string: {}", text); + log::trace!("Found target text: {}", text); let span = lit.span(); let start_line = span.start().line; diff --git a/devtools/text-optimizer/src/extractors/thiserror_extractor.rs b/devtools/text-optimizer/src/extractors/thiserror_extractor.rs index 85c7c96304..a33927bb10 100644 --- a/devtools/text-optimizer/src/extractors/thiserror_extractor.rs +++ b/devtools/text-optimizer/src/extractors/thiserror_extractor.rs @@ -1,18 +1,31 @@ use super::{extract_contents_in_brackets, ThiserrorExtractor}; use crate::types::{Category, Meta, TextInfo}; +use syn::spanned::Spanned; use syn::Expr::{self, Lit}; use syn::Lit::Str; impl syn::visit::Visit<'_> for ThiserrorExtractor { fn visit_attribute(&mut self, attr: &syn::Attribute) { if attr.path().is_ident("error") { - let precondition: Expr = attr.parse_args().unwrap(); + let precondition: Expr = { + if let Ok(precondition) = attr.parse_args() { + precondition + } else { + let span = attr.span(); + log::warn!( + "parse args failed, ignore the file: {:?}, code line: {:?}", + self.file_path, + span.start().line + ); + return; + } + }; if let Lit(lit) = precondition { if let Str(lit_str) = lit.lit { let lit = lit_str.token().to_string(); if let Some(text) = extract_contents_in_brackets(lit) { - println!("Found format string: {}", text); + log::trace!("Found target text: {}", text); let span = lit_str.span(); let start_line = span.start().line; diff --git a/devtools/text-optimizer/src/main.rs b/devtools/text-optimizer/src/main.rs index fde694c0b6..6291ae9849 100644 --- a/devtools/text-optimizer/src/main.rs +++ b/devtools/text-optimizer/src/main.rs @@ -40,6 +40,9 @@ enum Commands { fn main() { let cli = Cli::parse(); + env_logger::init_from_env( + env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info")); + match &cli.command { Some(Commands::Extract { output_dir }) => { extract(PathBuf::from(PROJECT_ROOT), output_dir); From 6e1b9c84e8307c825bda857cfd2ef5d0cf626512 Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Tue, 10 Oct 2023 22:11:03 +0800 Subject: [PATCH 06/22] outputs statistics when the scan is finished. --- devtools/text-optimizer/src/extractors/mod.rs | 21 +++++++++++++++++++ devtools/text-optimizer/src/main.rs | 5 +++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/devtools/text-optimizer/src/extractors/mod.rs b/devtools/text-optimizer/src/extractors/mod.rs index e253027ea1..bb4c5677d9 100644 --- a/devtools/text-optimizer/src/extractors/mod.rs +++ b/devtools/text-optimizer/src/extractors/mod.rs @@ -150,24 +150,45 @@ fn save_as_file( ) { fs::create_dir_all(output_dir).expect("create dir all"); + println!(); + save_yaml( &output_dir.join(LOG_TEXT_FILE), log_extractor.get_text_list(), ) .expect("save yaml"); + println!( + "Extract LOG category text: {:?}", + log_extractor.get_text_list().len() + ); + save_yaml( &output_dir.join(CLAP_TEXT_FILE), clap_extractor.get_text_list(), ) .expect("save yaml"); + println!( + "Extract CLAP category text: {:?}", + clap_extractor.get_text_list().len() + ); + save_yaml( &output_dir.join(STD_OUTPUT_TEXT_FILE), std_output_extractor.get_text_list(), ) .expect("save yaml"); + println!( + "Extract STD OUPUT category text: {:?}", + std_output_extractor.get_text_list().len() + ); + save_yaml( &output_dir.join(THISERROR_TEXT_FILE), thiserror_extractor.get_text_list(), ) .expect("save yaml"); + println!( + "Extract THISERROR category text: {:?}", + thiserror_extractor.get_text_list().len() + ); } diff --git a/devtools/text-optimizer/src/main.rs b/devtools/text-optimizer/src/main.rs index 6291ae9849..4f1fad8c85 100644 --- a/devtools/text-optimizer/src/main.rs +++ b/devtools/text-optimizer/src/main.rs @@ -41,8 +41,9 @@ fn main() { let cli = Cli::parse(); env_logger::init_from_env( - env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info")); - + env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"), + ); + match &cli.command { Some(Commands::Extract { output_dir }) => { extract(PathBuf::from(PROJECT_ROOT), output_dir); From 3ca095a34a1749a3d586957c710dbe77a79d55fb Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Tue, 10 Oct 2023 22:18:42 +0800 Subject: [PATCH 07/22] unify the way thiserror references variables. --- util/types/src/core/error.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/types/src/core/error.rs b/util/types/src/core/error.rs index 9e807ed03d..23f30f8ea1 100644 --- a/util/types/src/core/error.rs +++ b/util/types/src/core/error.rs @@ -153,7 +153,7 @@ pub enum TransactionError { }, /// The transaction version does not match with the system expected. - #[error("MismatchedVersion: expected {}, got {}", expected, actual)] + #[error("MismatchedVersion: expected {expected}, got {actual}")] MismatchedVersion { /// The expected transaction version. expected: Version, @@ -178,7 +178,7 @@ pub enum TransactionError { }, /// Nervos DAO lock size mismatch. - #[error("The lock script size of deposit cell at index {} does not match the withdrawing cell at the same index", index)] + #[error("The lock script size of deposit cell at index {index} does not match the withdrawing cell at the same index")] DaoLockSizeMismatch { /// The index of mismatched DAO cells. index: usize, From e9fc182106670d5e6904d67dfb72ac32ab7b819c Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Tue, 10 Oct 2023 23:29:19 +0800 Subject: [PATCH 08/22] Prevent text-optimizer from interfering with workspaces --- Cargo.toml | 1 - devtools/text-optimizer/Cargo.toml | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 8fd0cb3c2b..c7711140c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,7 +92,6 @@ members = [ "util/launcher", "ckb-bin" ] -exclude = ["devtools/text-optimizer"] [workspace.dependencies] tempfile = "3" diff --git a/devtools/text-optimizer/Cargo.toml b/devtools/text-optimizer/Cargo.toml index 9645c767cf..e56496b2ff 100644 --- a/devtools/text-optimizer/Cargo.toml +++ b/devtools/text-optimizer/Cargo.toml @@ -14,3 +14,7 @@ proc-macro2 = { version = "1", features = ["span-locations"] } serde = "1.0" serde_yaml = "0.9" syn = { version = "2", features = ["visit", "full"] } + +# Prevent this from interfering with workspaces +[workspace] +members = ["."] \ No newline at end of file From 69b11ad31cd80f102b1697d396900757fe97b13b Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Wed, 11 Oct 2023 00:09:40 +0800 Subject: [PATCH 09/22] add file_path for extractor. --- devtools/text-optimizer/src/extractors/mod.rs | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/devtools/text-optimizer/src/extractors/mod.rs b/devtools/text-optimizer/src/extractors/mod.rs index bb4c5677d9..e5d4019302 100644 --- a/devtools/text-optimizer/src/extractors/mod.rs +++ b/devtools/text-optimizer/src/extractors/mod.rs @@ -38,9 +38,14 @@ macro_rules! define_extractor { self.list.push(text_info) } - pub fn get_text_list(&self) -> &[TextInfo] { + pub fn text_list(&self) -> &[TextInfo] { &self.list } + + #[allow(dead_code)] + pub fn file_path(&self) -> &PathBuf { + &self.file_path + } } }; } @@ -152,43 +157,35 @@ fn save_as_file( println!(); - save_yaml( - &output_dir.join(LOG_TEXT_FILE), - log_extractor.get_text_list(), - ) - .expect("save yaml"); + save_yaml(&output_dir.join(LOG_TEXT_FILE), log_extractor.text_list()).expect("save yaml"); println!( "Extract LOG category text: {:?}", - log_extractor.get_text_list().len() + log_extractor.text_list().len() ); - save_yaml( - &output_dir.join(CLAP_TEXT_FILE), - clap_extractor.get_text_list(), - ) - .expect("save yaml"); + save_yaml(&output_dir.join(CLAP_TEXT_FILE), clap_extractor.text_list()).expect("save yaml"); println!( "Extract CLAP category text: {:?}", - clap_extractor.get_text_list().len() + clap_extractor.text_list().len() ); save_yaml( &output_dir.join(STD_OUTPUT_TEXT_FILE), - std_output_extractor.get_text_list(), + std_output_extractor.text_list(), ) .expect("save yaml"); println!( "Extract STD OUPUT category text: {:?}", - std_output_extractor.get_text_list().len() + std_output_extractor.text_list().len() ); save_yaml( &output_dir.join(THISERROR_TEXT_FILE), - thiserror_extractor.get_text_list(), + thiserror_extractor.text_list(), ) .expect("save yaml"); println!( "Extract THISERROR category text: {:?}", - thiserror_extractor.get_text_list().len() + thiserror_extractor.text_list().len() ); } From b6a7de86268d6da741230d3f3dd1337fc6980b2a Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Thu, 12 Oct 2023 11:42:25 +0800 Subject: [PATCH 10/22] start to impl backfill --- devtools/text-optimizer/src/backfill.rs | 46 ++++++------------------- devtools/text-optimizer/src/types.rs | 3 ++ 2 files changed, 14 insertions(+), 35 deletions(-) diff --git a/devtools/text-optimizer/src/backfill.rs b/devtools/text-optimizer/src/backfill.rs index 20765b5386..dc59d4e4b5 100644 --- a/devtools/text-optimizer/src/backfill.rs +++ b/devtools/text-optimizer/src/backfill.rs @@ -1,42 +1,18 @@ use crate::yaml_processor::load_yaml; -use crate::LOG_TEXT_FILE; +use std::fs::read_dir; use std::path::PathBuf; -use std::{ - fs::File, - io::{BufRead, BufReader}, -}; pub fn backfill(input_dir: &PathBuf) { - let log_text_file = input_dir.to_owned().join(LOG_TEXT_FILE); - let log_text_list = load_yaml(&log_text_file).expect("load yaml"); - log::debug!("{:#?}", log_text_list); - - for log_text in log_text_list { - let file_path = log_text.metadata().file(); - let code_line = log_text.metadata().start_line(); - - let file = File::open(file_path).expect("open file"); - let reader = BufReader::new(file); - let lines = reader - .lines() - .map(|line| line.unwrap()) - .collect::>(); - - let _start_line = code_line.saturating_sub(3); // Ensure start_line is at least 0 - let _end_line = (code_line + 3).min(lines.len()); - - // for i in start_line..end_line { - // if lines[i].contains(&log_text.original()) { - // // Replace the line containing the original text - // lines[i] = lines[i].replace(&log_info.original, &log_info.editable); - // } - // } - - // let updated_content = lines.join("\n"); - - // // Write the updated content back to the file - // let mut file = File::create(file_path)?; - // file.write_all(updated_content.as_bytes())?; + if let Ok(entries) = read_dir(input_dir) { + for entry in entries.flatten() { + let entry_path = entry.path(); + if let Some(file_name) = entry.file_name().to_str() { + if file_name.ends_with(".yml") { + log::trace!("{:#?}", entry_path); + let _log_text_list = load_yaml(&entry_path).expect("load yaml"); + } + } + } } } diff --git a/devtools/text-optimizer/src/types.rs b/devtools/text-optimizer/src/types.rs index 7bbf3bedfb..74b8919c24 100644 --- a/devtools/text-optimizer/src/types.rs +++ b/devtools/text-optimizer/src/types.rs @@ -28,6 +28,7 @@ impl TextInfo { &self.editable } + #[allow(dead_code)] pub fn metadata(&self) -> &Meta { &self.metadata } @@ -56,10 +57,12 @@ impl Meta { &self.category } + #[allow(dead_code)] pub fn file(&self) -> &PathBuf { &self.file } + #[allow(dead_code)] pub fn start_line(&self) -> usize { self.start_line } From fc901ab8aad9de0d5634a6fc1e627e429c7e5256 Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Thu, 12 Oct 2023 20:10:50 +0800 Subject: [PATCH 11/22] impl backfill_by_text_info func. --- devtools/text-optimizer/src/backfill.rs | 24 ++++++++++++++++++- devtools/text-optimizer/src/extractors/mod.rs | 10 ++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/devtools/text-optimizer/src/backfill.rs b/devtools/text-optimizer/src/backfill.rs index dc59d4e4b5..66bd021606 100644 --- a/devtools/text-optimizer/src/backfill.rs +++ b/devtools/text-optimizer/src/backfill.rs @@ -1,6 +1,8 @@ use crate::yaml_processor::load_yaml; use std::fs::read_dir; +use std::fs::File; +use std::io::{Read, Write}; use std::path::PathBuf; pub fn backfill(input_dir: &PathBuf) { @@ -9,10 +11,30 @@ pub fn backfill(input_dir: &PathBuf) { let entry_path = entry.path(); if let Some(file_name) = entry.file_name().to_str() { if file_name.ends_with(".yml") { + backfill_by_text_info(&entry_path); log::trace!("{:#?}", entry_path); - let _log_text_list = load_yaml(&entry_path).expect("load yaml"); } } } } } + +fn backfill_by_text_info(file_path: &PathBuf) { + let log_text_list = load_yaml(&file_path).expect("load yaml"); + for text_info in log_text_list { + let mut source_code = String::new(); + let mut file = File::open(text_info.metadata().file()).expect("Failed to open file"); + file.read_to_string(&mut source_code) + .expect("Failed to read file"); + + // Replace the match with the new string + let new_source_code = source_code.replace(text_info.original(), text_info.editable()); + + // Reopen the file in write mode and write the new source code + let mut new_file = + File::create(&text_info.metadata().file()).expect("Failed to create file"); + new_file + .write_all(new_source_code.as_bytes()) + .expect("Failed to write file"); + } +} diff --git a/devtools/text-optimizer/src/extractors/mod.rs b/devtools/text-optimizer/src/extractors/mod.rs index e5d4019302..333138487a 100644 --- a/devtools/text-optimizer/src/extractors/mod.rs +++ b/devtools/text-optimizer/src/extractors/mod.rs @@ -13,6 +13,7 @@ use cargo_metadata::MetadataCommand; use syn::visit::visit_file; use std::{ + collections::HashSet, fs, path::{Path, PathBuf}, }; @@ -189,3 +190,12 @@ fn save_as_file( thiserror_extractor.text_list().len() ); } + +fn _check_dup(text_list: &[TextInfo]) { + let mut set = HashSet::new(); + for text_info in text_list { + if !set.insert(text_info.original()) { + log::warn!("dup: {:?}", text_info.original()) + } + } +} From 0a6e581540f3a48f54d5a4dc267c32922a8b4ef3 Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Thu, 12 Oct 2023 22:30:35 +0800 Subject: [PATCH 12/22] backfill: check text info list. --- devtools/text-optimizer/src/backfill.rs | 92 ++++++++++++++----- devtools/text-optimizer/src/error.rs | 18 ++++ devtools/text-optimizer/src/main.rs | 1 + devtools/text-optimizer/src/yaml_processor.rs | 20 +--- 4 files changed, 88 insertions(+), 43 deletions(-) create mode 100644 devtools/text-optimizer/src/error.rs diff --git a/devtools/text-optimizer/src/backfill.rs b/devtools/text-optimizer/src/backfill.rs index 66bd021606..6a4e169138 100644 --- a/devtools/text-optimizer/src/backfill.rs +++ b/devtools/text-optimizer/src/backfill.rs @@ -1,3 +1,5 @@ +use crate::error::MyError; +use crate::types::TextInfo; use crate::yaml_processor::load_yaml; use std::fs::read_dir; @@ -6,35 +8,77 @@ use std::io::{Read, Write}; use std::path::PathBuf; pub fn backfill(input_dir: &PathBuf) { - if let Ok(entries) = read_dir(input_dir) { - for entry in entries.flatten() { - let entry_path = entry.path(); - if let Some(file_name) = entry.file_name().to_str() { - if file_name.ends_with(".yml") { - backfill_by_text_info(&entry_path); - log::trace!("{:#?}", entry_path); + if let Ok(text_info_lists) = load_all_text_info_files(input_dir) { + backfill_by_text_info(text_info_lists) + } else { + log::error!("Backfill failed to start, please fix text info file first."); + } +} + +fn load_all_text_info_files(input_dir: &PathBuf) -> Result)>, MyError> { + let mut text_info_lists: Vec<(String, Vec)> = vec![]; + let mut all_pass = true; + + let entries = read_dir(input_dir).expect("Read text info file failed."); + for entry in entries.flatten() { + if let Some(file_name) = entry.file_name().to_str() { + if file_name.ends_with(".yml") { + let entry_path = entry.path(); + let list = load_yaml(&entry_path).expect("load yaml"); + + // check text info list + for text_info in &list { + let original_lines = + text_info.metadata().end_line() - text_info.metadata().start_line() + 1; + let editable_lines = text_info.editable().lines().count(); + if original_lines != editable_lines { + log::error!("TextInfoFormatError: \n\ + text info file: {}\n\ + original: {}\n\ + editable: {}\n\ + file: {:?}\n\ + In cases where the number of lines in the original text and the edited text must remain consistent, \n\ + if inconsistency arises, it is recommended to directly modify the source code and submit a pull request.\n", + file_name, + text_info.original(), + text_info.editable(), + text_info.metadata().file() + ); + all_pass = false; + } } + + text_info_lists.push((file_name.to_owned(), list)) } } } + + if all_pass { + Ok(text_info_lists) + } else { + Err(MyError::TextInfoFormat) + } } -fn backfill_by_text_info(file_path: &PathBuf) { - let log_text_list = load_yaml(&file_path).expect("load yaml"); - for text_info in log_text_list { - let mut source_code = String::new(); - let mut file = File::open(text_info.metadata().file()).expect("Failed to open file"); - file.read_to_string(&mut source_code) - .expect("Failed to read file"); - - // Replace the match with the new string - let new_source_code = source_code.replace(text_info.original(), text_info.editable()); - - // Reopen the file in write mode and write the new source code - let mut new_file = - File::create(&text_info.metadata().file()).expect("Failed to create file"); - new_file - .write_all(new_source_code.as_bytes()) - .expect("Failed to write file"); +fn backfill_by_text_info(text_info_lists: Vec<(String, Vec)>) { + for list in text_info_lists { + log::info!("Parse text info file: {:?}", list.0); + for text_info in list.1 { + let mut source_code = String::new(); + let mut file = File::open(text_info.metadata().file()).expect("Failed to open file"); + file.read_to_string(&mut source_code) + .expect("Failed to read file"); + + // Replace the match with the new string + let new_source_code = source_code.replace(text_info.original(), text_info.editable()); + + // Reopen the file in write mode and write the new source code + let mut new_file = + File::create(text_info.metadata().file()).expect("Failed to create file"); + new_file + .write_all(new_source_code.as_bytes()) + .expect("Failed to write file"); + } } + log::info!("The backfill is completed, please review the modifications in the source code."); } diff --git a/devtools/text-optimizer/src/error.rs b/devtools/text-optimizer/src/error.rs new file mode 100644 index 0000000000..04dcfa0368 --- /dev/null +++ b/devtools/text-optimizer/src/error.rs @@ -0,0 +1,18 @@ +#[derive(Debug)] +pub enum MyError { + Io(std::io::Error), + Serde(serde_yaml::Error), + TextInfoFormat, +} + +impl From for MyError { + fn from(error: std::io::Error) -> Self { + MyError::Io(error) + } +} + +impl From for MyError { + fn from(error: serde_yaml::Error) -> Self { + MyError::Serde(error) + } +} diff --git a/devtools/text-optimizer/src/main.rs b/devtools/text-optimizer/src/main.rs index 4f1fad8c85..ef1a925a2c 100644 --- a/devtools/text-optimizer/src/main.rs +++ b/devtools/text-optimizer/src/main.rs @@ -1,4 +1,5 @@ mod backfill; +mod error; mod extractors; mod types; mod yaml_processor; diff --git a/devtools/text-optimizer/src/yaml_processor.rs b/devtools/text-optimizer/src/yaml_processor.rs index ebb6a4ed58..097cff99c3 100644 --- a/devtools/text-optimizer/src/yaml_processor.rs +++ b/devtools/text-optimizer/src/yaml_processor.rs @@ -1,26 +1,8 @@ -use super::types::TextInfo; +use super::{error::MyError, types::TextInfo}; use std::io::Read; use std::io::Write; use std::{fs::File, path::PathBuf}; -#[derive(Debug)] -pub enum MyError { - IoError(std::io::Error), - SerdeError(serde_yaml::Error), -} - -impl From for MyError { - fn from(error: std::io::Error) -> Self { - MyError::IoError(error) - } -} - -impl From for MyError { - fn from(error: serde_yaml::Error) -> Self { - MyError::SerdeError(error) - } -} - pub fn save_yaml(file: &PathBuf, data: &[TextInfo]) -> Result<(), MyError> { let mut file = File::create(file)?; file.write_fmt(format_args!( From 0065a2c9d861eb3d52dd00c5caa227e2cefd89cf Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Thu, 12 Oct 2023 23:48:08 +0800 Subject: [PATCH 13/22] Add a rule when extracting text info: Text from the same file should not be duplicated. In case of duplication, different line numbers will be recorded --- devtools/text-optimizer/src/backfill.rs | 38 ++----------------- devtools/text-optimizer/src/error.rs | 1 - .../src/extractors/clap_extractor.rs | 4 +- .../src/extractors/log_extractor.rs | 4 +- devtools/text-optimizer/src/extractors/mod.rs | 38 ++++++++++--------- .../src/extractors/std_output_extractor.rs | 4 +- .../src/extractors/thiserror_extractor.rs | 9 +---- devtools/text-optimizer/src/types.rs | 28 +++++++------- 8 files changed, 44 insertions(+), 82 deletions(-) diff --git a/devtools/text-optimizer/src/backfill.rs b/devtools/text-optimizer/src/backfill.rs index 6a4e169138..7e68d65463 100644 --- a/devtools/text-optimizer/src/backfill.rs +++ b/devtools/text-optimizer/src/backfill.rs @@ -9,7 +9,7 @@ use std::path::PathBuf; pub fn backfill(input_dir: &PathBuf) { if let Ok(text_info_lists) = load_all_text_info_files(input_dir) { - backfill_by_text_info(text_info_lists) + backfill_edited(text_info_lists) } else { log::error!("Backfill failed to start, please fix text info file first."); } @@ -17,50 +17,20 @@ pub fn backfill(input_dir: &PathBuf) { fn load_all_text_info_files(input_dir: &PathBuf) -> Result)>, MyError> { let mut text_info_lists: Vec<(String, Vec)> = vec![]; - let mut all_pass = true; - let entries = read_dir(input_dir).expect("Read text info file failed."); for entry in entries.flatten() { if let Some(file_name) = entry.file_name().to_str() { if file_name.ends_with(".yml") { let entry_path = entry.path(); let list = load_yaml(&entry_path).expect("load yaml"); - - // check text info list - for text_info in &list { - let original_lines = - text_info.metadata().end_line() - text_info.metadata().start_line() + 1; - let editable_lines = text_info.editable().lines().count(); - if original_lines != editable_lines { - log::error!("TextInfoFormatError: \n\ - text info file: {}\n\ - original: {}\n\ - editable: {}\n\ - file: {:?}\n\ - In cases where the number of lines in the original text and the edited text must remain consistent, \n\ - if inconsistency arises, it is recommended to directly modify the source code and submit a pull request.\n", - file_name, - text_info.original(), - text_info.editable(), - text_info.metadata().file() - ); - all_pass = false; - } - } - text_info_lists.push((file_name.to_owned(), list)) } } } - - if all_pass { - Ok(text_info_lists) - } else { - Err(MyError::TextInfoFormat) - } + Ok(text_info_lists) } -fn backfill_by_text_info(text_info_lists: Vec<(String, Vec)>) { +fn backfill_edited(text_info_lists: Vec<(String, Vec)>) { for list in text_info_lists { log::info!("Parse text info file: {:?}", list.0); for text_info in list.1 { @@ -69,7 +39,7 @@ fn backfill_by_text_info(text_info_lists: Vec<(String, Vec)>) { file.read_to_string(&mut source_code) .expect("Failed to read file"); - // Replace the match with the new string + // Replace the match origianl with the edited let new_source_code = source_code.replace(text_info.original(), text_info.editable()); // Reopen the file in write mode and write the new source code diff --git a/devtools/text-optimizer/src/error.rs b/devtools/text-optimizer/src/error.rs index 04dcfa0368..e3dd29f043 100644 --- a/devtools/text-optimizer/src/error.rs +++ b/devtools/text-optimizer/src/error.rs @@ -2,7 +2,6 @@ pub enum MyError { Io(std::io::Error), Serde(serde_yaml::Error), - TextInfoFormat, } impl From for MyError { diff --git a/devtools/text-optimizer/src/extractors/clap_extractor.rs b/devtools/text-optimizer/src/extractors/clap_extractor.rs index 9f76c839db..bebd234d55 100644 --- a/devtools/text-optimizer/src/extractors/clap_extractor.rs +++ b/devtools/text-optimizer/src/extractors/clap_extractor.rs @@ -17,11 +17,9 @@ impl syn::visit::Visit<'_> for ClapExtractor { let span = lit_str.span(); let start_line = span.start().line; - let end_line = span.end().line; let category = Category::from_str(method_ident.to_string().as_str()).unwrap(); - let meta = - Meta::new(category, self.file_path.to_owned(), start_line, end_line); + let meta = Meta::new(category, self.file_path.to_owned(), start_line); self.add_text_info(TextInfo::new(text, meta)); } } diff --git a/devtools/text-optimizer/src/extractors/log_extractor.rs b/devtools/text-optimizer/src/extractors/log_extractor.rs index 0c2e20ce09..5b0d141b52 100644 --- a/devtools/text-optimizer/src/extractors/log_extractor.rs +++ b/devtools/text-optimizer/src/extractors/log_extractor.rs @@ -18,10 +18,8 @@ impl syn::visit::Visit<'_> for LogExtractor { let span = lit.span(); let start_line = span.start().line; - let end_line = span.end().line; let category = Category::from_str(ident.to_string().as_str()).unwrap(); - let meta = - Meta::new(category, self.file_path.to_owned(), start_line, end_line); + let meta = Meta::new(category, self.file_path.to_owned(), start_line); self.add_text_info(TextInfo::new(text, meta)); } } diff --git a/devtools/text-optimizer/src/extractors/mod.rs b/devtools/text-optimizer/src/extractors/mod.rs index 333138487a..0f8fa263cb 100644 --- a/devtools/text-optimizer/src/extractors/mod.rs +++ b/devtools/text-optimizer/src/extractors/mod.rs @@ -13,7 +13,7 @@ use cargo_metadata::MetadataCommand; use syn::visit::visit_file; use std::{ - collections::HashSet, + collections::HashMap, fs, path::{Path, PathBuf}, }; @@ -22,7 +22,7 @@ macro_rules! define_extractor { ($struct_name:ident) => { #[derive(Default)] pub struct $struct_name { - list: Vec, + map: HashMap<(String, PathBuf), TextInfo>, file_path: PathBuf, } @@ -36,11 +36,18 @@ macro_rules! define_extractor { } pub fn add_text_info(&mut self, text_info: TextInfo) { - self.list.push(text_info) + let key = text_info.original().to_owned(); + let file = text_info.metadata().file().clone(); + + if let Some(existing) = self.map.get_mut(&(key.to_owned(), file.clone())) { + existing.append_new_line(text_info.metadata().start_lines()[0]); + } else { + self.map.insert((key, file), text_info); + } } - pub fn text_list(&self) -> &[TextInfo] { - &self.list + pub fn text_list(&self) -> Vec { + self.map.values().cloned().collect() } #[allow(dead_code)] @@ -158,13 +165,17 @@ fn save_as_file( println!(); - save_yaml(&output_dir.join(LOG_TEXT_FILE), log_extractor.text_list()).expect("save yaml"); + save_yaml(&output_dir.join(LOG_TEXT_FILE), &log_extractor.text_list()).expect("save yaml"); println!( "Extract LOG category text: {:?}", log_extractor.text_list().len() ); - save_yaml(&output_dir.join(CLAP_TEXT_FILE), clap_extractor.text_list()).expect("save yaml"); + save_yaml( + &output_dir.join(CLAP_TEXT_FILE), + &clap_extractor.text_list(), + ) + .expect("save yaml"); println!( "Extract CLAP category text: {:?}", clap_extractor.text_list().len() @@ -172,7 +183,7 @@ fn save_as_file( save_yaml( &output_dir.join(STD_OUTPUT_TEXT_FILE), - std_output_extractor.text_list(), + &std_output_extractor.text_list(), ) .expect("save yaml"); println!( @@ -182,7 +193,7 @@ fn save_as_file( save_yaml( &output_dir.join(THISERROR_TEXT_FILE), - thiserror_extractor.text_list(), + &thiserror_extractor.text_list(), ) .expect("save yaml"); println!( @@ -190,12 +201,3 @@ fn save_as_file( thiserror_extractor.text_list().len() ); } - -fn _check_dup(text_list: &[TextInfo]) { - let mut set = HashSet::new(); - for text_info in text_list { - if !set.insert(text_info.original()) { - log::warn!("dup: {:?}", text_info.original()) - } - } -} diff --git a/devtools/text-optimizer/src/extractors/std_output_extractor.rs b/devtools/text-optimizer/src/extractors/std_output_extractor.rs index 321f4581bd..39e5821f7b 100644 --- a/devtools/text-optimizer/src/extractors/std_output_extractor.rs +++ b/devtools/text-optimizer/src/extractors/std_output_extractor.rs @@ -13,10 +13,8 @@ impl syn::visit::Visit<'_> for StdOutputExtractor { let span = lit.span(); let start_line = span.start().line; - let end_line = span.end().line; let category = Category::from_str(ident.to_string().as_str()).unwrap(); - let meta = - Meta::new(category, self.file_path.to_owned(), start_line, end_line); + let meta = Meta::new(category, self.file_path.to_owned(), start_line); self.add_text_info(TextInfo::new(text, meta)); } } diff --git a/devtools/text-optimizer/src/extractors/thiserror_extractor.rs b/devtools/text-optimizer/src/extractors/thiserror_extractor.rs index a33927bb10..e9da4a67e5 100644 --- a/devtools/text-optimizer/src/extractors/thiserror_extractor.rs +++ b/devtools/text-optimizer/src/extractors/thiserror_extractor.rs @@ -29,13 +29,8 @@ impl syn::visit::Visit<'_> for ThiserrorExtractor { let span = lit_str.span(); let start_line = span.start().line; - let end_line = span.end().line; - let meta = Meta::new( - Category::ThisError, - self.file_path.to_owned(), - start_line, - end_line, - ); + let meta = + Meta::new(Category::ThisError, self.file_path.to_owned(), start_line); self.add_text_info(TextInfo::new(text, meta)); } } diff --git a/devtools/text-optimizer/src/types.rs b/devtools/text-optimizer/src/types.rs index 74b8919c24..b6237abbde 100644 --- a/devtools/text-optimizer/src/types.rs +++ b/devtools/text-optimizer/src/types.rs @@ -1,8 +1,9 @@ use serde::{Deserialize, Serialize}; use std::path::PathBuf; use std::str::FromStr; +use std::vec; -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Hash, Clone)] pub struct TextInfo { original: String, editable: String, @@ -32,23 +33,25 @@ impl TextInfo { pub fn metadata(&self) -> &Meta { &self.metadata } + + pub fn append_new_line(&mut self, new: usize) { + self.metadata.append_new_line(new) + } } -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Hash, Clone)] pub struct Meta { category: Category, file: PathBuf, - start_line: usize, - end_line: usize, + code_lines: Vec, } impl Meta { - pub fn new(category: Category, file: PathBuf, start_line: usize, end_line: usize) -> Self { + pub fn new(category: Category, file: PathBuf, start_line: usize) -> Self { Meta { category, file, - start_line, - end_line, + code_lines: vec![start_line], } } @@ -63,17 +66,16 @@ impl Meta { } #[allow(dead_code)] - pub fn start_line(&self) -> usize { - self.start_line + pub fn start_lines(&self) -> &[usize] { + &self.code_lines } - #[allow(dead_code)] - pub fn end_line(&self) -> usize { - self.end_line + pub fn append_new_line(&mut self, new: usize) { + self.code_lines.push(new) } } -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Hash, Clone)] pub enum Category { ClapHelp, ClapAbout, From bdcd1645cb78d055e750cbfda3be4589340fb358 Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Sat, 14 Oct 2023 20:48:20 +0800 Subject: [PATCH 14/22] Add get_macro_name to recognize logs like ckb_logger::info --- .../src/extractors/log_extractor.rs | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/devtools/text-optimizer/src/extractors/log_extractor.rs b/devtools/text-optimizer/src/extractors/log_extractor.rs index 5b0d141b52..d3f5fe15f0 100644 --- a/devtools/text-optimizer/src/extractors/log_extractor.rs +++ b/devtools/text-optimizer/src/extractors/log_extractor.rs @@ -5,12 +5,12 @@ use syn::Macro; impl syn::visit::Visit<'_> for LogExtractor { fn visit_macro(&mut self, node: &Macro) { - if let Some(ident) = node.path.get_ident() { - if ident == "error" - || ident == "warn" - || ident == "info" - || ident == "debug" - || ident == "trace" + if let Some(name) = get_macro_name(node) { + if name == "error" + || name == "warn" + || name == "info" + || name == "debug" + || name == "trace" { if let Some(lit) = node.tokens.clone().into_iter().next() { if let Some(text) = extract_contents_in_brackets(lit.to_string()) { @@ -18,7 +18,7 @@ impl syn::visit::Visit<'_> for LogExtractor { let span = lit.span(); let start_line = span.start().line; - let category = Category::from_str(ident.to_string().as_str()).unwrap(); + let category = Category::from_str(&name).unwrap(); let meta = Meta::new(category, self.file_path.to_owned(), start_line); self.add_text_info(TextInfo::new(text, meta)); } @@ -27,3 +27,14 @@ impl syn::visit::Visit<'_> for LogExtractor { } } } + +fn get_macro_name(node: &Macro) -> Option { + if let Some(ident) = node.path.get_ident() { + Some(ident.to_string()) + } else { + node.path + .segments + .last() + .map(|segment| segment.ident.to_string()) + } +} From 790d81a517e6e12dbc6649cc4a2928f2d8005d64 Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Sat, 14 Oct 2023 23:08:24 +0800 Subject: [PATCH 15/22] refactoring: add and impl trait Extractor. --- .../src/extractors/clap_extractor.rs | 58 +++++- .../src/extractors/log_extractor.rs | 58 +++++- devtools/text-optimizer/src/extractors/mod.rs | 185 +++++------------- .../src/extractors/std_output_extractor.rs | 58 +++++- .../src/extractors/thiserror_extractor.rs | 64 +++++- 5 files changed, 277 insertions(+), 146 deletions(-) diff --git a/devtools/text-optimizer/src/extractors/clap_extractor.rs b/devtools/text-optimizer/src/extractors/clap_extractor.rs index bebd234d55..3ba1a34e62 100644 --- a/devtools/text-optimizer/src/extractors/clap_extractor.rs +++ b/devtools/text-optimizer/src/extractors/clap_extractor.rs @@ -1,8 +1,61 @@ -use super::{extract_contents_in_brackets, ClapExtractor}; +use super::{extract_contents_in_brackets, Extractor}; use crate::types::{Category, Meta, TextInfo}; + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; use std::str::FromStr; use syn::Expr::{self}; use syn::Lit::Str; +use syn::{visit::visit_file, File}; + +#[derive(Default)] +pub struct ClapExtractor { + save_file: PathBuf, + map: HashMap<(String, PathBuf), TextInfo>, + scanning_file_path: PathBuf, +} + +impl ClapExtractor { + pub fn new(save_file: PathBuf) -> Self { + ClapExtractor { + save_file, + ..Default::default() + } + } +} + +impl Extractor for ClapExtractor { + fn reset_scanning_path(&mut self, file_path: &Path) { + self.scanning_file_path = file_path.to_owned(); + } + + fn add_text_info(&mut self, text_info: TextInfo) { + let key = text_info.original().to_owned(); + let file = text_info.metadata().file().clone(); + + if let Some(existing) = self.map.get_mut(&(key.to_owned(), file.clone())) { + existing.append_new_line(text_info.metadata().start_lines()[0]); + } else { + self.map.insert((key, file), text_info); + } + } + + fn text_list(&self) -> Vec { + self.map.values().cloned().collect() + } + + fn scanning_file_path(&self) -> &PathBuf { + &self.scanning_file_path + } + + fn save_file_path(&self) -> &PathBuf { + &self.save_file + } + + fn visit_file(&mut self, node: &File) { + visit_file(self, node) + } +} impl syn::visit::Visit<'_> for ClapExtractor { fn visit_expr_method_call(&mut self, expr: &syn::ExprMethodCall) { @@ -19,7 +72,8 @@ impl syn::visit::Visit<'_> for ClapExtractor { let start_line = span.start().line; let category = Category::from_str(method_ident.to_string().as_str()).unwrap(); - let meta = Meta::new(category, self.file_path.to_owned(), start_line); + let meta = + Meta::new(category, self.scanning_file_path.to_owned(), start_line); self.add_text_info(TextInfo::new(text, meta)); } } diff --git a/devtools/text-optimizer/src/extractors/log_extractor.rs b/devtools/text-optimizer/src/extractors/log_extractor.rs index d3f5fe15f0..90ec975349 100644 --- a/devtools/text-optimizer/src/extractors/log_extractor.rs +++ b/devtools/text-optimizer/src/extractors/log_extractor.rs @@ -1,7 +1,60 @@ -use super::{extract_contents_in_brackets, LogExtractor}; +use super::{extract_contents_in_brackets, Extractor}; use crate::types::{Category, Meta, TextInfo}; + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; use std::str::FromStr; use syn::Macro; +use syn::{visit::visit_file, File}; + +#[derive(Default)] +pub struct LogExtractor { + save_file: PathBuf, + map: HashMap<(String, PathBuf), TextInfo>, + scanning_file_path: PathBuf, +} + +impl LogExtractor { + pub fn new(save_file: PathBuf) -> Self { + LogExtractor { + save_file, + ..Default::default() + } + } +} + +impl Extractor for LogExtractor { + fn reset_scanning_path(&mut self, file_path: &Path) { + self.scanning_file_path = file_path.to_owned(); + } + + fn add_text_info(&mut self, text_info: TextInfo) { + let key = text_info.original().to_owned(); + let file = text_info.metadata().file().clone(); + + if let Some(existing) = self.map.get_mut(&(key.to_owned(), file.clone())) { + existing.append_new_line(text_info.metadata().start_lines()[0]); + } else { + self.map.insert((key, file), text_info); + } + } + + fn text_list(&self) -> Vec { + self.map.values().cloned().collect() + } + + fn scanning_file_path(&self) -> &PathBuf { + &self.scanning_file_path + } + + fn save_file_path(&self) -> &PathBuf { + &self.save_file + } + + fn visit_file(&mut self, node: &File) { + visit_file(self, node) + } +} impl syn::visit::Visit<'_> for LogExtractor { fn visit_macro(&mut self, node: &Macro) { @@ -19,7 +72,8 @@ impl syn::visit::Visit<'_> for LogExtractor { let span = lit.span(); let start_line = span.start().line; let category = Category::from_str(&name).unwrap(); - let meta = Meta::new(category, self.file_path.to_owned(), start_line); + let meta = + Meta::new(category, self.scanning_file_path.to_owned(), start_line); self.add_text_info(TextInfo::new(text, meta)); } } diff --git a/devtools/text-optimizer/src/extractors/mod.rs b/devtools/text-optimizer/src/extractors/mod.rs index 0f8fa263cb..9fc561ec0d 100644 --- a/devtools/text-optimizer/src/extractors/mod.rs +++ b/devtools/text-optimizer/src/extractors/mod.rs @@ -3,72 +3,49 @@ pub mod log_extractor; pub mod std_output_extractor; pub mod thiserror_extractor; -use crate::types::TextInfo; use crate::{ - yaml_processor::save_yaml, CLAP_TEXT_FILE, LOG_TEXT_FILE, STD_OUTPUT_TEXT_FILE, - THISERROR_TEXT_FILE, + types::TextInfo, yaml_processor::save_yaml, CLAP_TEXT_FILE, LOG_TEXT_FILE, + STD_OUTPUT_TEXT_FILE, THISERROR_TEXT_FILE, }; +use clap_extractor::ClapExtractor; +use log_extractor::LogExtractor; +use std_output_extractor::StdOutputExtractor; +use thiserror_extractor::ThiserrorExtractor; + use cargo_metadata::MetadataCommand; -use syn::visit::visit_file; +use syn::File; use std::{ - collections::HashMap, fs, path::{Path, PathBuf}, }; -macro_rules! define_extractor { - ($struct_name:ident) => { - #[derive(Default)] - pub struct $struct_name { - map: HashMap<(String, PathBuf), TextInfo>, - file_path: PathBuf, - } - - impl $struct_name { - pub fn new() -> Self { - $struct_name::default() - } - - pub fn reset_analysis_path(&mut self, file_path: &PathBuf) { - self.file_path = file_path.to_owned(); - } - - pub fn add_text_info(&mut self, text_info: TextInfo) { - let key = text_info.original().to_owned(); - let file = text_info.metadata().file().clone(); - - if let Some(existing) = self.map.get_mut(&(key.to_owned(), file.clone())) { - existing.append_new_line(text_info.metadata().start_lines()[0]); - } else { - self.map.insert((key, file), text_info); - } - } - - pub fn text_list(&self) -> Vec { - self.map.values().cloned().collect() - } - - #[allow(dead_code)] - pub fn file_path(&self) -> &PathBuf { - &self.file_path - } - } - }; +pub trait Extractor { + fn reset_scanning_path(&mut self, file_path: &Path); + fn add_text_info(&mut self, text_info: TextInfo); + fn text_list(&self) -> Vec; + fn scanning_file_path(&self) -> &PathBuf; + fn save_file_path(&self) -> &PathBuf; + fn save_as_file(&self) { + save_yaml(self.save_file_path(), &self.text_list()).expect("save yaml"); + } + fn visit_file(&mut self, node: &File); } -define_extractor!(ClapExtractor); -define_extractor!(LogExtractor); -define_extractor!(StdOutputExtractor); -define_extractor!(ThiserrorExtractor); - pub fn extract(project_root: PathBuf, output_dir: &PathBuf) { // extractors - let mut clap_extractor = ClapExtractor::new(); - let mut log_extractor = LogExtractor::new(); - let mut std_output_extractor = StdOutputExtractor::new(); - let mut thiserror_extractor = ThiserrorExtractor::new(); + let mut clap_extractor = ClapExtractor::new(output_dir.join(CLAP_TEXT_FILE)); + let mut log_extractor = LogExtractor::new(output_dir.join(LOG_TEXT_FILE)); + let mut std_output_extractor = StdOutputExtractor::new(output_dir.join(STD_OUTPUT_TEXT_FILE)); + let mut thiserror_extractor = ThiserrorExtractor::new(output_dir.join(THISERROR_TEXT_FILE)); + + let mut extractors: Vec<&mut dyn Extractor> = vec![ + &mut clap_extractor, + &mut log_extractor, + &mut std_output_extractor, + &mut thiserror_extractor, + ]; let project_metadata = MetadataCommand::new() .manifest_path(project_root) @@ -82,61 +59,33 @@ pub fn extract(project_root: PathBuf, output_dir: &PathBuf) { .parent() .expect("workspace member crate path") .join("src"); - process_rs_files_in_src( - &crate_src_path, - &mut log_extractor, - &mut std_output_extractor, - &mut thiserror_extractor, - &mut clap_extractor, - ); + process_rs_files_in_src(&crate_src_path, &mut extractors); } - save_as_file( - output_dir, - clap_extractor, - log_extractor, - std_output_extractor, - thiserror_extractor, - ); + save_extractors(output_dir, &extractors); } -pub fn process_rs_files_in_src( - dir_path: &Path, - log_extractor: &mut LogExtractor, - std_output_extractor: &mut StdOutputExtractor, - thiserror_extractor: &mut ThiserrorExtractor, - clap_extractor: &mut ClapExtractor, -) { +pub fn process_rs_files_in_src(dir_path: &Path, extractors: &mut [&mut dyn Extractor]) { if let Ok(entries) = fs::read_dir(dir_path) { for entry in entries.flatten() { let entry_path = entry.path(); if entry_path.is_dir() { - process_rs_files_in_src( - &entry_path, - log_extractor, - std_output_extractor, - thiserror_extractor, - clap_extractor, - ); + process_rs_files_in_src(&entry_path, extractors); } else if let Some(file_name) = entry.file_name().to_str() { if file_name.ends_with(".rs") { log::trace!("Found .rs file: {:?}", entry_path); - // reset file path - clap_extractor.reset_analysis_path(&entry_path); - log_extractor.reset_analysis_path(&entry_path); - std_output_extractor.reset_analysis_path(&entry_path); - thiserror_extractor.reset_analysis_path(&entry_path); - let file_content = fs::read_to_string(&entry_path).expect("Failed to read file"); - if let Ok(syntax_tree) = syn::parse_file(&file_content) { - visit_file(clap_extractor, &syntax_tree); - visit_file(log_extractor, &syntax_tree); - visit_file(std_output_extractor, &syntax_tree); - visit_file(thiserror_extractor, &syntax_tree); - } else { - log::error!("Failed to parse .rs file: {:?}", entry_path); + + for extractor in extractors.iter_mut() { + extractor.reset_scanning_path(&entry_path); + + if let Ok(syntax_tree) = syn::parse_file(&file_content) { + extractor.visit_file(&syntax_tree) + } else { + log::error!("Failed to parse .rs file: {:?}", entry_path); + } } } } @@ -154,50 +103,14 @@ pub fn extract_contents_in_brackets(lit: String) -> Option { None } -fn save_as_file( - output_dir: &PathBuf, - clap_extractor: ClapExtractor, - log_extractor: LogExtractor, - std_output_extractor: StdOutputExtractor, - thiserror_extractor: ThiserrorExtractor, -) { +fn save_extractors(output_dir: &PathBuf, extractors: &[&mut dyn Extractor]) { fs::create_dir_all(output_dir).expect("create dir all"); - println!(); - save_yaml(&output_dir.join(LOG_TEXT_FILE), &log_extractor.text_list()).expect("save yaml"); - println!( - "Extract LOG category text: {:?}", - log_extractor.text_list().len() - ); - - save_yaml( - &output_dir.join(CLAP_TEXT_FILE), - &clap_extractor.text_list(), - ) - .expect("save yaml"); - println!( - "Extract CLAP category text: {:?}", - clap_extractor.text_list().len() - ); - - save_yaml( - &output_dir.join(STD_OUTPUT_TEXT_FILE), - &std_output_extractor.text_list(), - ) - .expect("save yaml"); - println!( - "Extract STD OUPUT category text: {:?}", - std_output_extractor.text_list().len() - ); - - save_yaml( - &output_dir.join(THISERROR_TEXT_FILE), - &thiserror_extractor.text_list(), - ) - .expect("save yaml"); - println!( - "Extract THISERROR category text: {:?}", - thiserror_extractor.text_list().len() - ); + for extractor in extractors { + extractor.save_as_file(); + let file_name = extractor.save_file_path().file_name().unwrap(); + let text_len = extractor.text_list().len(); + println!("{:?}: {:?}", file_name, text_len); + } } diff --git a/devtools/text-optimizer/src/extractors/std_output_extractor.rs b/devtools/text-optimizer/src/extractors/std_output_extractor.rs index 39e5821f7b..e436be1d07 100644 --- a/devtools/text-optimizer/src/extractors/std_output_extractor.rs +++ b/devtools/text-optimizer/src/extractors/std_output_extractor.rs @@ -1,7 +1,60 @@ -use super::{extract_contents_in_brackets, StdOutputExtractor}; +use super::{extract_contents_in_brackets, Extractor}; use crate::types::{Category, Meta, TextInfo}; + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; use std::str::FromStr; use syn::Macro; +use syn::{visit::visit_file, File}; + +#[derive(Default)] +pub struct StdOutputExtractor { + save_file: PathBuf, + map: HashMap<(String, PathBuf), TextInfo>, + scanning_file_path: PathBuf, +} + +impl StdOutputExtractor { + pub fn new(save_file: PathBuf) -> Self { + StdOutputExtractor { + save_file, + ..Default::default() + } + } +} + +impl Extractor for StdOutputExtractor { + fn reset_scanning_path(&mut self, file_path: &Path) { + self.scanning_file_path = file_path.to_owned(); + } + + fn add_text_info(&mut self, text_info: TextInfo) { + let key = text_info.original().to_owned(); + let file = text_info.metadata().file().clone(); + + if let Some(existing) = self.map.get_mut(&(key.to_owned(), file.clone())) { + existing.append_new_line(text_info.metadata().start_lines()[0]); + } else { + self.map.insert((key, file), text_info); + } + } + + fn text_list(&self) -> Vec { + self.map.values().cloned().collect() + } + + fn scanning_file_path(&self) -> &PathBuf { + &self.scanning_file_path + } + + fn save_file_path(&self) -> &PathBuf { + &self.save_file + } + + fn visit_file(&mut self, node: &File) { + visit_file(self, node) + } +} impl syn::visit::Visit<'_> for StdOutputExtractor { fn visit_macro(&mut self, node: &Macro) { @@ -14,7 +67,8 @@ impl syn::visit::Visit<'_> for StdOutputExtractor { let span = lit.span(); let start_line = span.start().line; let category = Category::from_str(ident.to_string().as_str()).unwrap(); - let meta = Meta::new(category, self.file_path.to_owned(), start_line); + let meta = + Meta::new(category, self.scanning_file_path.to_owned(), start_line); self.add_text_info(TextInfo::new(text, meta)); } } diff --git a/devtools/text-optimizer/src/extractors/thiserror_extractor.rs b/devtools/text-optimizer/src/extractors/thiserror_extractor.rs index e9da4a67e5..9cb719500c 100644 --- a/devtools/text-optimizer/src/extractors/thiserror_extractor.rs +++ b/devtools/text-optimizer/src/extractors/thiserror_extractor.rs @@ -1,8 +1,61 @@ -use super::{extract_contents_in_brackets, ThiserrorExtractor}; +use super::{extract_contents_in_brackets, Extractor}; use crate::types::{Category, Meta, TextInfo}; + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; use syn::spanned::Spanned; use syn::Expr::{self, Lit}; use syn::Lit::Str; +use syn::{visit::visit_file, File}; + +#[derive(Default)] +pub struct ThiserrorExtractor { + save_file: PathBuf, + map: HashMap<(String, PathBuf), TextInfo>, + scanning_file_path: PathBuf, +} + +impl ThiserrorExtractor { + pub fn new(save_file: PathBuf) -> Self { + ThiserrorExtractor { + save_file, + ..Default::default() + } + } +} + +impl Extractor for ThiserrorExtractor { + fn reset_scanning_path(&mut self, file_path: &Path) { + self.scanning_file_path = file_path.to_owned(); + } + + fn add_text_info(&mut self, text_info: TextInfo) { + let key = text_info.original().to_owned(); + let file = text_info.metadata().file().clone(); + + if let Some(existing) = self.map.get_mut(&(key.to_owned(), file.clone())) { + existing.append_new_line(text_info.metadata().start_lines()[0]); + } else { + self.map.insert((key, file), text_info); + } + } + + fn text_list(&self) -> Vec { + self.map.values().cloned().collect() + } + + fn scanning_file_path(&self) -> &PathBuf { + &self.scanning_file_path + } + + fn save_file_path(&self) -> &PathBuf { + &self.save_file + } + + fn visit_file(&mut self, node: &File) { + visit_file(self, node) + } +} impl syn::visit::Visit<'_> for ThiserrorExtractor { fn visit_attribute(&mut self, attr: &syn::Attribute) { @@ -14,7 +67,7 @@ impl syn::visit::Visit<'_> for ThiserrorExtractor { let span = attr.span(); log::warn!( "parse args failed, ignore the file: {:?}, code line: {:?}", - self.file_path, + self.scanning_file_path, span.start().line ); return; @@ -29,8 +82,11 @@ impl syn::visit::Visit<'_> for ThiserrorExtractor { let span = lit_str.span(); let start_line = span.start().line; - let meta = - Meta::new(Category::ThisError, self.file_path.to_owned(), start_line); + let meta = Meta::new( + Category::ThisError, + self.scanning_file_path.to_owned(), + start_line, + ); self.add_text_info(TextInfo::new(text, meta)); } } From 9d573e241a0ce81fca61bd02aba5a014a7a323e8 Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Sat, 14 Oct 2023 23:20:24 +0800 Subject: [PATCH 16/22] Sort the output of the extractors. --- .../text-optimizer/src/extractors/clap_extractor.rs | 11 ++++++++++- .../text-optimizer/src/extractors/log_extractor.rs | 11 ++++++++++- .../src/extractors/std_output_extractor.rs | 11 ++++++++++- .../src/extractors/thiserror_extractor.rs | 11 ++++++++++- 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/devtools/text-optimizer/src/extractors/clap_extractor.rs b/devtools/text-optimizer/src/extractors/clap_extractor.rs index 3ba1a34e62..992d055b96 100644 --- a/devtools/text-optimizer/src/extractors/clap_extractor.rs +++ b/devtools/text-optimizer/src/extractors/clap_extractor.rs @@ -41,7 +41,16 @@ impl Extractor for ClapExtractor { } fn text_list(&self) -> Vec { - self.map.values().cloned().collect() + let mut text_list: Vec = self.map.values().cloned().collect(); + text_list.sort_by(|a, b| { + let cmp = a.metadata().file().cmp(b.metadata().file()); + if cmp == std::cmp::Ordering::Equal { + a.metadata().start_lines()[0].cmp(&b.metadata().start_lines()[0]) + } else { + cmp + } + }); + text_list } fn scanning_file_path(&self) -> &PathBuf { diff --git a/devtools/text-optimizer/src/extractors/log_extractor.rs b/devtools/text-optimizer/src/extractors/log_extractor.rs index 90ec975349..50ff2bd4b1 100644 --- a/devtools/text-optimizer/src/extractors/log_extractor.rs +++ b/devtools/text-optimizer/src/extractors/log_extractor.rs @@ -40,7 +40,16 @@ impl Extractor for LogExtractor { } fn text_list(&self) -> Vec { - self.map.values().cloned().collect() + let mut text_list: Vec = self.map.values().cloned().collect(); + text_list.sort_by(|a, b| { + let cmp = a.metadata().file().cmp(b.metadata().file()); + if cmp == std::cmp::Ordering::Equal { + a.metadata().start_lines()[0].cmp(&b.metadata().start_lines()[0]) + } else { + cmp + } + }); + text_list } fn scanning_file_path(&self) -> &PathBuf { diff --git a/devtools/text-optimizer/src/extractors/std_output_extractor.rs b/devtools/text-optimizer/src/extractors/std_output_extractor.rs index e436be1d07..e7d2b51583 100644 --- a/devtools/text-optimizer/src/extractors/std_output_extractor.rs +++ b/devtools/text-optimizer/src/extractors/std_output_extractor.rs @@ -40,7 +40,16 @@ impl Extractor for StdOutputExtractor { } fn text_list(&self) -> Vec { - self.map.values().cloned().collect() + let mut text_list: Vec = self.map.values().cloned().collect(); + text_list.sort_by(|a, b| { + let cmp = a.metadata().file().cmp(b.metadata().file()); + if cmp == std::cmp::Ordering::Equal { + a.metadata().start_lines()[0].cmp(&b.metadata().start_lines()[0]) + } else { + cmp + } + }); + text_list } fn scanning_file_path(&self) -> &PathBuf { diff --git a/devtools/text-optimizer/src/extractors/thiserror_extractor.rs b/devtools/text-optimizer/src/extractors/thiserror_extractor.rs index 9cb719500c..d110fe3068 100644 --- a/devtools/text-optimizer/src/extractors/thiserror_extractor.rs +++ b/devtools/text-optimizer/src/extractors/thiserror_extractor.rs @@ -41,7 +41,16 @@ impl Extractor for ThiserrorExtractor { } fn text_list(&self) -> Vec { - self.map.values().cloned().collect() + let mut text_list: Vec = self.map.values().cloned().collect(); + text_list.sort_by(|a, b| { + let cmp = a.metadata().file().cmp(b.metadata().file()); + if cmp == std::cmp::Ordering::Equal { + a.metadata().start_lines()[0].cmp(&b.metadata().start_lines()[0]) + } else { + cmp + } + }); + text_list } fn scanning_file_path(&self) -> &PathBuf { From 48c333e8b425ea7c4259b3e0a20b72b10f3fcb4d Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Tue, 17 Oct 2023 18:44:58 +0800 Subject: [PATCH 17/22] Local file paths in meta are recorded using relative paths. --- devtools/text-optimizer/src/error.rs | 2 + devtools/text-optimizer/src/extractors/mod.rs | 42 +++++++++--- devtools/text-optimizer/src/main.rs | 53 ++++++++++++++- devtools/text-optimizer/src/yaml_processor.rs | 67 ++++++++++++++++++- 4 files changed, 147 insertions(+), 17 deletions(-) diff --git a/devtools/text-optimizer/src/error.rs b/devtools/text-optimizer/src/error.rs index e3dd29f043..3f42ece230 100644 --- a/devtools/text-optimizer/src/error.rs +++ b/devtools/text-optimizer/src/error.rs @@ -1,7 +1,9 @@ #[derive(Debug)] pub enum MyError { + CommitId, Io(std::io::Error), Serde(serde_yaml::Error), + PathError, } impl From for MyError { diff --git a/devtools/text-optimizer/src/extractors/mod.rs b/devtools/text-optimizer/src/extractors/mod.rs index 9fc561ec0d..ad98909de7 100644 --- a/devtools/text-optimizer/src/extractors/mod.rs +++ b/devtools/text-optimizer/src/extractors/mod.rs @@ -4,7 +4,7 @@ pub mod std_output_extractor; pub mod thiserror_extractor; use crate::{ - types::TextInfo, yaml_processor::save_yaml, CLAP_TEXT_FILE, LOG_TEXT_FILE, + error::MyError, types::TextInfo, yaml_processor::save_yaml, CLAP_TEXT_FILE, LOG_TEXT_FILE, STD_OUTPUT_TEXT_FILE, THISERROR_TEXT_FILE, }; @@ -27,14 +27,11 @@ pub trait Extractor { fn text_list(&self) -> Vec; fn scanning_file_path(&self) -> &PathBuf; fn save_file_path(&self) -> &PathBuf; - fn save_as_file(&self) { - save_yaml(self.save_file_path(), &self.text_list()).expect("save yaml"); - } fn visit_file(&mut self, node: &File); } -pub fn extract(project_root: PathBuf, output_dir: &PathBuf) { - // extractors +pub fn extract(root_cargo_config_path: PathBuf, commit_id: &str, output_dir: &PathBuf) { + // init all extractors let mut clap_extractor = ClapExtractor::new(output_dir.join(CLAP_TEXT_FILE)); let mut log_extractor = LogExtractor::new(output_dir.join(LOG_TEXT_FILE)); let mut std_output_extractor = StdOutputExtractor::new(output_dir.join(STD_OUTPUT_TEXT_FILE)); @@ -48,21 +45,24 @@ pub fn extract(project_root: PathBuf, output_dir: &PathBuf) { ]; let project_metadata = MetadataCommand::new() - .manifest_path(project_root) + .manifest_path(&root_cargo_config_path) .exec() .expect("Failed to get current directory"); for package in project_metadata.workspace_packages() { log::info!("Scanning Crate: {}", package.name); - let crate_src_path = Path::new(&package.manifest_path) + let crate_src_path = PathBuf::from(&package.manifest_path) .parent() .expect("workspace member crate path") .join("src"); + let crate_src_path = + to_relative_path(&crate_src_path, &root_cargo_config_path).expect("to_relative_path"); + process_rs_files_in_src(&crate_src_path, &mut extractors); } - save_extractors(output_dir, &extractors); + save_text_info_files(output_dir, &extractors, commit_id); } pub fn process_rs_files_in_src(dir_path: &Path, extractors: &mut [&mut dyn Extractor]) { @@ -103,14 +103,34 @@ pub fn extract_contents_in_brackets(lit: String) -> Option { None } -fn save_extractors(output_dir: &PathBuf, extractors: &[&mut dyn Extractor]) { +fn save_text_info_files(output_dir: &PathBuf, extractors: &[&mut dyn Extractor], commit_id: &str) { fs::create_dir_all(output_dir).expect("create dir all"); println!(); for extractor in extractors { - extractor.save_as_file(); + save_yaml( + extractor.save_file_path(), + &extractor.text_list(), + commit_id, + ) + .expect("save yaml"); + let file_name = extractor.save_file_path().file_name().unwrap(); let text_len = extractor.text_list().len(); println!("{:?}: {:?}", file_name, text_len); } } + +fn to_relative_path( + absolut_path: &Path, + root_cargo_config_path: &Path, +) -> Result { + let cargo_config_path_abs = root_cargo_config_path.canonicalize()?; + let project_root_path = cargo_config_path_abs.parent().ok_or(MyError::PathError)?; + let relative_path = absolut_path + .strip_prefix(project_root_path) + .map_err(|_| MyError::PathError)? + .to_str() + .ok_or(MyError::PathError)?; + Ok(Path::new("../..").join(relative_path)) +} diff --git a/devtools/text-optimizer/src/main.rs b/devtools/text-optimizer/src/main.rs index ef1a925a2c..a326de8b0a 100644 --- a/devtools/text-optimizer/src/main.rs +++ b/devtools/text-optimizer/src/main.rs @@ -6,10 +6,14 @@ mod yaml_processor; use backfill::backfill; use clap::{Parser, Subcommand}; +use error::MyError; use extractors::extract; use std::path::PathBuf; +use std::process::Command; +use std::result::Result; -pub const PROJECT_ROOT: &str = "../../Cargo.toml"; +pub const GITHUB_REPO: &str = "https://github.com/nervosnetwork/ckb/tree"; +pub const PROJECT_ROOT_CARGO_CONFIG_PATH: &str = "../../Cargo.toml"; pub const LOG_TEXT_FILE: &str = "log_text_list.yml"; pub const CLAP_TEXT_FILE: &str = "clap_text_list.yml"; pub const STD_OUTPUT_TEXT_FILE: &str = "std_output_text_list.yml"; @@ -26,6 +30,10 @@ struct Cli { enum Commands { /// extracting text Extract { + /// specify a github commit id as the rev for generating the source link + #[arg(short, long)] + commit_id: String, + /// specifies a directory path for .yml text files output #[arg(short, long, default_value = PathBuf::from("./").into_os_string())] output_dir: PathBuf, @@ -46,8 +54,16 @@ fn main() { ); match &cli.command { - Some(Commands::Extract { output_dir }) => { - extract(PathBuf::from(PROJECT_ROOT), output_dir); + Some(Commands::Extract { + commit_id, + output_dir, + }) => { + check_commit_id(commit_id).expect("check commit id"); + extract( + PathBuf::from(PROJECT_ROOT_CARGO_CONFIG_PATH), + commit_id, + output_dir, + ); } Some(Commands::Backfill { input_dir }) => { backfill(input_dir); @@ -55,3 +71,34 @@ fn main() { None => {} } } + +fn check_commit_id(commit_id: &str) -> Result<(), MyError> { + let output = Command::new("git") + .arg("rev-parse") + .arg("HEAD") + .output() + .expect("Failed to execute git command"); + + if output.status.success() { + let current_commit_id = String::from_utf8_lossy(&output.stdout); + let current_commit_id: String = current_commit_id.trim().chars().take(7).collect(); + let commit_id: String = commit_id.trim().chars().take(7).collect(); + if current_commit_id == commit_id { + log::info!( + "Current commit ID matches the expected commit ID: {}", + current_commit_id + ); + Ok(()) + } else { + log::warn!( + "Current commit ID {} does not match the expected commit ID {}.", + current_commit_id, + commit_id + ); + Err(MyError::CommitId) + } + } else { + log::error!("Failed to retrieve the current commit ID"); + Err(MyError::CommitId) + } +} diff --git a/devtools/text-optimizer/src/yaml_processor.rs b/devtools/text-optimizer/src/yaml_processor.rs index 097cff99c3..caa71b3c7b 100644 --- a/devtools/text-optimizer/src/yaml_processor.rs +++ b/devtools/text-optimizer/src/yaml_processor.rs @@ -1,15 +1,76 @@ -use super::{error::MyError, types::TextInfo}; +use super::{ + error::MyError, + types::{Category, Meta, TextInfo}, + GITHUB_REPO, +}; +use serde::{Deserialize, Serialize}; use std::io::Read; use std::io::Write; use std::{fs::File, path::PathBuf}; -pub fn save_yaml(file: &PathBuf, data: &[TextInfo]) -> Result<(), MyError> { +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Hash, Clone)] +pub struct TextInfoSave { + original: String, + editable: String, + metadata: Metadata, +} + +impl TextInfoSave { + pub fn from_text_info(text_info: TextInfo, git_repo: &str, commit_id: &str) -> Self { + // 使用 Metadata 的 from_meta 方法进行 Meta 到 Metadata 的转换 + let metadata = Metadata::from_meta(text_info.metadata(), git_repo, commit_id); + + // 创建 TextInfoSave 结构体并返回 + TextInfoSave { + original: text_info.original().to_owned(), + editable: text_info.editable().to_owned(), + metadata, + } + } +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Hash, Clone)] +pub struct Metadata { + category: Category, + file: PathBuf, + code_line_link: Vec, +} + +impl Metadata { + // 定义从 Meta 到 Metadata 的转换方法 + pub fn from_meta(meta: &Meta, github_repo: &str, commit_id: &str) -> Self { + // 创建 GitHub 代码行链接的前缀 + let github_link_prefix = format!("{}/{}/", github_repo, commit_id); + + // 为每个代码行生成 GitHub 链接 + let code_line_link: Vec = meta + .start_lines() + .iter() + .map(|line| format!("{}#L{}", github_link_prefix, line)) + .collect(); + + Metadata { + category: meta.category().to_owned(), + file: meta.file().to_owned(), + code_line_link, + } + } +} + +pub fn save_yaml(file: &PathBuf, data: &[TextInfo], commit_id: &str) -> Result<(), MyError> { let mut file = File::create(file)?; + + // Convert TextInfo to TextInfoSave + let data_save: Vec = data + .iter() + .map(|text_info| TextInfoSave::from_text_info(text_info.clone(), GITHUB_REPO, commit_id)) + .collect(); + file.write_fmt(format_args!( "# Number of TextInfo items: {}\n\n", data.len() ))?; - serde_yaml::to_writer(file, data)?; + serde_yaml::to_writer(file, &data_save)?; Ok(()) } From 2a73a6da7a6ac39abc288c3497d6927b3d7cdd57 Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Tue, 17 Oct 2023 19:09:38 +0800 Subject: [PATCH 18/22] add github link to yml files. --- devtools/text-optimizer/src/yaml_processor.rs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/devtools/text-optimizer/src/yaml_processor.rs b/devtools/text-optimizer/src/yaml_processor.rs index caa71b3c7b..c62d72d567 100644 --- a/devtools/text-optimizer/src/yaml_processor.rs +++ b/devtools/text-optimizer/src/yaml_processor.rs @@ -4,9 +4,9 @@ use super::{ GITHUB_REPO, }; use serde::{Deserialize, Serialize}; -use std::io::Read; use std::io::Write; use std::{fs::File, path::PathBuf}; +use std::{io::Read, path::Path}; #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Hash, Clone)] pub struct TextInfoSave { @@ -17,10 +17,13 @@ pub struct TextInfoSave { impl TextInfoSave { pub fn from_text_info(text_info: TextInfo, git_repo: &str, commit_id: &str) -> Self { - // 使用 Metadata 的 from_meta 方法进行 Meta 到 Metadata 的转换 - let metadata = Metadata::from_meta(text_info.metadata(), git_repo, commit_id); + let metadata = Metadata::from_meta( + text_info.metadata(), + git_repo, + commit_id, + text_info.metadata().file(), + ); - // 创建 TextInfoSave 结构体并返回 TextInfoSave { original: text_info.original().to_owned(), editable: text_info.editable().to_owned(), @@ -37,12 +40,9 @@ pub struct Metadata { } impl Metadata { - // 定义从 Meta 到 Metadata 的转换方法 - pub fn from_meta(meta: &Meta, github_repo: &str, commit_id: &str) -> Self { - // 创建 GitHub 代码行链接的前缀 - let github_link_prefix = format!("{}/{}/", github_repo, commit_id); - - // 为每个代码行生成 GitHub 链接 + pub fn from_meta(meta: &Meta, github_repo: &str, commit_id: &str, file: &Path) -> Self { + let file = file.strip_prefix("../..").expect("strip prefix"); + let github_link_prefix = format!("{}/{}/{}", github_repo, commit_id, file.display()); let code_line_link: Vec = meta .start_lines() .iter() From 1e493a87be99abc3c9ac4e50d654732dd144c38a Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Mon, 20 Nov 2023 21:36:23 +0800 Subject: [PATCH 19/22] add README. --- devtools/text-optimizer/README.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 devtools/text-optimizer/README.md diff --git a/devtools/text-optimizer/README.md b/devtools/text-optimizer/README.md new file mode 100644 index 0000000000..f288ded3d8 --- /dev/null +++ b/devtools/text-optimizer/README.md @@ -0,0 +1,4 @@ +- compile tools +- extract text on rev +- edit +- fillback on rev From 81fb4e773840540a912c0835265a8072d76074b6 Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Mon, 20 Nov 2023 21:57:46 +0800 Subject: [PATCH 20/22] add logs. --- devtools/text-optimizer/src/backfill.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/devtools/text-optimizer/src/backfill.rs b/devtools/text-optimizer/src/backfill.rs index 7e68d65463..10fa7da64b 100644 --- a/devtools/text-optimizer/src/backfill.rs +++ b/devtools/text-optimizer/src/backfill.rs @@ -22,6 +22,7 @@ fn load_all_text_info_files(input_dir: &PathBuf) -> Result Date: Tue, 28 Nov 2023 11:47:54 +0800 Subject: [PATCH 21/22] fix load_yaml. --- .../src/extractors/clap_extractor.rs | 9 ++++-- .../src/extractors/log_extractor.rs | 9 ++++-- .../src/extractors/std_output_extractor.rs | 9 ++++-- .../src/extractors/thiserror_extractor.rs | 4 +-- devtools/text-optimizer/src/types.rs | 16 +++++++--- devtools/text-optimizer/src/yaml_processor.rs | 29 +++++++++++++++++-- 6 files changed, 59 insertions(+), 17 deletions(-) diff --git a/devtools/text-optimizer/src/extractors/clap_extractor.rs b/devtools/text-optimizer/src/extractors/clap_extractor.rs index 992d055b96..b6f2d1a39e 100644 --- a/devtools/text-optimizer/src/extractors/clap_extractor.rs +++ b/devtools/text-optimizer/src/extractors/clap_extractor.rs @@ -81,9 +81,12 @@ impl syn::visit::Visit<'_> for ClapExtractor { let start_line = span.start().line; let category = Category::from_str(method_ident.to_string().as_str()).unwrap(); - let meta = - Meta::new(category, self.scanning_file_path.to_owned(), start_line); - self.add_text_info(TextInfo::new(text, meta)); + let meta = Meta::new_line( + category, + self.scanning_file_path.to_owned(), + start_line, + ); + self.add_text_info(TextInfo::new(text.clone(), text, meta)); } } } diff --git a/devtools/text-optimizer/src/extractors/log_extractor.rs b/devtools/text-optimizer/src/extractors/log_extractor.rs index 50ff2bd4b1..3bc598d89c 100644 --- a/devtools/text-optimizer/src/extractors/log_extractor.rs +++ b/devtools/text-optimizer/src/extractors/log_extractor.rs @@ -81,9 +81,12 @@ impl syn::visit::Visit<'_> for LogExtractor { let span = lit.span(); let start_line = span.start().line; let category = Category::from_str(&name).unwrap(); - let meta = - Meta::new(category, self.scanning_file_path.to_owned(), start_line); - self.add_text_info(TextInfo::new(text, meta)); + let meta = Meta::new_line( + category, + self.scanning_file_path.to_owned(), + start_line, + ); + self.add_text_info(TextInfo::new(text.clone(), text, meta)); } } } diff --git a/devtools/text-optimizer/src/extractors/std_output_extractor.rs b/devtools/text-optimizer/src/extractors/std_output_extractor.rs index e7d2b51583..4afd199a1e 100644 --- a/devtools/text-optimizer/src/extractors/std_output_extractor.rs +++ b/devtools/text-optimizer/src/extractors/std_output_extractor.rs @@ -76,9 +76,12 @@ impl syn::visit::Visit<'_> for StdOutputExtractor { let span = lit.span(); let start_line = span.start().line; let category = Category::from_str(ident.to_string().as_str()).unwrap(); - let meta = - Meta::new(category, self.scanning_file_path.to_owned(), start_line); - self.add_text_info(TextInfo::new(text, meta)); + let meta = Meta::new_line( + category, + self.scanning_file_path.to_owned(), + start_line, + ); + self.add_text_info(TextInfo::new(text.clone(), text, meta)); } } } diff --git a/devtools/text-optimizer/src/extractors/thiserror_extractor.rs b/devtools/text-optimizer/src/extractors/thiserror_extractor.rs index d110fe3068..7f9365cbec 100644 --- a/devtools/text-optimizer/src/extractors/thiserror_extractor.rs +++ b/devtools/text-optimizer/src/extractors/thiserror_extractor.rs @@ -91,12 +91,12 @@ impl syn::visit::Visit<'_> for ThiserrorExtractor { let span = lit_str.span(); let start_line = span.start().line; - let meta = Meta::new( + let meta = Meta::new_line( Category::ThisError, self.scanning_file_path.to_owned(), start_line, ); - self.add_text_info(TextInfo::new(text, meta)); + self.add_text_info(TextInfo::new(text.clone(), text, meta)); } } } diff --git a/devtools/text-optimizer/src/types.rs b/devtools/text-optimizer/src/types.rs index b6237abbde..6d7136b378 100644 --- a/devtools/text-optimizer/src/types.rs +++ b/devtools/text-optimizer/src/types.rs @@ -11,10 +11,10 @@ pub struct TextInfo { } impl TextInfo { - pub fn new(original: String, metadata: Meta) -> Self { + pub fn new(original: String, editable: String, metadata: Meta) -> Self { TextInfo { - original: original.to_owned(), - editable: original, + original, + editable, metadata, } } @@ -47,7 +47,7 @@ pub struct Meta { } impl Meta { - pub fn new(category: Category, file: PathBuf, start_line: usize) -> Self { + pub fn new_line(category: Category, file: PathBuf, start_line: usize) -> Self { Meta { category, file, @@ -55,6 +55,14 @@ impl Meta { } } + pub fn new(category: Category, file: PathBuf, code_lines: Vec) -> Self { + Meta { + category, + file, + code_lines, + } + } + #[allow(dead_code)] pub fn category(&self) -> &Category { &self.category diff --git a/devtools/text-optimizer/src/yaml_processor.rs b/devtools/text-optimizer/src/yaml_processor.rs index c62d72d567..a47d366f24 100644 --- a/devtools/text-optimizer/src/yaml_processor.rs +++ b/devtools/text-optimizer/src/yaml_processor.rs @@ -5,6 +5,7 @@ use super::{ }; use serde::{Deserialize, Serialize}; use std::io::Write; +use std::str::FromStr; use std::{fs::File, path::PathBuf}; use std::{io::Read, path::Path}; @@ -32,6 +33,13 @@ impl TextInfoSave { } } +impl From for TextInfo { + fn from(text_info_save: TextInfoSave) -> Self { + let metadata = Meta::from(text_info_save.metadata); + TextInfo::new(text_info_save.original, text_info_save.editable, metadata) + } +} + #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Hash, Clone)] pub struct Metadata { category: Category, @@ -39,6 +47,20 @@ pub struct Metadata { code_line_link: Vec, } +impl From for Meta { + fn from(metadata: Metadata) -> Self { + let code_lines: Vec = metadata + .code_line_link + .iter() + .map(|link| { + let line = link.split("#L").last().expect("split line"); + usize::from_str(line).expect("parse line") + }) + .collect(); + Meta::new(metadata.category, metadata.file, code_lines) + } +} + impl Metadata { pub fn from_meta(meta: &Meta, github_repo: &str, commit_id: &str, file: &Path) -> Self { let file = file.strip_prefix("../..").expect("strip prefix"); @@ -79,7 +101,10 @@ pub fn load_yaml(filename: &PathBuf) -> Result, MyError> { let mut yaml_content = String::new(); file.read_to_string(&mut yaml_content)?; - let extracted_texts: Vec = serde_yaml::from_str(&yaml_content)?; - + let extracted_texts: Vec = serde_yaml::from_str(&yaml_content)?; + let extracted_texts = extracted_texts + .iter() + .map(|text_info_save| TextInfo::from(text_info_save.clone())) + .collect(); Ok(extracted_texts) } From abf6c2a8f5697f926735381b913c5db6af2f6527 Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Tue, 28 Nov 2023 21:55:39 +0800 Subject: [PATCH 22/22] refactoring: backfill_edited. --- devtools/text-optimizer/src/backfill.rs | 46 +++++++++++++++++-------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/devtools/text-optimizer/src/backfill.rs b/devtools/text-optimizer/src/backfill.rs index 10fa7da64b..23e59da7bd 100644 --- a/devtools/text-optimizer/src/backfill.rs +++ b/devtools/text-optimizer/src/backfill.rs @@ -2,6 +2,7 @@ use crate::error::MyError; use crate::types::TextInfo; use crate::yaml_processor::load_yaml; +use std::collections::HashMap; use std::fs::read_dir; use std::fs::File; use std::io::{Read, Write}; @@ -32,24 +33,41 @@ fn load_all_text_info_files(input_dir: &PathBuf) -> Result)>) { + let mut file_to_text_infos: HashMap> = HashMap::new(); + for list in text_info_lists { log::info!("Parse text info file: {:?}", list.0); for text_info in list.1 { - let mut source_code = String::new(); - let mut file = File::open(text_info.metadata().file()).expect("Failed to open file"); - file.read_to_string(&mut source_code) - .expect("Failed to read file"); - - // Replace the match origianl with the edited - let new_source_code = source_code.replace(text_info.original(), text_info.editable()); - - // Reopen the file in write mode and write the new source code - let mut new_file = - File::create(text_info.metadata().file()).expect("Failed to create file"); - new_file - .write_all(new_source_code.as_bytes()) - .expect("Failed to write file"); + let file_name = text_info + .metadata() + .file() + .to_str() + .expect("Convert file path to string.") + .to_owned(); + file_to_text_infos + .entry(file_name) + .or_default() + .push(text_info); } } + + for (file_name, text_infos) in file_to_text_infos { + let mut source_code = String::new(); + let mut file = File::open(&file_name).expect("Failed to open file"); + file.read_to_string(&mut source_code) + .expect("Failed to read file"); + + for text_info in text_infos { + // Replace the match original with the edited + source_code = source_code.replace(text_info.original(), text_info.editable()); + } + + // Reopen the file in write mode and write the new source code + let mut new_file = File::create(&file_name).expect("Failed to create file"); + new_file + .write_all(source_code.as_bytes()) + .expect("Failed to write file"); + } + log::info!("The backfill is completed, please review the modifications in the source code."); }