From 988889020e6cf2807e3180ed460d28e27b4ab022 Mon Sep 17 00:00:00 2001 From: Mohamed Achaq Date: Fri, 20 Dec 2024 12:48:50 +0100 Subject: [PATCH 1/5] feat: slightly faster tree and recursive listing --- .gitignore | 4 +- lla/src/formatter/tree.rs | 233 ++++++++++++++---------------------- lla/src/lister/recursive.rs | 57 +++++++-- 3 files changed, 143 insertions(+), 151 deletions(-) diff --git a/.gitignore b/.gitignore index faea388..dfd5557 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,9 @@ .DS_Store target/ -# First ignore everything in completions +.cache/ + completions/* -# Then explicitly allow the shell completion files !completions/*.bash !completions/*.fish !completions/_lla diff --git a/lla/src/formatter/tree.rs b/lla/src/formatter/tree.rs index ee6bd1d..83b173c 100644 --- a/lla/src/formatter/tree.rs +++ b/lla/src/formatter/tree.rs @@ -1,35 +1,15 @@ use super::FileFormatter; use crate::error::Result; use crate::plugin::PluginManager; -use crate::theme::{self, ColorValue}; -use crate::utils::color::{colorize_file_name, colorize_file_name_with_icon}; +use crate::utils::color::*; use crate::utils::icons::format_with_icon; -use colored::*; +use colored::Colorize; use lla_plugin_interface::proto::DecoratedEntry; -use std::path::Path; +use std::collections::{HashMap, HashSet}; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; -#[derive(PartialEq, Eq, Debug, Copy, Clone)] -enum TreePart { - Edge, - Line, - Corner, -} - -impl TreePart { - #[inline] - const fn as_str(self) -> &'static str { - match self { - Self::Edge => "├── ", - Self::Line => "│ ", - Self::Corner => "└── ", - } - } - - fn colored(self) -> ColoredString { - let color = theme::color_value_to_color(&ColorValue::Named("bright black".to_string())); - self.as_str().color(color) - } -} +const BUFFER_SIZE: usize = 16384; pub struct TreeFormatter { pub show_icons: bool, @@ -39,79 +19,95 @@ impl TreeFormatter { pub fn new(show_icons: bool) -> Self { Self { show_icons } } -} -impl TreeFormatter { - fn format_entry( - entry: &DecoratedEntry, - prefix: &str, - plugin_manager: &mut PluginManager, - buf: &mut String, - show_icons: bool, - ) { - buf.clear(); - let path = Path::new(&entry.path); - buf.reserve(prefix.len() + path.as_os_str().len() + 1); - buf.push_str(prefix); + fn format_entry(&self, path: &Path) -> String { let colored_name = colorize_file_name(path).to_string(); - buf.push_str(&colorize_file_name_with_icon( - path, - format_with_icon(path, colored_name, show_icons), - )); - - let plugin_fields = plugin_manager.format_fields(entry, "tree").join(" "); - if !plugin_fields.is_empty() { - buf.push(' '); - buf.push_str(&plugin_fields); + if self.show_icons { + format_with_icon(path, colored_name, true) + } else { + colored_name } - - buf.push('\n'); } -} - -#[derive(Debug)] -struct TreeTrunk { - stack: Vec, - last_depth: Option<(usize, bool)>, -} -impl Default for TreeTrunk { - fn default() -> Self { - Self { - stack: Vec::with_capacity(32), - last_depth: None, + fn build_tree( + &self, + entries: &[DecoratedEntry], + ) -> (Vec, HashMap>) { + let mut tree: HashMap> = HashMap::with_capacity(entries.len()); + let mut path_set: HashSet = HashSet::with_capacity(entries.len()); + let mut child_paths = HashSet::new(); + + for entry in entries { + path_set.insert(PathBuf::from(&entry.path)); } - } -} -impl TreeTrunk { - #[inline] - fn get_prefix(&mut self, depth: usize, is_absolute_last: bool, buf: &mut String) { - if let Some((last_depth, _)) = self.last_depth { - if last_depth < self.stack.len() { - self.stack[last_depth] = TreePart::Line; + for path in path_set.iter() { + if let Some(parent) = path.parent() { + if path_set.contains(parent) { + tree.entry(parent.to_path_buf()) + .or_insert_with(Vec::new) + .push(path.clone()); + child_paths.insert(path.clone()); + } } } - - if depth + 1 > self.stack.len() { - self.stack.resize(depth + 1, TreePart::Line); + for children in tree.values_mut() { + children.sort_unstable(); } + let mut root_paths: Vec<_> = path_set + .into_iter() + .filter(|path| !child_paths.contains(path)) + .collect(); + root_paths.sort_unstable(); - if depth < self.stack.len() { - self.stack[depth] = if is_absolute_last { - TreePart::Corner - } else { - TreePart::Edge - }; - } + (root_paths, tree) + } - self.last_depth = Some((depth, is_absolute_last)); + fn write_tree_recursive( + &self, + path: &Path, + prefix: &str, + is_last: bool, + tree: &HashMap>, + writer: &mut impl Write, + current_depth: usize, + max_depth: Option, + ) -> io::Result<()> { + if let Some(max) = max_depth { + if current_depth > max { + return Ok(()); + } + } - buf.clear(); - buf.reserve(depth * 4); - for part in self.stack[1..=depth].iter() { - buf.push_str(&part.colored()); + let node_prefix = if is_last { "└── " } else { "├── " }; + let child_prefix = if is_last { " " } else { "│ " }; + + let formatted_name = self.format_entry(path); + write!( + writer, + "{}{}{}\n", + prefix.bright_black(), + node_prefix.bright_black(), + formatted_name + )?; + + if let Some(children) = tree.get(path) { + let new_prefix = format!("{}{}", prefix, child_prefix); + let last_idx = children.len().saturating_sub(1); + for (i, child) in children.iter().enumerate() { + let is_last_child = i == last_idx; + self.write_tree_recursive( + child, + &new_prefix, + is_last_child, + tree, + writer, + current_depth + 1, + max_depth, + )?; + } } + Ok(()) } } @@ -119,69 +115,26 @@ impl FileFormatter for TreeFormatter { fn format_files( &self, files: &[DecoratedEntry], - plugin_manager: &mut PluginManager, - max_depth: Option, + _plugin_manager: &mut PluginManager, + depth: Option, ) -> Result { if files.is_empty() { return Ok(String::new()); } - let mut trunk = TreeTrunk::default(); - let mut prefix_buf = String::with_capacity(128); - let mut entry_buf = String::with_capacity(256); - let mut result = String::new(); - - let mut entries: Vec<_> = files - .iter() - .map(|entry| { - let path = Path::new(&entry.path); - let depth = path.components().count(); - (entry, depth, path.to_path_buf()) - }) - .collect(); - - entries.sort_by(|a, b| a.2.cmp(&b.2)); - - if let Some(max_depth) = max_depth { - entries.retain(|(_, depth, _)| *depth <= max_depth); + if depth == Some(0) { + return Ok(String::new()); } - let avg_line_len = entries - .first() - .map(|(e, d, _)| { - let path = Path::new(&e.path); - let name_len = path.file_name().map_or(0, |n| n.len()); - let prefix_len = *d * 4; - name_len + prefix_len + 1 - }) - .unwrap_or(64); - - result.reserve(entries.len() * avg_line_len); - - const CHUNK_SIZE: usize = 8192; - for chunk in entries.chunks(CHUNK_SIZE) { - let chunk_len = chunk.len(); - for (i, (entry, depth, path)) in chunk.iter().enumerate() { - let is_last = if i + 1 < chunk_len { - let (next_entry, next_depth, _) = &chunk[i + 1]; - *depth > *next_depth - || !Path::new(&next_entry.path).starts_with(path.parent().unwrap_or(path)) - } else { - true - }; - - trunk.get_prefix(*depth, is_last, &mut prefix_buf); - Self::format_entry( - entry, - &prefix_buf, - plugin_manager, - &mut entry_buf, - self.show_icons, - ); - result.push_str(&entry_buf); - } + let (root_paths, tree) = self.build_tree(files); + let mut buffer = Vec::with_capacity(BUFFER_SIZE); + + let last_idx = root_paths.len().saturating_sub(1); + for (i, path) in root_paths.iter().enumerate() { + let is_last = i == last_idx; + self.write_tree_recursive(path, "", is_last, &tree, &mut buffer, 0, depth)?; } - Ok(result) + Ok(String::from_utf8_lossy(&buffer).into_owned()) } } diff --git a/lla/src/lister/recursive.rs b/lla/src/lister/recursive.rs index c6130e8..1bf0545 100644 --- a/lla/src/lister/recursive.rs +++ b/lla/src/lister/recursive.rs @@ -4,7 +4,12 @@ use crate::error::Result; use crate::lister::BasicLister; use rayon::prelude::*; use std::path::PathBuf; -use walkdir::WalkDir; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use walkdir::{DirEntry, WalkDir}; + +const PARALLEL_THRESHOLD: usize = 1000; +const BUFFER_CAPACITY: usize = 1024; pub struct RecursiveLister { config: Config, @@ -14,6 +19,35 @@ impl RecursiveLister { pub fn new(config: Config) -> Self { Self { config } } + + fn is_hidden(entry: &DirEntry) -> bool { + entry + .file_name() + .to_str() + .map(|s| s.starts_with('.')) + .unwrap_or(false) + } + + fn should_process_entry( + entry: &DirEntry, + counter: &Arc, + max_entries: usize, + ) -> bool { + if counter.load(Ordering::Relaxed) >= max_entries { + return false; + } + + if !entry.file_type().is_file() { + return true; + } + + if !Self::is_hidden(entry) { + counter.fetch_add(1, Ordering::Relaxed); + true + } else { + false + } + } } impl FileLister for RecursiveLister { @@ -34,21 +68,26 @@ impl FileLister for RecursiveLister { .recursive .max_entries .unwrap_or(usize::MAX); - let mut entries = Vec::with_capacity(128); + + let counter = Arc::new(AtomicUsize::new(0)); + let mut entries = Vec::with_capacity(BUFFER_CAPACITY); + let walker = WalkDir::new(directory) .min_depth(0) .max_depth(max_depth) .follow_links(false) - .same_file_system(true); + .same_file_system(true) + .into_iter() + .filter_map(|e| e.ok()) + .filter(|e| Self::should_process_entry(e, &counter, max_entries)) + .collect::>(); - for entry in walker.into_iter().filter_map(|e| e.ok()) { - entries.push(entry.into_path()); - if entries.len() >= max_entries { - break; - } + if walker.len() > PARALLEL_THRESHOLD { + entries.par_extend(walker.into_par_iter().map(|e| e.into_path())); + } else { + entries.extend(walker.into_iter().map(|e| e.into_path())); } - entries.par_sort_unstable(); Ok(entries) } } From b65bbdcc4bec6fe714141d3de9d076fbfccfd0dc Mon Sep 17 00:00:00 2001 From: Mohamed Achaq Date: Fri, 20 Dec 2024 12:49:45 +0100 Subject: [PATCH 2/5] feat: slightly faster fuzzy seaarch --- lla/src/lister/fuzzy.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lla/src/lister/fuzzy.rs b/lla/src/lister/fuzzy.rs index 09a71ba..255e553 100644 --- a/lla/src/lister/fuzzy.rs +++ b/lla/src/lister/fuzzy.rs @@ -468,7 +468,7 @@ impl FuzzyLister { .hidden(false) .git_ignore(false) .ignore(false) - .follow_links(true) + .follow_links(false) .same_file_system(false) .threads(num_cpus::get()) .build_parallel(); From 348f104f1dadee69d362db2af5d04a01cbb9d70d Mon Sep 17 00:00:00 2001 From: Mohamed Achaq Date: Fri, 20 Dec 2024 12:50:40 +0100 Subject: [PATCH 3/5] feat: optimize directory size calculation --- lla/src/commands/file_utils.rs | 40 ++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/lla/src/commands/file_utils.rs b/lla/src/commands/file_utils.rs index 2ff6406..1aca3b1 100644 --- a/lla/src/commands/file_utils.rs +++ b/lla/src/commands/file_utils.rs @@ -102,19 +102,35 @@ pub fn convert_metadata(metadata: &std::fs::Metadata) -> EntryMetadata { } fn calculate_dir_size(path: &std::path::Path) -> std::io::Result { - let mut total_size = 0; - if path.is_dir() { - for entry in std::fs::read_dir(path)? { - let entry = entry?; - let path = entry.path(); - if path.is_dir() { - total_size += calculate_dir_size(&path)?; - } else { - total_size += entry.metadata()?.len(); - } - } + use rayon::prelude::*; + + if !path.is_dir() { + return Ok(0); } - Ok(total_size) + + let entries: Vec<_> = std::fs::read_dir(path)?.collect::>()?; + + entries + .par_iter() + .try_fold( + || 0u64, + |acc, entry| { + let metadata = entry.metadata()?; + if metadata.is_symlink() { + return Ok(acc); + } + + let path = entry.path(); + let size = if metadata.is_dir() { + calculate_dir_size(&path)? + } else { + metadata.len() + }; + + Ok(acc + size) + }, + ) + .try_reduce(|| 0, |a, b| Ok(a + b)) } pub fn list_and_decorate_files( From e66ca8bcc816bcacbfab8583997f47030d5d5657 Mon Sep 17 00:00:00 2001 From: Mohamed Achaq Date: Fri, 20 Dec 2024 13:00:20 +0100 Subject: [PATCH 4/5] chore: bump version to 0.3.7 --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- lla/Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 22cd9e3..77aac35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -721,7 +721,7 @@ checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "lla" -version = "0.3.6" +version = "0.3.7" dependencies = [ "atty", "chrono", @@ -759,7 +759,7 @@ dependencies = [ [[package]] name = "lla_plugin_interface" -version = "0.3.6" +version = "0.3.7" dependencies = [ "prost", "prost-build", diff --git a/Cargo.toml b/Cargo.toml index a9b2e86..1900710 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ members = ["lla", "lla_plugin_interface", "plugins/*"] [workspace.package] description = "Blazing Fast and highly customizable ls Replacement with Superpowers" authors = ["Achaq "] -version = "0.3.6" +version = "0.3.7" categories = ["utilities", "file-system", "cli", "file-management"] edition = "2021" license = "MIT" diff --git a/lla/Cargo.toml b/lla/Cargo.toml index cdcfba0..3c1e641 100644 --- a/lla/Cargo.toml +++ b/lla/Cargo.toml @@ -28,7 +28,7 @@ walkdir.workspace = true tempfile.workspace = true users.workspace = true parking_lot.workspace = true -lla_plugin_interface = { version = "0.3.6", path = "../lla_plugin_interface" } +lla_plugin_interface = { version = "0.3.7", path = "../lla_plugin_interface" } once_cell.workspace = true dashmap.workspace = true unicode-width.workspace = true From 55c8fcd4bea0a2bc78f9e4328ba7a3ea44bab130 Mon Sep 17 00:00:00 2001 From: Mohamed Achaq Date: Fri, 20 Dec 2024 13:07:50 +0100 Subject: [PATCH 5/5] chore: update CHANGELOG for version 0.3.7 with performance improvements and bug fixes --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65e0b84..7f24746 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.3.7] - 2024-12-20 + +### Changed + +- Faster recursive directory listing with optimized traversal +- Improved fuzzy search performance and accuracy +- Enhanced tree format with more efficient rendering +- Redesigned size calculation logic for faster and more accurate results +- General stability improvements and bug fixes + ## [0.3.6] - 2024-12-18 ### Added