From a6113d86df9f56f320f27cdcada564050331a5ed Mon Sep 17 00:00:00 2001 From: Daniel Rebelo de Oliveira Date: Mon, 16 Sep 2024 23:42:25 +0100 Subject: [PATCH] fix bug with repeated excludes from restic restic can return repeated excluded paths for a snapshot which resulted in a primary key constraint failure when inserting. Here we turn paths, excludes and tags into sets to ensure uniqueness. --- benches/cache.rs | 24 ++++++++++++++++------ src/cache/tests.rs | 51 +++++++++++++++++++++++++++++----------------- src/restic.rs | 7 ++++--- 3 files changed, 54 insertions(+), 28 deletions(-) diff --git a/benches/cache.rs b/benches/cache.rs index cc3a81b..b3270f5 100644 --- a/benches/cache.rs +++ b/benches/cache.rs @@ -25,7 +25,9 @@ pub fn criterion_benchmark(c: &mut Criterion) { "/home/user".to_string(), "/etc".to_string(), "/var".to_string(), - ], + ] + .into_iter() + .collect(), hostname: Some("foo.com".to_string()), username: Some("user".to_string()), uid: Some(123), @@ -34,8 +36,12 @@ pub fn criterion_benchmark(c: &mut Criterion) { ".cache".to_string(), "Cache".to_string(), "/home/user/Downloads".to_string(), - ], - tags: vec!["foo_machine".to_string(), "rewrite".to_string()], + ] + .into_iter() + .collect(), + tags: vec!["foo_machine".to_string(), "rewrite".to_string()] + .into_iter() + .collect(), original_id: Some("fefwfwew".to_string()), program_version: Some("restic 0.16.0".to_string()), }; @@ -63,7 +69,9 @@ pub fn criterion_benchmark(c: &mut Criterion) { "/home/user".to_string(), "/etc".to_string(), "/var".to_string(), - ], + ] + .into_iter() + .collect(), hostname: Some("foo.com".to_string()), username: Some("user".to_string()), uid: Some(123), @@ -72,8 +80,12 @@ pub fn criterion_benchmark(c: &mut Criterion) { ".cache".to_string(), "Cache".to_string(), "/home/user/Downloads".to_string(), - ], - tags: vec!["foo_machine".to_string(), "rewrite".to_string()], + ] + .into_iter() + .collect(), + tags: vec!["foo_machine".to_string(), "rewrite".to_string()] + .into_iter() + .collect(), original_id: Some("fefwfwew".to_string()), program_version: Some("restic 0.16.0".to_string()), } diff --git a/src/cache/tests.rs b/src/cache/tests.rs index 76f30ab..e565ed8 100644 --- a/src/cache/tests.rs +++ b/src/cache/tests.rs @@ -1,5 +1,6 @@ use std::{ - cmp::Reverse, convert::Infallible, env, fs, iter, mem, path::PathBuf, + cmp::Reverse, collections::HashSet, convert::Infallible, env, fs, iter, + mem, path::PathBuf, }; use camino::{Utf8Path, Utf8PathBuf}; @@ -323,21 +324,23 @@ fn cache_snapshots_entries() { assert_eq!(s0.original_id, s1.original_id); assert_eq!(s0.program_version, s1.program_version); - let mut s0_paths: Vec = s0.paths.to_vec(); + let mut s0_paths: Vec = s0.paths.iter().cloned().collect(); s0_paths.sort(); - let mut s1_paths: Vec = s1.paths.to_vec(); + let mut s1_paths: Vec = s1.paths.iter().cloned().collect(); s1_paths.sort(); assert_eq!(s0_paths, s1_paths); - let mut s0_excludes: Vec = s0.excludes.to_vec(); + let mut s0_excludes: Vec = + s0.excludes.iter().cloned().collect(); s0_excludes.sort(); - let mut s1_excludes: Vec = s1.excludes.to_vec(); + let mut s1_excludes: Vec = + s1.excludes.iter().cloned().collect(); s1_excludes.sort(); assert_eq!(s0_excludes, s1_excludes); - let mut s0_tags: Vec = s0.tags.to_vec(); + let mut s0_tags: Vec = s0.tags.iter().cloned().collect(); s0_tags.sort(); - let mut s1_tags: Vec = s1.tags.to_vec(); + let mut s1_tags: Vec = s1.tags.iter().cloned().collect(); s1_tags.sort(); assert_eq!(s0_tags, s1_tags); } @@ -355,7 +358,9 @@ fn cache_snapshots_entries() { "/home/user".to_string(), "/etc".to_string(), "/var".to_string(), - ], + ] + .into_iter() + .collect(), hostname: Some("foo.com".to_string()), username: Some("user".to_string()), uid: Some(123), @@ -364,8 +369,12 @@ fn cache_snapshots_entries() { ".cache".to_string(), "Cache".to_string(), "/home/user/Downloads".to_string(), - ], - tags: vec!["foo_machine".to_string(), "rewrite".to_string()], + ] + .into_iter() + .collect(), + tags: vec!["foo_machine".to_string(), "rewrite".to_string()] + .into_iter() + .collect(), original_id: Some("fefwfwew".to_string()), program_version: Some("restic 0.16.0".to_string()), }; @@ -375,7 +384,7 @@ fn cache_snapshots_entries() { time: mk_datetime(2025, 5, 12, 17, 00, 00), parent: Some("wat".to_string()), tree: "anothertree".to_string(), - paths: vec!["/home/user".to_string()], + paths: vec!["/home/user".to_string()].into_iter().collect(), hostname: Some("foo.com".to_string()), username: Some("user".to_string()), uid: Some(123), @@ -384,8 +393,12 @@ fn cache_snapshots_entries() { ".cache".to_string(), "Cache".to_string(), "/home/user/Downloads".to_string(), - ], - tags: vec!["foo_machine".to_string(), "rewrite".to_string()], + ] + .into_iter() + .collect(), + tags: vec!["foo_machine".to_string(), "rewrite".to_string()] + .into_iter() + .collect(), original_id: Some("fefwfwew".to_string()), program_version: Some("restic 0.16.0".to_string()), }; @@ -395,13 +408,13 @@ fn cache_snapshots_entries() { time: mk_datetime(2023, 5, 12, 17, 00, 00), parent: None, tree: "fwefwfwwefwefwe".to_string(), - paths: vec![], + paths: HashSet::new(), hostname: None, username: None, uid: None, gid: None, - excludes: vec![], - tags: vec![], + excludes: HashSet::new(), + tags: HashSet::new(), original_id: None, program_version: None, }; @@ -460,13 +473,13 @@ fn lots_of_snapshots() { time: timestamp_to_datetime(i as i64).unwrap(), parent: None, tree: i.to_string(), - paths: vec![], + paths: HashSet::new(), hostname: None, username: None, uid: None, gid: None, - excludes: vec![], - tags: vec![], + excludes: HashSet::new(), + tags: HashSet::new(), original_id: None, program_version: None, }; diff --git a/src/restic.rs b/src/restic.rs index dbfa12e..ebccea6 100644 --- a/src/restic.rs +++ b/src/restic.rs @@ -3,6 +3,7 @@ use core::str; use std::os::unix::process::CommandExt; use std::{ borrow::Cow, + collections::HashSet, ffi::OsStr, fmt::{self, Display, Formatter}, io::{self, BufRead, BufReader, Lines, Read}, @@ -313,7 +314,7 @@ pub struct Snapshot { #[serde(default)] pub parent: Option, pub tree: String, - pub paths: Vec, + pub paths: HashSet, #[serde(default)] pub hostname: Option, #[serde(default)] @@ -323,9 +324,9 @@ pub struct Snapshot { #[serde(default)] pub gid: Option, #[serde(default)] - pub excludes: Vec, + pub excludes: HashSet, #[serde(default)] - pub tags: Vec, + pub tags: HashSet, #[serde(default)] pub original_id: Option, #[serde(default)]