CSAF Downloader: Added "after_date" and "before_date"

giterlizzi · Oct 17, 2024 · 10b9c86 · 10b9c86
1 parent b8858d8
commit 10b9c86
Show file tree

Hide file tree

Showing 4 changed files with 76 additions and 4 deletions.
diff --git a/bin/csaf-downloader b/bin/csaf-downloader
@@ -25,6 +25,7 @@ csaf-downloader - CSAF Downloader
 
     Options:
       -u, --url=URL                "provider-metadata.json", "index.txt" or ROLIE feed URL
+      -d, --directory=PATH          Output directory
       -k, --insecure                Skip TLS verification
           --parallel-downloads=NUM  Number of parallel downloads (default: 10)
 
@@ -35,6 +36,9 @@ csaf-downloader - CSAF Downloader
           --include=REGEXP          Include file
           --exclude=REGEXP          Exclude file
 
+          --after=Validate          Download CSAF documents more recent than a specific date
+          --before=DATE             Download CSAF documents older than a specific date
+
       -v, --verbose                 Verbose
 
           --config=FILE             Configuration file
@@ -61,6 +65,10 @@ Download using ROLIE feed document:
 
     $ csaf-downloader -u https://psirt.domain.tld/advisories/csaf/feed-tlp-white.json
 
+Download CSAF documents more recent than a specific date:
+
+    $ csaf-downloader -u https://psirt.domain.tld/advisories/csaf/feed-tlp-white.json --after 2024-10-01
+
 =head1 SEE ALSO
 
 L<csaf-rolie>

diff --git a/lib/App/CSAF/Downloader.pm b/lib/App/CSAF/Downloader.pm
@@ -25,8 +25,10 @@ sub run {
         \@args, \%options, qw(
             url|u=s
             directory|d=s
+            after=s
+            before=s
             insecure|k
-            verbose!
+            verbose|v
             validate:s
             integrity-check
             signature-check
@@ -39,7 +41,7 @@ sub run {
 
             help|h
             man
-            version|v
+            version
         )
     ) or pod2usage(-verbose => 0);
 
@@ -67,6 +69,8 @@ sub run {
     $downloader->options->include_pattern($options{include})                 if defined $options{include};
     $downloader->options->exclude_pattern($options{exclude})                 if defined $options{exclude};
     $downloader->options->parallel_downloads($options{'parallel-downloads'}) if defined $options{'parallel-downloads'};
+    $downloader->options->after_date($options{'after'})                      if defined $options{'after'};
+    $downloader->options->before_date($options{'before'})                    if defined $options{'before'};
 
     unless ($downloader->options->url) {
         cli_error("Specify URL");

diff --git a/lib/CSAF/Downloader.pm b/lib/CSAF/Downloader.pm
@@ -111,17 +111,23 @@ sub _mirror_via_rolie_feed {
 
     my $rolie = eval { Cpanel::JSON::XS->new->decode($res->content) };
 
+    my $after_date  = $self->options->after_date;
+    my $before_date = $self->options->before_date;
+
     my $idx = 0;
     my $pm  = Parallel::ForkManager->new($self->options->parallel_downloads);
 
 ENTRY:
     foreach my $entry (@{$rolie->{feed}->{entry}}) {
 
-        my $options  = {signature => 0, integrity => {sha256 => 0, sha512 => 0}};
-        my $csaf_url = undef;
+        my $options     = {signature => 0, integrity => {sha256 => 0, sha512 => 0}};
+        my $csaf_url    = undef;
+        my $last_update = undef;
 
         foreach my $link (@{$entry->{link}}) {
 
+            next unless defined $link->{rel};
+
             $options->{signature} = 1 if ($link->{rel} eq 'signature');
 
             $options->{integrity}->{sha256} = 1 if ($link->{rel} eq 'hash' & $link->{href} =~ /sha256/);
@@ -131,6 +137,23 @@ ENTRY:
 
         }
 
+        if (defined $entry->{updated}) {
+
+            $last_update = Time::Piece->strptime(substr($entry->{updated}, 0, 19), '%Y-%m-%dT%H:%M:%S');
+
+            my $skip = undef;
+
+            $skip = 1 if ($before_date && $before_date < $last_update);
+            $skip = 1 if ($after_date  && $after_date > $last_update);
+
+            if ($skip) {
+                $log->debug(sprintf("[#$idx] Skip Download CSAF document: $csaf_url (last updated: %s)",
+                    $last_update->datetime));
+                next;
+            }
+
+        }
+
         $idx++;
 
         $pm->start and next ENTRY;
@@ -168,6 +191,29 @@ sub _mirror_via_provider_metadata {
             $self->_mirror_via_index_txt($distribution->{directory_url});
         }
 
+        if (defined $distribution->{rolie} && defined $distribution->{rolie}->{feeds}) {
+
+            $log->debug("Use ROLIE feeds");
+
+            foreach my $feed (@{$distribution->{rolie}->{feeds}}) {
+
+                my $rolie_url = $feed->{url};
+
+                $log->debug("ROLIE URL $rolie_url");
+
+                my $rolie_file     = catfile($self->options->directory, URI::URL->new($rolie_url)->path);
+                my $rolie_base_dir = dirname($rolie_file);
+
+                make_path($rolie_base_dir) unless -e $rolie_base_dir;
+
+                $log->debug("Download: $rolie_url => $rolie_file");
+
+                $ua->mirror($rolie_url, $rolie_file);
+                $self->_mirror_via_rolie_feed($rolie_url);
+
+            }
+        }
+
     }
 
 }

diff --git a/lib/CSAF/Options/Downloader.pm b/lib/CSAF/Options/Downloader.pm
@@ -5,6 +5,8 @@ use strict;
 use warnings;
 use utf8;
 
+use CSAF::Util qw(parse_datetime);
+
 use Moo;
 with 'CSAF::Util::Options';
 
@@ -35,6 +37,18 @@ has timeout_after_download =>
 has url      => (is => 'rw');
 has insecure => (is => 'rw', default => FALSE);
 
+has after_date => (
+    is     => 'rw',
+    coerce => \&parse_datetime,
+    isa    => sub { Carp::croak 'MUST BE an instance of "Time::Piece"' unless ref($_[0]) eq 'Time::Piece' }
+);
+
+has before_date => (
+    is     => 'rw',
+    coerce => \&parse_datetime,
+    isa    => sub { Carp::croak 'MUST BE an instance of "Time::Piece"' unless ref($_[0]) eq 'Time::Piece' }
+);
+
 1;
 
 __END__