Skip to content

Commit

Permalink
Add button to set scrapers back to some timestamp
Browse files Browse the repository at this point in the history
When scrapers break and report success, like in twitters case, see #108
  • Loading branch information
Earlopain committed Dec 19, 2023
1 parent 230e5af commit de7a704
Show file tree
Hide file tree
Showing 4 changed files with 25 additions and 1 deletion.
5 changes: 5 additions & 0 deletions app/controllers/submission_files_controller.rb
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,11 @@ def modify_hidden
submission_file.update(hidden_from_search_at: hide_from_search ? Time.current : nil)
end

def set_last_known_good
submission_file = SubmissionFile.find(params[:id])
submission_file.artist_url.update(last_scraped_at: submission_file.created_at_on_site - 1.day)
end

def update_e6_posts
submission_file = SubmissionFile.find(params[:id])
submission_file.update_e6_posts(priority: E6IqdbQueryJob::PRIORITIES[:immediate])
Expand Down
6 changes: 5 additions & 1 deletion app/views/submission_files/_secondary_links.html.erb
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,14 @@
<% if instance_variable_defined?(:@search_params) %>
<%= subnav_link_to "Update IQDB for matching", update_matching_e6_posts_submission_files_path(search: @search_params), method: :post, data: { confirm: "Are you sure?" } %>
<% end %>
<% if @submission_file&.artist %>
<% if @submission_file&.id %>
|
<%= subnav_link_to "All From Artist", artist_path(@submission_file.artist) %>
<%= subnav_link_to "All From URL", artist_path(@submission_file.artist, search: { artist_url_id: [@submission_file.artist_url.id] }) %>
<%= subnav_link_to "Update E6 IQDB", update_e6_posts_submission_file_path(@submission_file), method: :post %>
<% if @submission_file.artist_url.last_scraped_at %>
|
<%= subnav_link_to "Last known good", set_last_known_good_submission_file_path(@submission_file), method: :put, data: { confirm: "Fix broken scrapers. Everything after this may be downloaded again if it failed in the past but is now fixed through an update. Will not cause duplicate entries." } %>
<% end %>
<% end %>
| <%= render "submission_files/multiselect" %>
1 change: 1 addition & 0 deletions config/routes.rb
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
post :update_e6_posts
put :modify_backlog
put :modify_hidden
put :set_last_known_good
end
collection do
get :backlog
Expand Down
14 changes: 14 additions & 0 deletions test/controllers/submission_files_controller_test.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# frozen_string_literal: true

require "test_helper"

class SubmissionFilesControllerTest < ActionDispatch::IntegrationTest
test "last known good" do
artist_url = create(:artist_url, last_scraped_at: Time.current)
sm = create(:submission_file, artist_submission: create(:artist_submission, artist_url: artist_url), created_at_on_site: 7.days.ago)

put set_last_known_good_submission_file_path(sm)
assert_response :success
assert_in_delta artist_url.reload.last_scraped_at, 8.days.ago, 1
end
end

0 comments on commit de7a704

Please sign in to comment.