diff --git a/Gemfile b/Gemfile index 9c055b93bc0aef..54e616b1b9d04f 100644 --- a/Gemfile +++ b/Gemfile @@ -158,3 +158,5 @@ gem 'connection_pool', require: false gem 'xorcist', '~> 1.1' gem 'hcaptcha', '~> 7.1' +gem 'cocoon', '~> 1.2' +gem 'mail', '~> 2.8' diff --git a/Gemfile.lock b/Gemfile.lock index 28cceb6ebeac0f..43f7143a0dcc56 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -155,6 +155,7 @@ GEM cld3 (3.4.4) ffi (>= 1.1.0, < 1.16.0) climate_control (0.2.0) + cocoon (1.2.15) coderay (1.1.3) color_diff (0.1) concurrent-ruby (1.1.9) @@ -167,6 +168,7 @@ GEM crass (1.0.6) css_parser (1.7.1) addressable + date (3.3.4) debug_inspector (1.0.0) devise (4.8.1) bcrypt (~> 3.0) @@ -358,8 +360,11 @@ GEM loofah (2.13.0) crass (~> 1.0.2) nokogiri (>= 1.5.9) - mail (2.7.1) + mail (2.8.1) mini_mime (>= 0.1.1) + net-imap + net-pop + net-smtp makara (0.5.1) activerecord (>= 5.2.0) marcel (1.0.1) @@ -380,9 +385,18 @@ GEM msgpack (1.4.4) multi_json (1.15.0) multipart-post (2.1.1) + net-imap (0.4.14) + date + net-protocol net-ldap (0.17.0) + net-pop (0.1.2) + net-protocol + net-protocol (0.2.2) + timeout net-scp (3.0.0) net-ssh (>= 2.6.5, < 7.0.0) + net-smtp (0.5.0) + net-protocol net-ssh (6.1.0) nio4r (2.5.8) nokogiri (1.13.1) @@ -619,6 +633,7 @@ GEM thwait (0.2.0) e2mmap tilt (2.0.10) + timeout (0.4.1) tpm-key_attestation (0.9.0) bindata (~> 2.4) openssl-signature_algorithm (~> 0.4.0) @@ -703,6 +718,7 @@ DEPENDENCIES chewy (~> 7.2) cld3 (~> 3.4.4) climate_control (~> 0.2) + cocoon (~> 1.2) color_diff (~> 0.1) concurrent-ruby connection_pool @@ -738,6 +754,7 @@ DEPENDENCIES letter_opener_web (~> 2.0) link_header (~> 0.0) lograge (~> 0.11) + mail (~> 2.8) makara (~> 0.5) mario-redis-lock (~> 1.2) memory_profiler diff --git a/app/controllers/admin/domain_allows_controller.rb b/app/controllers/admin/domain_allows_controller.rb index 31be1978bbbb10..b0f139e3a8284e 100644 --- a/app/controllers/admin/domain_allows_controller.rb +++ b/app/controllers/admin/domain_allows_controller.rb @@ -25,6 +25,8 @@ def create def destroy authorize @domain_allow, :destroy? UnallowDomainService.new.call(@domain_allow) + log_action :destroy, @domain_allow + redirect_to admin_instances_path, notice: I18n.t('admin.domain_allows.destroyed_msg') end diff --git a/app/controllers/api/v1/featured_tags/suggestions_controller.rb b/app/controllers/api/v1/featured_tags/suggestions_controller.rb index 75545d3c7f7930..38c51fb31e0c16 100644 --- a/app/controllers/api/v1/featured_tags/suggestions_controller.rb +++ b/app/controllers/api/v1/featured_tags/suggestions_controller.rb @@ -12,6 +12,10 @@ def index private def set_recently_used_tags - @recently_used_tags = Tag.recently_used(current_account).where.not(id: current_account.featured_tags).limit(10) + @recently_used_tags = Tag.recently_used(current_account).where.not(id: featured_tag_ids).limit(10) + end + + def featured_tag_ids + current_account.featured_tags.pluck(:tag_id) end end diff --git a/app/controllers/oauth/authorized_applications_controller.rb b/app/controllers/oauth/authorized_applications_controller.rb index b2564a7915afc5..125b9beb7d29f1 100644 --- a/app/controllers/oauth/authorized_applications_controller.rb +++ b/app/controllers/oauth/authorized_applications_controller.rb @@ -15,6 +15,7 @@ class Oauth::AuthorizedApplicationsController < Doorkeeper::AuthorizedApplicatio def destroy Web::PushSubscription.unsubscribe_for(params[:id], current_resource_owner) + Doorkeeper::Application.find_by(id: params[:id])&.close_streaming_sessions(current_resource_owner) super end diff --git a/app/controllers/well_known/webfinger_controller.rb b/app/controllers/well_known/webfinger_controller.rb index 2b296ea3be46ad..6dae6a44382def 100644 --- a/app/controllers/well_known/webfinger_controller.rb +++ b/app/controllers/well_known/webfinger_controller.rb @@ -18,7 +18,14 @@ def show private def set_account - @account = Account.find_local!(username_from_resource) + username = username_from_resource + @account = begin + if username == Rails.configuration.x.local_domain || username == Rails.configuration.x.web_domain + Account.representative + else + Account.find_local!(username) + end + end end def username_from_resource diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index f61f1a06c9bb0b..2d8923cf215d81 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -104,7 +104,7 @@ def distribute def find_existing_status status = status_from_uri(object_uri) status ||= Status.find_by(uri: @object['atomUri']) if @object['atomUri'].present? - status + status if status&.account_id == @account.id end def process_status_params diff --git a/app/lib/application_extension.rb b/app/lib/application_extension.rb index d1222656b75e18..c7bd347cde1501 100644 --- a/app/lib/application_extension.rb +++ b/app/lib/application_extension.rb @@ -14,17 +14,19 @@ module ApplicationExtension # dependent: delete_all, which means the ActiveRecord callback in # AccessTokenExtension is not run, so instead we manually announce to # streaming that these tokens are being deleted. - before_destroy :push_to_streaming_api, prepend: true + before_destroy :close_streaming_sessions, prepend: true end def confirmation_redirect_uri redirect_uri.lines.first.strip end - def push_to_streaming_api + def close_streaming_sessions(resource_owner = nil) # TODO: #28793 Combine into a single topic payload = Oj.dump(event: :kill) - access_tokens.in_batches do |tokens| + scope = access_tokens + scope = scope.where(resource_owner_id: resource_owner.id) unless resource_owner.nil? + scope.in_batches do |tokens| redis.pipelined do |pipeline| tokens.ids.each do |id| pipeline.publish("timeline:access_token:#{id}", payload) diff --git a/app/lib/request.rb b/app/lib/request.rb index 60ad2e84709613..0e8b96a53fbb37 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -69,6 +69,7 @@ def initialize(verb, url, **options) @http_client = options.delete(:http_client) @options = options.merge(socket_class: use_proxy? ? ProxySocket : Socket) @options = @options.merge(Rails.configuration.x.http_client_proxy) if use_proxy? + @options = @options.merge(Rails.configuration.x.http_client_second_proxy) if use_second_proxy? @headers = {} raise Mastodon::HostValidationError, 'Instance does not support hidden service connections' if block_hidden_service? @@ -181,6 +182,10 @@ def use_proxy? Rails.configuration.x.http_client_proxy.present? end + def use_second_proxy? + Rails.configuration.x.http_client_second_proxy.present? && Rails.configuration.x.domain_to_use_second_proxy.include?(Addressable::URI.parse(@url).host) + end + def block_hidden_service? !Rails.configuration.x.access_to_hidden_service && /\.(onion|i2p)$/.match?(@url.host) end diff --git a/app/lib/search_query_transformer.rb b/app/lib/search_query_transformer.rb index e07ebfffed76d3..f7574403f88752 100644 --- a/app/lib/search_query_transformer.rb +++ b/app/lib/search_query_transformer.rb @@ -70,7 +70,7 @@ def initialize(prefix, operator, phrase) end rule(clause: subtree(:clause)) do - prefix = clause[:prefix][:term].to_s if clause[:prefix] + prefix = clause[:prefix][:term].to_s.downcase if clause[:prefix] operator = clause[:operator]&.to_s if clause[:term] diff --git a/app/lib/video_metadata_extractor.rb b/app/lib/video_metadata_extractor.rb index 03e40f923e5f0f..c33d3cf0f3a16c 100644 --- a/app/lib/video_metadata_extractor.rb +++ b/app/lib/video_metadata_extractor.rb @@ -22,7 +22,7 @@ def valid? private def ffmpeg_command_output - command = Terrapin::CommandLine.new('ffprobe', '-i :path -print_format :format -show_format -show_streams -show_error -loglevel :loglevel') + command = Terrapin::CommandLine.new(Rails.configuration.x.ffprobe_binary, '-i :path -print_format :format -show_format -show_streams -show_error -loglevel :loglevel') command.run(path: @path, format: 'json', loglevel: 'fatal') end diff --git a/app/models/concerns/account_interactions.rb b/app/models/concerns/account_interactions.rb index ad1665dc41e20b..d5cdd4bf722f2b 100644 --- a/app/models/concerns/account_interactions.rb +++ b/app/models/concerns/account_interactions.rb @@ -179,7 +179,7 @@ def unmute_conversation!(conversation) end def unblock_domain!(other_domain) - block = domain_blocks.find_by(domain: other_domain) + block = domain_blocks.find_by(domain: normalized_domain(other_domain)) block&.destroy end @@ -287,4 +287,8 @@ def remove_potential_friendship(other_account, mutual = false) PotentialFriendshipTracker.remove(id, other_account.id) PotentialFriendshipTracker.remove(other_account.id, id) if mutual end + + def normalized_domain(domain) + TagManager.instance.normalize_domain(domain) + end end diff --git a/app/models/status.rb b/app/models/status.rb index 02824ab65f04ec..156c621673fc10 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -176,7 +176,7 @@ def in_reply_to_local_account? end def reblog? - !reblog_of_id.nil? + !reblog_of_id.nil? && reblog end def within_realtime_window? diff --git a/app/models/user.rb b/app/models/user.rb index 4e30894187c944..f0b31c1879b9e5 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -91,7 +91,10 @@ class User < ApplicationRecord validates :invite_request, presence: true, on: :create, if: :invite_text_required? validates :locale, inclusion: I18n.available_locales.map(&:to_s), if: :locale? - validates_with BlacklistedEmailValidator, if: -> { !confirmed? } + + validates :email, presence: true, email_address: true + + validates_with BlacklistedEmailValidator, if: -> { ENV['EMAIL_DOMAIN_LISTS_APPLY_AFTER_CONFIRMATION'] == 'true' || !confirmed? } validates_with EmailMxValidator, if: :validate_email_dns? validates :agreement, acceptance: { allow_nil: false, accept: [true, 'true', '1'] }, on: :create diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb index 94dc6389fa2c7c..25e60468977f7b 100644 --- a/app/services/fetch_link_card_service.rb +++ b/app/services/fetch_link_card_service.rb @@ -12,6 +12,9 @@ class FetchLinkCardService < BaseService ) }iox + # URL size limit to safely store in PosgreSQL's unique indexes + BYTESIZE_LIMIT = 2692 + def call(status) @status = status @original_url = parse_urls @@ -86,7 +89,7 @@ def parse_urls def bad_url?(uri) # Avoid local instance URLs and invalid URLs - uri.host.blank? || TagManager.instance.local_url?(uri.to_s) || !%w(http https).include?(uri.scheme) + uri.host.blank? || TagManager.instance.local_url?(uri.to_s) || !%w(http https).include?(uri.scheme) || uri.to_s.bytesize > BYTESIZE_LIMIT end def mention_link?(anchor) diff --git a/app/services/notify_service.rb b/app/services/notify_service.rb index 7539a9a0a5fbab..e6a3a70e977000 100644 --- a/app/services/notify_service.rb +++ b/app/services/notify_service.rb @@ -76,47 +76,24 @@ def response_to_recipient? return false if @notification.target_status.in_reply_to_id.nil? # Using an SQL CTE to avoid unneeded back-and-forth with SQL server in case of long threads - !Status.count_by_sql([<<-SQL.squish, id: @notification.target_status.in_reply_to_id, recipient_id: @recipient.id, sender_id: @notification.from_account.id]).zero? - WITH RECURSIVE ancestors(id, in_reply_to_id, replying_to_sender, path) AS ( - SELECT - s.id, - s.in_reply_to_id, - (CASE - WHEN s.account_id = :recipient_id THEN - EXISTS ( - SELECT * - FROM mentions m - WHERE m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id - ) - ELSE - FALSE - END), - ARRAY[s.id] + !Status.count_by_sql([<<-SQL.squish, id: @notification.target_status.in_reply_to_id, recipient_id: @recipient.id, sender_id: @notification.from_account.id, depth_limit: 100]).zero? + WITH RECURSIVE ancestors(id, in_reply_to_id, mention_id, path, depth) AS ( + SELECT s.id, s.in_reply_to_id, m.id, ARRAY[s.id], 0 FROM statuses s + LEFT JOIN mentions m ON m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id WHERE s.id = :id UNION ALL - SELECT - s.id, - s.in_reply_to_id, - (CASE - WHEN s.account_id = :recipient_id THEN - EXISTS ( - SELECT * - FROM mentions m - WHERE m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id - ) - ELSE - FALSE - END), - st.path || s.id - FROM ancestors st - JOIN statuses s ON s.id = st.in_reply_to_id - WHERE st.replying_to_sender IS FALSE AND NOT s.id = ANY(path) + SELECT s.id, s.in_reply_to_id, m.id, ancestors.path || s.id, ancestors.depth + 1 + FROM ancestors + JOIN statuses s ON s.id = ancestors.in_reply_to_id + /* early exit if we already have a mention matching our requirements */ + LEFT JOIN mentions m ON m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id AND s.account_id = :recipient_id + WHERE ancestors.mention_id IS NULL AND NOT s.id = ANY(path) AND ancestors.depth < :depth_limit ) SELECT COUNT(*) - FROM ancestors st - JOIN statuses s ON s.id = st.id - WHERE st.replying_to_sender IS TRUE AND s.visibility = 3 + FROM ancestors + JOIN statuses s ON s.id = ancestors.id + WHERE ancestors.mention_id IS NOT NULL AND s.account_id = :recipient_id AND s.visibility = 3 SQL end diff --git a/app/validators/email_address_validator.rb b/app/validators/email_address_validator.rb new file mode 100644 index 00000000000000..ed0bb116524aec --- /dev/null +++ b/app/validators/email_address_validator.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# NOTE: I initially wrote this as `EmailValidator` but it ended up clashing +# with an indirect dependency of ours, `validate_email`, which, turns out, +# has the same approach as we do, but with an extra check disallowing +# single-label domains. Decided to not switch to `validate_email` because +# we do want to allow at least `localhost`. + +class EmailAddressValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + value = value.strip + + address = Mail::Address.new(value) + record.errors.add(attribute, :invalid) if address.address != value + rescue Mail::Field::FieldError + record.errors.add(attribute, :invalid) + end +end diff --git a/app/workers/mute_worker.rb b/app/workers/mute_worker.rb index c74f657cbacad8..50bfda9d2a53df 100644 --- a/app/workers/mute_worker.rb +++ b/app/workers/mute_worker.rb @@ -5,6 +5,7 @@ class MuteWorker def perform(account_id, target_account_id) FeedManager.instance.clear_from_home(Account.find(account_id), Account.find(target_account_id)) + FeedManager.instance.clear_from_lists(Account.find(account_id), Account.find(target_account_id)) rescue ActiveRecord::RecordNotFound true end diff --git a/config/application.rb b/config/application.rb index 734207908c3484..2222817f2ab5be 100644 --- a/config/application.rb +++ b/config/application.rb @@ -44,6 +44,7 @@ require_relative '../lib/webpacker/helper_extensions' require_relative '../lib/action_dispatch/cookie_jar_extensions' require_relative '../lib/rails/engine_extensions' +require_relative '../lib/action_dispatch/remote_ip_extensions' require_relative '../lib/active_record/database_tasks_extensions' require_relative '../lib/active_record/batches' diff --git a/config/initializers/ffmpeg.rb b/config/initializers/ffmpeg.rb index 4c0bf779d88992..87f85eeec70007 100644 --- a/config/initializers/ffmpeg.rb +++ b/config/initializers/ffmpeg.rb @@ -1,3 +1,6 @@ -if ENV['FFMPEG_BINARY'].present? - FFMPEG.ffmpeg_binary = ENV['FFMPEG_BINARY'] +# frozen_string_literal: true + +Rails.application.configure do + config.x.ffmpeg_binary = ENV['FFMPEG_BINARY'] || 'ffmpeg' + config.x.ffprobe_binary = ENV['FFPROBE_BINARY'] || 'ffprobe' end diff --git a/config/initializers/http_client_proxy.rb b/config/initializers/http_client_proxy.rb index 7a9b7b86d7d900..29a6032feed718 100644 --- a/config/initializers/http_client_proxy.rb +++ b/config/initializers/http_client_proxy.rb @@ -18,5 +18,24 @@ }.compact end + if ENV['SECOND_HTTP_PROXY'].present? && ENV['DOMAIN_TO_USE_SECOND_PROXY'].present? + proxy = URI.parse(ENV['SECOND_HTTP_PROXY']) + + raise "Unsupported proxy type: #{proxy.scheme}" unless %w(http https).include? proxy.scheme + raise "No proxy host" unless proxy.host + + host = proxy.host + host = host[1...-1] if host[0] == '[' # for IPv6 address + + config.x.http_client_second_proxy[:proxy] = { + proxy_address: host, + proxy_port: proxy.port, + proxy_username: proxy.user, + proxy_password: proxy.password, + }.compact + + config.x.domain_to_use_second_proxy = ENV['DOMAIN_TO_USE_SECOND_PROXY'].split(',') + end + config.x.access_to_hidden_service = ENV['ALLOW_ACCESS_TO_HIDDEN_SERVICE'] == 'true' end diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb index f4bd5d93999a81..425ed83a75d7fb 100644 --- a/config/initializers/rack_attack.rb +++ b/config/initializers/rack_attack.rb @@ -33,6 +33,14 @@ def authenticated_user_id authenticated_token&.resource_owner_id end + def authenticated_token_id + authenticated_token&.id + end + + def warden_user_id + @env['warden']&.user&.id + end + def unauthenticated? !authenticated_user_id end @@ -54,10 +62,6 @@ def paging_request? end end - Rack::Attack.safelist('allow from localhost') do |req| - req.remote_ip == '127.0.0.1' || req.remote_ip == '::1' - end - Rack::Attack.blocklist('deny from blocklist') do |req| IpBlock.blocked?(req.remote_ip) end @@ -129,6 +133,10 @@ def paging_request? req.session[:attempt_user_id] || req.params.dig('user', 'email').presence if req.post? && req.path_matches?('/auth/sign_in') end + throttle('throttle_password_change/account', limit: 10, period: 10.minutes) do |req| + req.warden_user_id if (req.put? || req.patch?) && req.path_matches?('/auth') + end + self.throttled_response = lambda do |env| now = Time.now.utc match_data = env['rack.attack.match_data'] diff --git a/config/initializers/statsd.rb b/config/initializers/statsd.rb index 93ea1d1e4a7d1f..f1628a9d12ca32 100644 --- a/config/initializers/statsd.rb +++ b/config/initializers/statsd.rb @@ -3,13 +3,17 @@ if ENV['STATSD_ADDR'].present? host, port = ENV['STATSD_ADDR'].split(':') - $statsd = ::Statsd.new(host, port) - $statsd.namespace = ENV.fetch('STATSD_NAMESPACE') { ['Mastodon', Rails.env].join('.') } + begin + statsd = Statsd.new(host, port) + statsd.namespace = ENV.fetch('STATSD_NAMESPACE') { ['Mastodon', Rails.env].join('.') } - ::NSA.inform_statsd($statsd) do |informant| - informant.collect(:action_controller, :web) - informant.collect(:active_record, :db) - informant.collect(:active_support_cache, :cache) - informant.collect(:sidekiq, :sidekiq) + NSA.inform_statsd(statsd) do |informant| + informant.collect(:action_controller, :web) + informant.collect(:active_record, :db) + informant.collect(:active_support_cache, :cache) + informant.collect(:sidekiq, :sidekiq) if ENV['STATSD_SIDEKIQ'] == 'true' + end + rescue + Rails.logger.warn("statsd address #{ENV['STATSD_ADDR']} not reachable, proceeding without statsd") end end diff --git a/lib/action_dispatch/remote_ip_extensions.rb b/lib/action_dispatch/remote_ip_extensions.rb new file mode 100644 index 00000000000000..e5c48bf3c5b0ee --- /dev/null +++ b/lib/action_dispatch/remote_ip_extensions.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +# Mastodon is not made to be directly accessed without a reverse proxy. +# This monkey-patch prevents remote IP address spoofing when being accessed +# directly. +# +# See PR: https://github.com/rails/rails/pull/51610 + +# In addition to the PR above, it also raises an error if a request with +# `X-Forwarded-For` or `Client-Ip` comes directly from a client without +# going through a trusted proxy. + +# rubocop:disable all -- This is a mostly vendored file + +module ActionDispatch + class RemoteIp + module GetIpExtensions + def calculate_ip + # Set by the Rack web server, this is a single value. + remote_addr = ips_from(@req.remote_addr).last + + # Could be a CSV list and/or repeated headers that were concatenated. + client_ips = ips_from(@req.client_ip).reverse! + forwarded_ips = ips_from(@req.x_forwarded_for).reverse! + + # `Client-Ip` and `X-Forwarded-For` should not, generally, both be set. If they + # are both set, it means that either: + # + # 1) This request passed through two proxies with incompatible IP header + # conventions. + # + # 2) The client passed one of `Client-Ip` or `X-Forwarded-For` + # (whichever the proxy servers weren't using) themselves. + # + # Either way, there is no way for us to determine which header is the right one + # after the fact. Since we have no idea, if we are concerned about IP spoofing + # we need to give up and explode. (If you're not concerned about IP spoofing you + # can turn the `ip_spoofing_check` option off.) + should_check_ip = @check_ip && client_ips.last && forwarded_ips.last + if should_check_ip && !forwarded_ips.include?(client_ips.last) + # We don't know which came from the proxy, and which from the user + raise IpSpoofAttackError, "IP spoofing attack?! " \ + "HTTP_CLIENT_IP=#{@req.client_ip.inspect} " \ + "HTTP_X_FORWARDED_FOR=#{@req.x_forwarded_for.inspect}" + end + + # NOTE: Mastodon addition to make sure we don't get requests from a non-trusted client + if @check_ip && (forwarded_ips.last || client_ips.last) && !@proxies.any? { |proxy| proxy === remote_addr } + raise IpSpoofAttackError, "IP spoofing attack?! client #{remote_addr} is not a trusted proxy " \ + "HTTP_CLIENT_IP=#{@req.client_ip.inspect} " \ + "HTTP_X_FORWARDED_FOR=#{@req.x_forwarded_for.inspect}" + end + + # We assume these things about the IP headers: + # + # - X-Forwarded-For will be a list of IPs, one per proxy, or blank + # - Client-Ip is propagated from the outermost proxy, or is blank + # - REMOTE_ADDR will be the IP that made the request to Rack + ips = forwarded_ips + client_ips + ips.compact! + + # If every single IP option is in the trusted list, return the IP that's + # furthest away + filter_proxies([remote_addr] + ips).first || ips.last || remote_addr + end + end + end +end + +ActionDispatch::RemoteIp::GetIp.prepend(ActionDispatch::RemoteIp::GetIpExtensions) + +# rubocop:enable all diff --git a/lib/mastodon/maintenance_cli.rb b/lib/mastodon/maintenance_cli.rb index 00861df7743475..76901bb31a878c 100644 --- a/lib/mastodon/maintenance_cli.rb +++ b/lib/mastodon/maintenance_cli.rb @@ -224,7 +224,7 @@ def deduplicate_users! users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse ref_user = users.shift @prompt.warn "Multiple users registered with e-mail address #{ref_user.email}." - @prompt.warn "e-mail will be disabled for the following accounts: #{user.map(&:account).map(&:acct).join(', ')}" + @prompt.warn "e-mail will be disabled for the following accounts: #{users.map(&:account).map(&:acct).join(', ')}" @prompt.warn 'Please reach out to them and set another address with `tootctl account modify` or delete them.' i = 0 diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index 36ca71844f0dcd..8dc5c2ddf10687 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -100,7 +100,7 @@ def remove_orphans model_name = path_segments.first.classify attachment_name = path_segments[1].singularize - record_id = path_segments[2..-2].join.to_i + record_id = path_segments[2...-2].join.to_i file_name = path_segments.last record = record_map.dig(model_name, record_id) attachment = record&.public_send(attachment_name) @@ -143,7 +143,7 @@ def remove_orphans end model_name = path_segments.first.classify - record_id = path_segments[2..-2].join.to_i + record_id = path_segments[2...-2].join.to_i attachment_name = path_segments[1].singularize file_name = path_segments.last @@ -269,7 +269,7 @@ def lookup(url) end model_name = path_segments.first.classify - record_id = path_segments[2..-2].join.to_i + record_id = path_segments[2...-2].join.to_i unless PRELOAD_MODEL_WHITELIST.include?(model_name) say("Cannot find corresponding model: #{model_name}", :red) @@ -319,7 +319,7 @@ def preload_records_from_mixed_objects(objects) next unless [7, 10].include?(segments.size) model_name = segments.first.classify - record_id = segments[2..-2].join.to_i + record_id = segments[2...-2].join.to_i next unless PRELOAD_MODEL_WHITELIST.include?(model_name) diff --git a/lib/paperclip/image_extractor.rb b/lib/paperclip/image_extractor.rb index 17fe4326fdbf08..8a565d0469e1f5 100644 --- a/lib/paperclip/image_extractor.rb +++ b/lib/paperclip/image_extractor.rb @@ -35,7 +35,7 @@ def extract_image_from_file! dst.binmode begin - command = Terrapin::CommandLine.new('ffmpeg', '-i :source -loglevel :loglevel -y :destination', logger: Paperclip.logger) + command = Terrapin::CommandLine.new(Rails.configuration.x.ffmpeg_binary, '-i :source -loglevel :loglevel -y :destination', logger: Paperclip.logger) command.run(source: @file.path, destination: dst.path, loglevel: 'fatal') rescue Terrapin::ExitStatusError dst.close(true) diff --git a/lib/paperclip/transcoder.rb b/lib/paperclip/transcoder.rb index d060ab57f6cb11..cbce381378b7a5 100644 --- a/lib/paperclip/transcoder.rb +++ b/lib/paperclip/transcoder.rb @@ -43,7 +43,7 @@ def make command_arguments, interpolations = prepare_command(destination) begin - command = Terrapin::CommandLine.new('ffmpeg', command_arguments.join(' '), logger: Paperclip.logger) + command = Terrapin::CommandLine.new(Rails.configuration.x.ffmpeg_binary, command_arguments.join(' '), logger: Paperclip.logger) command.run(interpolations) rescue Terrapin::ExitStatusError => e raise Paperclip::Error, "Error while transcoding #{@basename}: #{e}" diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index a89af677807439..601526aba6787a 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -427,6 +427,7 @@ namespace :mastodon do user = User.new(admin: true, email: email, password: password, confirmed_at: Time.now.utc, account_attributes: { username: username }, bypass_invite_request_check: true) user.save(validate: false) + user.approve! prompt.ok "You can login with the password: #{password}" prompt.warn 'You can change your password once you login.' diff --git a/spec/config/initializers/rack_attack_spec.rb b/spec/config/initializers/rack_attack_spec.rb new file mode 100644 index 00000000000000..50e5d5052a44c1 --- /dev/null +++ b/spec/config/initializers/rack_attack_spec.rb @@ -0,0 +1,104 @@ +require 'rails_helper' + +describe Rack::Attack, type: :request do + def app + Rails.application + end + + shared_examples 'throttled endpoint' do + context 'when the number of requests is lower than the limit' do + it 'does not change the request status' do + limit.times do + request.call + expect(response.status).to_not eq(429) + end + end + end + + context 'when the number of requests is higher than the limit' do + it 'returns http too many requests' do + (limit * 2).times do |i| + request.call + expect(response.status).to eq(429) if i > limit + end + end + end + end + + let(:remote_ip) { '1.2.3.5' } + + describe 'throttle excessive sign-up requests by IP address' do + context 'through the website' do + let(:limit) { 25 } + let(:request) { ->() { post path, headers: { 'REMOTE_ADDR' => remote_ip } } } + + context 'for exact path' do + let(:path) { '/auth' } + it_behaves_like 'throttled endpoint' + end + + context 'for path with format' do + let(:path) { '/auth.html' } + it_behaves_like 'throttled endpoint' + end + end + + context 'through the API' do + let(:limit) { 5 } + let(:request) { ->() { post path, headers: { 'REMOTE_ADDR' => remote_ip } } } + + context 'for exact path' do + let(:path) { '/api/v1/accounts' } + it_behaves_like 'throttled endpoint' + end + + context 'for path with format' do + let(:path) { '/api/v1/accounts.json' } + + it 'returns http not found' do + request.call + expect(response.status).to eq(404) + end + end + end + end + + describe 'throttle excessive sign-in requests by IP address' do + let(:limit) { 25 } + let(:request) { ->() { post path, headers: { 'REMOTE_ADDR' => remote_ip } } } + + context 'for exact path' do + let(:path) { '/auth/sign_in' } + it_behaves_like 'throttled endpoint' + end + + context 'for path with format' do + let(:path) { '/auth/sign_in.html' } + it_behaves_like 'throttled endpoint' + end + end + + describe 'throttle excessive password change requests by account' do + let(:user) { Fabricate(:user, email: 'user@host.example') } + let(:limit) { 10 } + let(:period) { 10.minutes } + let(:request) { -> { put path, headers: { 'REMOTE_ADDR' => remote_ip } } } + let(:path) { '/auth' } + + before do + sign_in user, scope: :user + + # Unfortunately, devise's `sign_in` helper causes the `session` to be + # loaded in the next request regardless of whether it's actually accessed + # by the client code. + # + # So, we make an extra query to clear issue a session cookie instead. + # + # A less resource-intensive way to deal with that would be to generate the + # session cookie manually, but this seems pretty involved. + get '/' + end + + it_behaves_like 'throttled endpoint' + end +end diff --git a/spec/controllers/api/v1/featured_tags/suggestions_controller_spec.rb b/spec/controllers/api/v1/featured_tags/suggestions_controller_spec.rb new file mode 100644 index 00000000000000..8cb928ea2c5e3e --- /dev/null +++ b/spec/controllers/api/v1/featured_tags/suggestions_controller_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe Api::V1::FeaturedTags::SuggestionsController do + render_views + + let(:user) { Fabricate(:user) } + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:accounts') } + let(:account) { Fabricate(:account, user: user) } + + before do + allow(controller).to receive(:doorkeeper_token) { token } + end + + describe 'GET #index' do + let!(:unused_featured_tag) { Fabricate(:tag, name: 'unused_featured_tag') } + let!(:used_tag) { Fabricate(:tag, name: 'used_tag') } + let!(:used_featured_tag) { Fabricate(:tag, name: 'used_featured_tag') } + + before do + _unused_tag = Fabricate(:tag, name: 'unused_tag') + + # Make relevant tags used by account + status = Fabricate(:status, account: account) + status.tags << used_tag + status.tags << used_featured_tag + + # Feature the relevant tags + Fabricate :featured_tag, account: account, name: unused_featured_tag.name + Fabricate :featured_tag, account: account, name: used_featured_tag.name + end + + it 'returns http success and recently used but not featured tags', :aggregate_failures do + get :index, params: { account_id: account.id, limit: 2 } + + expect(response) + .to have_http_status(200) + expect(body_as_json) + .to contain_exactly( + include(name: used_tag.name) + ) + end + end +end diff --git a/spec/controllers/oauth/authorized_applications_controller_spec.rb b/spec/controllers/oauth/authorized_applications_controller_spec.rb index 901e538e95cb68..7689edb01d8e05 100644 --- a/spec/controllers/oauth/authorized_applications_controller_spec.rb +++ b/spec/controllers/oauth/authorized_applications_controller_spec.rb @@ -45,9 +45,11 @@ let!(:application) { Fabricate(:application) } let!(:access_token) { Fabricate(:accessible_access_token, application: application, resource_owner_id: user.id) } let!(:web_push_subscription) { Fabricate(:web_push_subscription, user: user, access_token: access_token) } + let(:redis_pipeline_stub) { instance_double(Redis::Namespace, publish: nil) } before do sign_in user, scope: :user + allow(redis).to receive(:pipelined).and_yield(redis_pipeline_stub) post :destroy, params: { id: application.id } end @@ -58,5 +60,13 @@ it 'removes subscriptions for the application\'s access tokens' do expect(Web::PushSubscription.where(user: user).count).to eq 0 end + + it 'removes the web_push_subscription' do + expect { web_push_subscription.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + + it 'sends a session kill payload to the streaming server' do + expect(redis_pipeline_stub).to have_received(:publish).with("timeline:access_token:#{access_token.id}", '{"event":"kill"}') + end end end diff --git a/spec/controllers/settings/applications_controller_spec.rb b/spec/controllers/settings/applications_controller_spec.rb index 29c278148463f8..b49e849edbf04e 100644 --- a/spec/controllers/settings/applications_controller_spec.rb +++ b/spec/controllers/settings/applications_controller_spec.rb @@ -160,7 +160,11 @@ def call_update end describe 'destroy' do + let(:redis_pipeline_stub) { instance_double(Redis::Namespace, publish: nil) } + let!(:access_token) { Fabricate(:accessible_access_token, application: app) } + before do + allow(redis).to receive(:pipelined).and_yield(redis_pipeline_stub) post :destroy, params: { id: app.id } end @@ -171,6 +175,10 @@ def call_update it 'removes the app' do expect(Doorkeeper::Application.find_by(id: app.id)).to be_nil end + + it 'sends a session kill payload to the streaming server' do + expect(redis_pipeline_stub).to have_received(:publish).with("timeline:access_token:#{access_token.id}", '{"event":"kill"}') + end end describe 'regenerate' do diff --git a/spec/models/concerns/account_interactions_spec.rb b/spec/models/concerns/account_interactions_spec.rb index 656dd66ccfdc19..f0b8388bfe88ef 100644 --- a/spec/models/concerns/account_interactions_spec.rb +++ b/spec/models/concerns/account_interactions_spec.rb @@ -255,6 +255,24 @@ end end + describe '#block_idna_domain!' do + subject do + [ + account.block_domain!(idna_domain), + account.block_domain!(punycode_domain), + ] + end + + let(:idna_domain) { '대한민국.한국' } + let(:punycode_domain) { 'xn--3e0bs9hfvinn1a.xn--3e0b707e' } + + it 'creates single AccountDomainBlock' do + expect do + expect(subject).to all(be_a AccountDomainBlock) + end.to change { account.domain_blocks.count }.by 1 + end + end + describe '#unfollow!' do subject { account.unfollow!(target_account) } @@ -350,6 +368,28 @@ end end + describe '#unblock_idna_domain!' do + subject { account.unblock_domain!(punycode_domain) } + + let(:idna_domain) { '대한민국.한국' } + let(:punycode_domain) { 'xn--3e0bs9hfvinn1a.xn--3e0b707e' } + + context 'when blocking the domain' do + it 'returns destroyed AccountDomainBlock' do + account_domain_block = Fabricate(:account_domain_block, domain: idna_domain) + account.domain_blocks << account_domain_block + expect(subject).to be_a AccountDomainBlock + expect(subject).to be_destroyed + end + end + + context 'when unblocking idna domain' do + it 'returns nil' do + expect(subject).to be_nil + end + end + end + describe '#following?' do subject { account.following?(target_account) } diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index 8a90dc64764112..b0102ba07782d4 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -40,6 +40,12 @@ expect(user.valid?).to be true end + it 'is valid with a localhost e-mail address' do + user = Fabricate.build(:user, email: 'admin@localhost') + user.valid? + expect(user.valid?).to be true + end + it 'cleans out empty string from languages' do user = Fabricate.build(:user, chosen_languages: ['']) user.valid? diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index 86c2a9c52fd9e8..2f3e4ac4ff70d3 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -42,6 +42,7 @@ def sign_in(resource, _deprecated = nil, scope: nil) config.include Devise::Test::ControllerHelpers, type: :controller config.include Devise::Test::ControllerHelpers, type: :view + config.include Devise::Test::IntegrationHelpers, type: :request config.include Paperclip::Shoulda::Matchers config.include ActiveSupport::Testing::TimeHelpers diff --git a/spec/services/fetch_link_card_service_spec.rb b/spec/services/fetch_link_card_service_spec.rb index 4914c275326eb9..4faa0f15ed011d 100644 --- a/spec/services/fetch_link_card_service_spec.rb +++ b/spec/services/fetch_link_card_service_spec.rb @@ -85,6 +85,19 @@ expect(a_request(:get, 'http://example.com/sjis')).to_not have_been_made end end + + context 'with an URL too long for PostgreSQL unique indexes' do + let(:url) { "http://example.com/#{'a' * 2674}" } + let(:status) { Fabricate(:status, text: url) } + + it 'does not fetch the URL' do + expect(a_request(:get, url)).to_not have_been_made + end + + it 'does not create a preview card' do + expect(status.preview_card).to be_nil + end + end end context 'in a remote status' do diff --git a/spec/services/notify_service_spec.rb b/spec/services/notify_service_spec.rb index b38c096d2ddc44..7c7dc5b945242b 100644 --- a/spec/services/notify_service_spec.rb +++ b/spec/services/notify_service_spec.rb @@ -48,7 +48,7 @@ recipient.suspend! is_expected.to_not change(Notification, :count) end - + context 'for direct messages' do let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct)) } let(:type) { :mention } @@ -74,11 +74,11 @@ end end - context 'if the message chain is initiated by recipient, but without a mention to the sender, even if the sender sends multiple messages in a row' do - let(:reply_to) { Fabricate(:status, account: recipient) } - let!(:mention) { Fabricate(:mention, account: sender, status: reply_to) } - let(:dummy_reply) { Fabricate(:status, account: sender, visibility: :direct, thread: reply_to) } - let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: dummy_reply)) } + context 'when the message chain is initiated by recipient, but without a mention to the sender, even if the sender sends multiple messages in a row' do + let(:public_status) { Fabricate(:status, account: recipient) } + let(:intermediate_reply) { Fabricate(:status, account: sender, thread: public_status, visibility: :direct) } + let!(:intermediate_mention) { Fabricate(:mention, account: sender, status: intermediate_reply) } + let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: intermediate_reply)) } it 'does not notify' do is_expected.to_not change(Notification, :count)