Validate HTTP response length while receiving (#6891)
to_s method of HTTP::Response keeps blocking while it receives the whole content, no matter how it is big. This means it may waste time to receive unacceptably large files. It may also consume memory and disk in the process. This solves the inefficency by checking response length while receiving.remotes/1727458204337373841/tmp_refs/heads/signup-info-prompt
parent
18965cb0e6
commit
40e5d2303b
|
@ -61,7 +61,7 @@ module JsonLdHelper
|
||||||
|
|
||||||
def fetch_resource_without_id_validation(uri)
|
def fetch_resource_without_id_validation(uri)
|
||||||
build_request(uri).perform do |response|
|
build_request(uri).perform do |response|
|
||||||
response.code == 200 ? body_to_json(response.to_s) : nil
|
response.code == 200 ? body_to_json(response.body_with_limit) : nil
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
@ -5,6 +5,7 @@ module Mastodon
|
||||||
class NotPermittedError < Error; end
|
class NotPermittedError < Error; end
|
||||||
class ValidationError < Error; end
|
class ValidationError < Error; end
|
||||||
class HostValidationError < ValidationError; end
|
class HostValidationError < ValidationError; end
|
||||||
|
class LengthValidationError < ValidationError; end
|
||||||
class RaceConditionError < Error; end
|
class RaceConditionError < Error; end
|
||||||
|
|
||||||
class UnexpectedResponseError < Error
|
class UnexpectedResponseError < Error
|
||||||
|
|
|
@ -18,7 +18,7 @@ class ProviderDiscovery < OEmbed::ProviderDiscovery
|
||||||
else
|
else
|
||||||
Request.new(:get, url).perform do |res|
|
Request.new(:get, url).perform do |res|
|
||||||
raise OEmbed::NotFound, url if res.code != 200 || res.mime_type != 'text/html'
|
raise OEmbed::NotFound, url if res.code != 200 || res.mime_type != 'text/html'
|
||||||
Nokogiri::HTML(res.to_s)
|
Nokogiri::HTML(res.body_with_limit)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
@ -40,7 +40,7 @@ class Request
|
||||||
end
|
end
|
||||||
|
|
||||||
begin
|
begin
|
||||||
yield response
|
yield response.extend(ClientLimit)
|
||||||
ensure
|
ensure
|
||||||
http_client.close
|
http_client.close
|
||||||
end
|
end
|
||||||
|
@ -99,6 +99,33 @@ class Request
|
||||||
@http_client ||= HTTP.timeout(:per_operation, timeout).follow(max_hops: 2)
|
@http_client ||= HTTP.timeout(:per_operation, timeout).follow(max_hops: 2)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
module ClientLimit
|
||||||
|
def body_with_limit(limit = 1.megabyte)
|
||||||
|
raise Mastodon::LengthValidationError if content_length.present? && content_length > limit
|
||||||
|
|
||||||
|
if charset.nil?
|
||||||
|
encoding = Encoding::BINARY
|
||||||
|
else
|
||||||
|
begin
|
||||||
|
encoding = Encoding.find(charset)
|
||||||
|
rescue ArgumentError
|
||||||
|
encoding = Encoding::BINARY
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
contents = String.new(encoding: encoding)
|
||||||
|
|
||||||
|
while (chunk = readpartial)
|
||||||
|
contents << chunk
|
||||||
|
chunk.clear
|
||||||
|
|
||||||
|
raise Mastodon::LengthValidationError if contents.bytesize > limit
|
||||||
|
end
|
||||||
|
|
||||||
|
contents
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
class Socket < TCPSocket
|
class Socket < TCPSocket
|
||||||
class << self
|
class << self
|
||||||
def open(host, *args)
|
def open(host, *args)
|
||||||
|
@ -118,5 +145,5 @@ class Request
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
private_constant :Socket
|
private_constant :ClientLimit, :Socket
|
||||||
end
|
end
|
||||||
|
|
|
@ -55,7 +55,6 @@ class Account < ApplicationRecord
|
||||||
include AccountHeader
|
include AccountHeader
|
||||||
include AccountInteractions
|
include AccountInteractions
|
||||||
include Attachmentable
|
include Attachmentable
|
||||||
include Remotable
|
|
||||||
include Paginable
|
include Paginable
|
||||||
|
|
||||||
enum protocol: [:ostatus, :activitypub]
|
enum protocol: [:ostatus, :activitypub]
|
||||||
|
|
|
@ -2,4 +2,5 @@
|
||||||
|
|
||||||
class ApplicationRecord < ActiveRecord::Base
|
class ApplicationRecord < ActiveRecord::Base
|
||||||
self.abstract_class = true
|
self.abstract_class = true
|
||||||
|
include Remotable
|
||||||
end
|
end
|
||||||
|
|
|
@ -4,6 +4,7 @@ module AccountAvatar
|
||||||
extend ActiveSupport::Concern
|
extend ActiveSupport::Concern
|
||||||
|
|
||||||
IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze
|
IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze
|
||||||
|
LIMIT = 2.megabytes
|
||||||
|
|
||||||
class_methods do
|
class_methods do
|
||||||
def avatar_styles(file)
|
def avatar_styles(file)
|
||||||
|
@ -19,7 +20,8 @@ module AccountAvatar
|
||||||
# Avatar upload
|
# Avatar upload
|
||||||
has_attached_file :avatar, styles: ->(f) { avatar_styles(f) }, convert_options: { all: '-strip' }, processors: [:lazy_thumbnail]
|
has_attached_file :avatar, styles: ->(f) { avatar_styles(f) }, convert_options: { all: '-strip' }, processors: [:lazy_thumbnail]
|
||||||
validates_attachment_content_type :avatar, content_type: IMAGE_MIME_TYPES
|
validates_attachment_content_type :avatar, content_type: IMAGE_MIME_TYPES
|
||||||
validates_attachment_size :avatar, less_than: 2.megabytes
|
validates_attachment_size :avatar, less_than: LIMIT
|
||||||
|
remotable_attachment :avatar, LIMIT
|
||||||
end
|
end
|
||||||
|
|
||||||
def avatar_original_url
|
def avatar_original_url
|
||||||
|
|
|
@ -4,6 +4,7 @@ module AccountHeader
|
||||||
extend ActiveSupport::Concern
|
extend ActiveSupport::Concern
|
||||||
|
|
||||||
IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze
|
IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze
|
||||||
|
LIMIT = 2.megabytes
|
||||||
|
|
||||||
class_methods do
|
class_methods do
|
||||||
def header_styles(file)
|
def header_styles(file)
|
||||||
|
@ -19,7 +20,8 @@ module AccountHeader
|
||||||
# Header upload
|
# Header upload
|
||||||
has_attached_file :header, styles: ->(f) { header_styles(f) }, convert_options: { all: '-strip' }, processors: [:lazy_thumbnail]
|
has_attached_file :header, styles: ->(f) { header_styles(f) }, convert_options: { all: '-strip' }, processors: [:lazy_thumbnail]
|
||||||
validates_attachment_content_type :header, content_type: IMAGE_MIME_TYPES
|
validates_attachment_content_type :header, content_type: IMAGE_MIME_TYPES
|
||||||
validates_attachment_size :header, less_than: 2.megabytes
|
validates_attachment_size :header, less_than: LIMIT
|
||||||
|
remotable_attachment :header, LIMIT
|
||||||
end
|
end
|
||||||
|
|
||||||
def header_original_url
|
def header_original_url
|
||||||
|
|
|
@ -3,8 +3,8 @@
|
||||||
module Remotable
|
module Remotable
|
||||||
extend ActiveSupport::Concern
|
extend ActiveSupport::Concern
|
||||||
|
|
||||||
included do
|
class_methods do
|
||||||
attachment_definitions.each_key do |attachment_name|
|
def remotable_attachment(attachment_name, limit)
|
||||||
attribute_name = "#{attachment_name}_remote_url".to_sym
|
attribute_name = "#{attachment_name}_remote_url".to_sym
|
||||||
method_name = "#{attribute_name}=".to_sym
|
method_name = "#{attribute_name}=".to_sym
|
||||||
alt_method_name = "reset_#{attachment_name}!".to_sym
|
alt_method_name = "reset_#{attachment_name}!".to_sym
|
||||||
|
@ -33,7 +33,7 @@ module Remotable
|
||||||
File.extname(filename)
|
File.extname(filename)
|
||||||
end
|
end
|
||||||
|
|
||||||
send("#{attachment_name}=", StringIO.new(response.to_s))
|
send("#{attachment_name}=", StringIO.new(response.body_with_limit(limit)))
|
||||||
send("#{attachment_name}_file_name=", basename + extname)
|
send("#{attachment_name}_file_name=", basename + extname)
|
||||||
|
|
||||||
self[attribute_name] = url if has_attribute?(attribute_name)
|
self[attribute_name] = url if has_attribute?(attribute_name)
|
||||||
|
|
|
@ -19,6 +19,8 @@
|
||||||
#
|
#
|
||||||
|
|
||||||
class CustomEmoji < ApplicationRecord
|
class CustomEmoji < ApplicationRecord
|
||||||
|
LIMIT = 50.kilobytes
|
||||||
|
|
||||||
SHORTCODE_RE_FRAGMENT = '[a-zA-Z0-9_]{2,}'
|
SHORTCODE_RE_FRAGMENT = '[a-zA-Z0-9_]{2,}'
|
||||||
|
|
||||||
SCAN_RE = /(?<=[^[:alnum:]:]|\n|^)
|
SCAN_RE = /(?<=[^[:alnum:]:]|\n|^)
|
||||||
|
@ -29,14 +31,14 @@ class CustomEmoji < ApplicationRecord
|
||||||
|
|
||||||
has_attached_file :image, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } }
|
has_attached_file :image, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } }
|
||||||
|
|
||||||
validates_attachment :image, content_type: { content_type: 'image/png' }, presence: true, size: { in: 0..50.kilobytes }
|
validates_attachment :image, content_type: { content_type: 'image/png' }, presence: true, size: { less_than: LIMIT }
|
||||||
validates :shortcode, uniqueness: { scope: :domain }, format: { with: /\A#{SHORTCODE_RE_FRAGMENT}\z/ }, length: { minimum: 2 }
|
validates :shortcode, uniqueness: { scope: :domain }, format: { with: /\A#{SHORTCODE_RE_FRAGMENT}\z/ }, length: { minimum: 2 }
|
||||||
|
|
||||||
scope :local, -> { where(domain: nil) }
|
scope :local, -> { where(domain: nil) }
|
||||||
scope :remote, -> { where.not(domain: nil) }
|
scope :remote, -> { where.not(domain: nil) }
|
||||||
scope :alphabetic, -> { order(domain: :asc, shortcode: :asc) }
|
scope :alphabetic, -> { order(domain: :asc, shortcode: :asc) }
|
||||||
|
|
||||||
include Remotable
|
remotable_attachment :image, LIMIT
|
||||||
|
|
||||||
def local?
|
def local?
|
||||||
domain.nil?
|
domain.nil?
|
||||||
|
|
|
@ -56,6 +56,8 @@ class MediaAttachment < ApplicationRecord
|
||||||
},
|
},
|
||||||
}.freeze
|
}.freeze
|
||||||
|
|
||||||
|
LIMIT = 8.megabytes
|
||||||
|
|
||||||
belongs_to :account, inverse_of: :media_attachments, optional: true
|
belongs_to :account, inverse_of: :media_attachments, optional: true
|
||||||
belongs_to :status, inverse_of: :media_attachments, optional: true
|
belongs_to :status, inverse_of: :media_attachments, optional: true
|
||||||
|
|
||||||
|
@ -64,10 +66,9 @@ class MediaAttachment < ApplicationRecord
|
||||||
processors: ->(f) { file_processors f },
|
processors: ->(f) { file_processors f },
|
||||||
convert_options: { all: '-quality 90 -strip' }
|
convert_options: { all: '-quality 90 -strip' }
|
||||||
|
|
||||||
include Remotable
|
|
||||||
|
|
||||||
validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES
|
validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES
|
||||||
validates_attachment_size :file, less_than: 8.megabytes
|
validates_attachment_size :file, less_than: LIMIT
|
||||||
|
remotable_attachment :file, LIMIT
|
||||||
|
|
||||||
validates :account, presence: true
|
validates :account, presence: true
|
||||||
validates :description, length: { maximum: 420 }, if: :local?
|
validates :description, length: { maximum: 420 }, if: :local?
|
||||||
|
|
|
@ -26,6 +26,7 @@
|
||||||
|
|
||||||
class PreviewCard < ApplicationRecord
|
class PreviewCard < ApplicationRecord
|
||||||
IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze
|
IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze
|
||||||
|
LIMIT = 1.megabytes
|
||||||
|
|
||||||
self.inheritance_column = false
|
self.inheritance_column = false
|
||||||
|
|
||||||
|
@ -36,11 +37,11 @@ class PreviewCard < ApplicationRecord
|
||||||
has_attached_file :image, styles: { original: { geometry: '400x400>', file_geometry_parser: FastGeometryParser } }, convert_options: { all: '-quality 80 -strip' }
|
has_attached_file :image, styles: { original: { geometry: '400x400>', file_geometry_parser: FastGeometryParser } }, convert_options: { all: '-quality 80 -strip' }
|
||||||
|
|
||||||
include Attachmentable
|
include Attachmentable
|
||||||
include Remotable
|
|
||||||
|
|
||||||
validates :url, presence: true, uniqueness: true
|
validates :url, presence: true, uniqueness: true
|
||||||
validates_attachment_content_type :image, content_type: IMAGE_MIME_TYPES
|
validates_attachment_content_type :image, content_type: IMAGE_MIME_TYPES
|
||||||
validates_attachment_size :image, less_than: 1.megabytes
|
validates_attachment_size :image, less_than: LIMIT
|
||||||
|
remotable_attachment :image, LIMIT
|
||||||
|
|
||||||
before_save :extract_dimensions, if: :link?
|
before_save :extract_dimensions, if: :link?
|
||||||
|
|
||||||
|
|
|
@ -38,13 +38,14 @@ class FetchAtomService < BaseService
|
||||||
return nil if response.code != 200
|
return nil if response.code != 200
|
||||||
|
|
||||||
if response.mime_type == 'application/atom+xml'
|
if response.mime_type == 'application/atom+xml'
|
||||||
[@url, { prefetched_body: response.to_s }, :ostatus]
|
[@url, { prefetched_body: response.body_with_limit }, :ostatus]
|
||||||
elsif ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(response.mime_type)
|
elsif ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(response.mime_type)
|
||||||
json = body_to_json(response.to_s)
|
body = response.body_with_limit
|
||||||
|
json = body_to_json(body)
|
||||||
if supported_context?(json) && json['type'] == 'Person' && json['inbox'].present?
|
if supported_context?(json) && json['type'] == 'Person' && json['inbox'].present?
|
||||||
[json['id'], { prefetched_body: response.to_s, id: true }, :activitypub]
|
[json['id'], { prefetched_body: body, id: true }, :activitypub]
|
||||||
elsif supported_context?(json) && json['type'] == 'Note'
|
elsif supported_context?(json) && json['type'] == 'Note'
|
||||||
[json['id'], { prefetched_body: response.to_s, id: true }, :activitypub]
|
[json['id'], { prefetched_body: body, id: true }, :activitypub]
|
||||||
else
|
else
|
||||||
@unsupported_activity = true
|
@unsupported_activity = true
|
||||||
nil
|
nil
|
||||||
|
@ -61,7 +62,7 @@ class FetchAtomService < BaseService
|
||||||
end
|
end
|
||||||
|
|
||||||
def process_html(response)
|
def process_html(response)
|
||||||
page = Nokogiri::HTML(response.to_s)
|
page = Nokogiri::HTML(response.body_with_limit)
|
||||||
|
|
||||||
json_link = page.xpath('//link[@rel="alternate"]').find { |link| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link['type']) }
|
json_link = page.xpath('//link[@rel="alternate"]').find { |link| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link['type']) }
|
||||||
atom_link = page.xpath('//link[@rel="alternate"]').find { |link| link['type'] == 'application/atom+xml' }
|
atom_link = page.xpath('//link[@rel="alternate"]').find { |link| link['type'] == 'application/atom+xml' }
|
||||||
|
|
|
@ -45,7 +45,7 @@ class FetchLinkCardService < BaseService
|
||||||
|
|
||||||
Request.new(:get, @url).perform do |res|
|
Request.new(:get, @url).perform do |res|
|
||||||
if res.code == 200 && res.mime_type == 'text/html'
|
if res.code == 200 && res.mime_type == 'text/html'
|
||||||
@html = res.to_s
|
@html = res.body_with_limit
|
||||||
@html_charset = res.charset
|
@html_charset = res.charset
|
||||||
else
|
else
|
||||||
@html = nil
|
@html = nil
|
||||||
|
|
|
@ -181,7 +181,7 @@ class ResolveAccountService < BaseService
|
||||||
|
|
||||||
@atom_body = Request.new(:get, atom_url).perform do |response|
|
@atom_body = Request.new(:get, atom_url).perform do |response|
|
||||||
raise Mastodon::UnexpectedResponseError, response unless response.code == 200
|
raise Mastodon::UnexpectedResponseError, response unless response.code == 200
|
||||||
response.to_s
|
response.body_with_limit
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
@ -57,7 +57,7 @@ class Pubsubhubbub::ConfirmationWorker
|
||||||
|
|
||||||
def callback_get_with_params
|
def callback_get_with_params
|
||||||
Request.new(:get, subscription.callback_url, params: callback_params).perform do |response|
|
Request.new(:get, subscription.callback_url, params: callback_params).perform do |response|
|
||||||
@callback_response_body = response.body.to_s
|
@callback_response_body = response.body_with_limit
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
# frozen_string_literal: true
|
# frozen_string_literal: true
|
||||||
|
|
||||||
require 'rails_helper'
|
require 'rails_helper'
|
||||||
|
require 'securerandom'
|
||||||
|
|
||||||
describe Request do
|
describe Request do
|
||||||
subject { Request.new(:get, 'http://example.com') }
|
subject { Request.new(:get, 'http://example.com') }
|
||||||
|
@ -64,6 +65,12 @@ describe Request do
|
||||||
expect_any_instance_of(HTTP::Client).to receive(:close)
|
expect_any_instance_of(HTTP::Client).to receive(:close)
|
||||||
expect { |block| subject.perform &block }.to yield_control
|
expect { |block| subject.perform &block }.to yield_control
|
||||||
end
|
end
|
||||||
|
|
||||||
|
it 'returns response which implements body_with_limit' do
|
||||||
|
subject.perform do |response|
|
||||||
|
expect(response).to respond_to :body_with_limit
|
||||||
|
end
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
context 'with private host' do
|
context 'with private host' do
|
||||||
|
@ -81,4 +88,46 @@ describe Request do
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
describe "response's body_with_limit method" do
|
||||||
|
it 'rejects body more than 1 megabyte by default' do
|
||||||
|
stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.megabytes))
|
||||||
|
expect { subject.perform { |response| response.body_with_limit } }.to raise_error Mastodon::LengthValidationError
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'accepts body less than 1 megabyte by default' do
|
||||||
|
stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.kilobytes))
|
||||||
|
expect { subject.perform { |response| response.body_with_limit } }.not_to raise_error
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'rejects body by given size' do
|
||||||
|
stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.kilobytes))
|
||||||
|
expect { subject.perform { |response| response.body_with_limit(1.kilobyte) } }.to raise_error Mastodon::LengthValidationError
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'rejects too large chunked body' do
|
||||||
|
stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.megabytes), headers: { 'Transfer-Encoding' => 'chunked' })
|
||||||
|
expect { subject.perform { |response| response.body_with_limit } }.to raise_error Mastodon::LengthValidationError
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'rejects too large monolithic body' do
|
||||||
|
stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.megabytes), headers: { 'Content-Length' => 2.megabytes })
|
||||||
|
expect { subject.perform { |response| response.body_with_limit } }.to raise_error Mastodon::LengthValidationError
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'uses binary encoding if Content-Type does not tell encoding' do
|
||||||
|
stub_request(:any, 'http://example.com').to_return(body: '', headers: { 'Content-Type' => 'text/html' })
|
||||||
|
expect(subject.perform { |response| response.body_with_limit.encoding }).to eq Encoding::BINARY
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'uses binary encoding if Content-Type tells unknown encoding' do
|
||||||
|
stub_request(:any, 'http://example.com').to_return(body: '', headers: { 'Content-Type' => 'text/html; charset=unknown' })
|
||||||
|
expect(subject.perform { |response| response.body_with_limit.encoding }).to eq Encoding::BINARY
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'uses encoding specified by Content-Type' do
|
||||||
|
stub_request(:any, 'http://example.com').to_return(body: '', headers: { 'Content-Type' => 'text/html; charset=UTF-8' })
|
||||||
|
expect(subject.perform { |response| response.body_with_limit.encoding }).to eq Encoding::UTF_8
|
||||||
|
end
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -29,7 +29,10 @@ RSpec.describe Remotable do
|
||||||
|
|
||||||
context 'Remotable module is included' do
|
context 'Remotable module is included' do
|
||||||
before do
|
before do
|
||||||
class Foo; include Remotable; end
|
class Foo
|
||||||
|
include Remotable
|
||||||
|
remotable_attachment :hoge, 1.kilobyte
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
let(:attribute_name) { "#{hoge}_remote_url".to_sym }
|
let(:attribute_name) { "#{hoge}_remote_url".to_sym }
|
||||||
|
|
Loading…
Reference in New Issue