From f499f086a9dd8803e3adefa016468acec245c4ba Mon Sep 17 00:00:00 2001 From: Claire Date: Tue, 22 Feb 2022 17:11:22 +0100 Subject: [PATCH] Fix some media attachments being converted with too high framerates (#17619) Video files with variable framerates are converted to constant framerate videos and the output framerate picked by ffmpeg is based on the original file's container framerate (which can be different from the average framerate). This means that an input video with variable framerate with about 30 frames per second on average, but a maximum of 120 fps will be converted to a constant 120 fps file, which won't be processed by other Mastodon servers. This commit changes it so that input files with VFR and a maximum framerate above the framerate threshold are converted to VFR files with the maximum frame rate enforced. --- app/lib/video_metadata_extractor.rb | 3 ++- app/models/media_attachment.rb | 13 +++++++------ lib/paperclip/transcoder.rb | 16 +++++++++++++++- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/app/lib/video_metadata_extractor.rb b/app/lib/video_metadata_extractor.rb index 03e40f923e..2896620cb2 100644 --- a/app/lib/video_metadata_extractor.rb +++ b/app/lib/video_metadata_extractor.rb @@ -2,7 +2,7 @@ class VideoMetadataExtractor attr_reader :duration, :bitrate, :video_codec, :audio_codec, - :colorspace, :width, :height, :frame_rate + :colorspace, :width, :height, :frame_rate, :r_frame_rate def initialize(path) @path = path @@ -42,6 +42,7 @@ class VideoMetadataExtractor @width = video_stream[:width] @height = video_stream[:height] @frame_rate = video_stream[:avg_frame_rate] == '0/0' ? nil : Rational(video_stream[:avg_frame_rate]) + @r_frame_rate = video_stream[:r_frame_rate] == '0/0' ? nil : Rational(video_stream[:r_frame_rate]) end if (audio_stream = audio_streams.first) diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index 0a9d05f1d7..a3115637e5 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -38,6 +38,12 @@ class MediaAttachment < ApplicationRecord MAX_DESCRIPTION_LENGTH = 1_500 + IMAGE_LIMIT = 10.megabytes + VIDEO_LIMIT = 40.megabytes + + MAX_VIDEO_MATRIX_LIMIT = 2_304_000 # 1920x1200px + MAX_VIDEO_FRAME_RATE = 60 + IMAGE_FILE_EXTENSIONS = %w(.jpg .jpeg .png .gif).freeze VIDEO_FILE_EXTENSIONS = %w(.webm .mp4 .m4v .mov).freeze AUDIO_FILE_EXTENSIONS = %w(.ogg .oga .mp3 .wav .flac .opus .aac .m4a .3gp .wma).freeze @@ -75,6 +81,7 @@ class MediaAttachment < ApplicationRecord VIDEO_FORMAT = { format: 'mp4', content_type: 'video/mp4', + vfr_frame_rate_threshold: MAX_VIDEO_FRAME_RATE, convert_options: { output: { 'loglevel' => 'fatal', @@ -152,12 +159,6 @@ class MediaAttachment < ApplicationRecord all: '-quality 90 -strip +set modify-date +set create-date', }.freeze - IMAGE_LIMIT = 10.megabytes - VIDEO_LIMIT = 40.megabytes - - MAX_VIDEO_MATRIX_LIMIT = 2_304_000 # 1920x1200px - MAX_VIDEO_FRAME_RATE = 60 - belongs_to :account, inverse_of: :media_attachments, optional: true belongs_to :status, inverse_of: :media_attachments, optional: true belongs_to :scheduled_status, inverse_of: :media_attachments, optional: true diff --git a/lib/paperclip/transcoder.rb b/lib/paperclip/transcoder.rb index ec13050382..afd9f58ff6 100644 --- a/lib/paperclip/transcoder.rb +++ b/lib/paperclip/transcoder.rb @@ -13,6 +13,7 @@ module Paperclip @time = options[:time] || 3 @passthrough_options = options[:passthrough_options] @convert_options = options[:convert_options].dup + @vfr_threshold = options[:vfr_frame_rate_threshold] end def make @@ -41,6 +42,11 @@ module Paperclip when 'mp4' @output_options['acodec'] = 'aac' @output_options['strict'] = 'experimental' + + if high_vfr?(metadata) && !eligible_to_passthrough?(metadata) + @output_options['vsync'] = 'vfr' + @output_options['r'] = @vfr_threshold + end end command_arguments, interpolations = prepare_command(destination) @@ -88,13 +94,21 @@ module Paperclip end def update_options_from_metadata(metadata) - return unless @passthrough_options && @passthrough_options[:video_codecs].include?(metadata.video_codec) && @passthrough_options[:audio_codecs].include?(metadata.audio_codec) && @passthrough_options[:colorspaces].include?(metadata.colorspace) + return unless eligible_to_passthrough?(metadata) @format = @passthrough_options[:options][:format] || @format @time = @passthrough_options[:options][:time] || @time @convert_options = @passthrough_options[:options][:convert_options].dup end + def high_vfr?(metadata) + @vfr_threshold && metadata.r_frame_rate && metadata.r_frame_rate > @vfr_threshold + end + + def eligible_to_passthrough?(metadata) + @passthrough_options && @passthrough_options[:video_codecs].include?(metadata.video_codec) && @passthrough_options[:audio_codecs].include?(metadata.audio_codec) && @passthrough_options[:colorspaces].include?(metadata.colorspace) + end + def update_attachment_type(metadata) @attachment.instance.type = MediaAttachment.types[:gifv] unless metadata.audio_codec end