From a6d2fe716529f86310954a40f66edb57ca03055f Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 27 Apr 2019 03:24:09 +0200 Subject: [PATCH] Add blurhash (#10630) * Add blurhash * Use fallback color for spoiler when blurhash missing * Federate the blurhash and accept it as long as it's at most 5x5 * Display unknown media attachments as blurhash placeholders * Improve style of embed actions and spoiler button * Change blurhash resolution from 3x3 to 4x4 * Improve dependency definitions * Fix code style issues --- Gemfile | 1 + Gemfile.lock | 3 + .../mastodon/components/media_gallery.js | 96 +++++++++++++------ app/javascript/mastodon/components/status.js | 3 +- .../report/components/status_check_box.js | 1 + .../status/components/detailed_status.js | 6 +- .../features/ui/components/media_modal.js | 1 + .../features/ui/components/video_modal.js | 1 + .../mastodon/features/video/index.js | 49 ++++++++-- .../styles/mastodon/components.scss | 70 ++++++++++++-- app/lib/activitypub/activity/create.rb | 7 +- app/lib/activitypub/adapter.rb | 1 + app/models/media_attachment.rb | 15 ++- .../activitypub/note_serializer.rb | 4 +- .../rest/media_attachment_serializer.rb | 2 +- .../stream_entries/_detailed_status.html.haml | 2 +- .../stream_entries/_simple_status.html.haml | 2 +- ...25523_add_blurhash_to_media_attachments.rb | 5 + db/schema.rb | 3 +- lib/paperclip/blurhash_transcoder.rb | 16 ++++ package.json | 1 + yarn.lock | 5 + 22 files changed, 234 insertions(+), 60 deletions(-) create mode 100644 db/migrate/20190420025523_add_blurhash_to_media_attachments.rb create mode 100644 lib/paperclip/blurhash_transcoder.rb diff --git a/Gemfile b/Gemfile index 6fe97412b8..fa8478d897 100644 --- a/Gemfile +++ b/Gemfile @@ -21,6 +21,7 @@ gem 'fog-openstack', '~> 0.3', require: false gem 'paperclip', '~> 6.0' gem 'paperclip-av-transcoder', '~> 0.6' gem 'streamio-ffmpeg', '~> 3.0' +gem 'blurhash', '~> 0.1' gem 'active_model_serializers', '~> 0.10' gem 'addressable', '~> 2.6' diff --git a/Gemfile.lock b/Gemfile.lock index 66fc8d1f4b..0148cb5ea9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -99,6 +99,8 @@ GEM rack (>= 0.9.0) binding_of_caller (0.8.0) debug_inspector (>= 0.0.1) + blurhash (0.1.2) + ffi (~> 1.10.0) bootsnap (1.4.4) msgpack (~> 1.0) brakeman (4.5.0) @@ -661,6 +663,7 @@ DEPENDENCIES aws-sdk-s3 (~> 1.36) better_errors (~> 2.5) binding_of_caller (~> 0.7) + blurhash (~> 0.1) bootsnap (~> 1.4) brakeman (~> 4.5) browser diff --git a/app/javascript/mastodon/components/media_gallery.js b/app/javascript/mastodon/components/media_gallery.js index a2bc952556..f548296d06 100644 --- a/app/javascript/mastodon/components/media_gallery.js +++ b/app/javascript/mastodon/components/media_gallery.js @@ -7,6 +7,7 @@ import { defineMessages, injectIntl, FormattedMessage } from 'react-intl'; import { isIOS } from '../is_mobile'; import classNames from 'classnames'; import { autoPlayGif, displayMedia } from '../initial_state'; +import { decode } from 'blurhash'; const messages = defineMessages({ toggle_visible: { id: 'media_gallery.toggle_visible', defaultMessage: 'Toggle visibility' }, @@ -21,6 +22,7 @@ class Item extends React.PureComponent { size: PropTypes.number.isRequired, onClick: PropTypes.func.isRequired, displayWidth: PropTypes.number, + visible: PropTypes.bool.isRequired, }; static defaultProps = { @@ -29,6 +31,10 @@ class Item extends React.PureComponent { size: 1, }; + state = { + loaded: false, + }; + handleMouseEnter = (e) => { if (this.hoverToPlay()) { e.target.play(); @@ -62,8 +68,40 @@ class Item extends React.PureComponent { e.stopPropagation(); } + componentDidMount () { + if (this.props.attachment.get('blurhash')) { + this._decode(); + } + } + + componentDidUpdate (prevProps) { + if (prevProps.attachment.get('blurhash') !== this.props.attachment.get('blurhash') && this.props.attachment.get('blurhash')) { + this._decode(); + } + } + + _decode () { + const hash = this.props.attachment.get('blurhash'); + const pixels = decode(hash, 32, 32); + + if (pixels) { + const ctx = this.canvas.getContext('2d'); + const imageData = new ImageData(pixels, 32, 32); + + ctx.putImageData(imageData, 0, 0); + } + } + + setCanvasRef = c => { + this.canvas = c; + } + + handleImageLoad = () => { + this.setState({ loaded: true }); + } + render () { - const { attachment, index, size, standalone, displayWidth } = this.props; + const { attachment, index, size, standalone, displayWidth, visible } = this.props; let width = 50; let height = 100; @@ -116,12 +154,20 @@ class Item extends React.PureComponent { let thumbnail = ''; - if (attachment.get('type') === 'image') { + if (attachment.get('type') === 'unknown') { + return ( +
+ + + +
+ ); + } else if (attachment.get('type') === 'image') { const previewUrl = attachment.get('preview_url'); const previewWidth = attachment.getIn(['meta', 'small', 'width']); - const originalUrl = attachment.get('url'); - const originalWidth = attachment.getIn(['meta', 'original', 'width']); + const originalUrl = attachment.get('url'); + const originalWidth = attachment.getIn(['meta', 'original', 'width']); const hasSize = typeof originalWidth === 'number' && typeof previewWidth === 'number'; @@ -147,6 +193,7 @@ class Item extends React.PureComponent { alt={attachment.get('description')} title={attachment.get('description')} style={{ objectPosition: `${x}% ${y}%` }} + onLoad={this.handleImageLoad} /> ); @@ -176,7 +223,8 @@ class Item extends React.PureComponent { return (
- {thumbnail} + + {visible && thumbnail}
); } @@ -225,6 +273,7 @@ class MediaGallery extends React.PureComponent { if (node /*&& this.isStandaloneEligible()*/) { // offsetWidth triggers a layout, so only calculate when we need to if (this.props.cacheWidth) this.props.cacheWidth(node.offsetWidth); + this.setState({ width: node.offsetWidth, }); @@ -242,7 +291,7 @@ class MediaGallery extends React.PureComponent { const width = this.state.width || defaultWidth; - let children; + let children, spoilerButton; const style = {}; @@ -256,35 +305,28 @@ class MediaGallery extends React.PureComponent { style.height = height; } - if (!visible) { - let warning; + const size = media.take(4).size; - if (sensitive) { - warning = ; - } else { - warning = ; - } + if (this.isStandaloneEligible()) { + children = ; + } else { + children = media.take(4).map((attachment, i) => ); + } - children = ( - ); - } else { - const size = media.take(4).size; - - if (this.isStandaloneEligible()) { - children = ; - } else { - children = media.take(4).map((attachment, i) => ); - } } return (
-
- +
+ {spoilerButton}
{children} diff --git a/app/javascript/mastodon/components/status.js b/app/javascript/mastodon/components/status.js index cea9a0c2e5..95ca4a5485 100644 --- a/app/javascript/mastodon/components/status.js +++ b/app/javascript/mastodon/components/status.js @@ -274,7 +274,7 @@ class Status extends ImmutablePureComponent { if (status.get('poll')) { media = ; } else if (status.get('media_attachments').size > 0) { - if (this.props.muted || status.get('media_attachments').some(item => item.get('type') === 'unknown')) { + if (this.props.muted) { media = ( ( ( ; } else if (status.get('media_attachments').size > 0) { - if (status.get('media_attachments').some(item => item.get('type') === 'unknown')) { - media = ; - } else if (status.getIn(['media_attachments', 0, 'type']) === 'video') { + if (status.getIn(['media_attachments', 0, 'type']) === 'video') { const video = status.getIn(['media_attachments', 0]); media = (