From 99f3a5554074d9a12619797c474b3de4c6085f02 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 5 Jul 2020 18:28:25 +0200 Subject: [PATCH] Add color extraction for audio thumbnails (#14209) --- Gemfile | 1 + Gemfile.lock | 2 + app/javascript/mastodon/components/status.js | 4 +- .../mastodon/features/audio/index.js | 238 +++--------------- .../status/components/detailed_status.js | 4 +- .../features/ui/components/audio_modal.js | 4 +- .../ui/components/focal_point_modal.js | 4 +- .../styles/mastodon/components.scss | 45 ++-- app/models/media_attachment.rb | 13 +- app/views/media/player.html.haml | 2 +- app/views/statuses/_detailed_status.html.haml | 2 +- app/views/statuses/_simple_status.html.haml | 2 +- app/workers/post_process_media_worker.rb | 2 +- config/application.rb | 1 + lib/paperclip/color_extractor.rb | 189 ++++++++++++++ lib/paperclip/transcoder_extensions.rb | 14 ++ 16 files changed, 284 insertions(+), 243 deletions(-) create mode 100644 lib/paperclip/color_extractor.rb create mode 100644 lib/paperclip/transcoder_extensions.rb diff --git a/Gemfile b/Gemfile index d9415d874b9..04eb41bdca7 100644 --- a/Gemfile +++ b/Gemfile @@ -48,6 +48,7 @@ gem 'omniauth-cas', '~> 1.1' gem 'omniauth-saml', '~> 1.10' gem 'omniauth', '~> 1.9' +gem 'color_diff', '~> 0.1' gem 'discard', '~> 1.2' gem 'doorkeeper', '~> 5.4' gem 'ed25519', '~> 1.2' diff --git a/Gemfile.lock b/Gemfile.lock index fcea8100257..bc7106e3097 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -165,6 +165,7 @@ GEM cocaine (0.5.8) climate_control (>= 0.0.3, < 1.0) coderay (1.1.3) + color_diff (0.1) concurrent-ruby (1.1.6) connection_pool (2.2.3) crack (0.4.3) @@ -689,6 +690,7 @@ DEPENDENCIES chewy (~> 5.1) cld3 (~> 3.3.0) climate_control (~> 0.2) + color_diff (~> 0.1) concurrent-ruby connection_pool devise (~> 4.7) diff --git a/app/javascript/mastodon/components/status.js b/app/javascript/mastodon/components/status.js index 827b69500d0..f9f6736e62e 100644 --- a/app/javascript/mastodon/components/status.js +++ b/app/javascript/mastodon/components/status.js @@ -353,7 +353,9 @@ class Status extends ImmutablePureComponent { src={attachment.get('url')} alt={attachment.get('description')} poster={attachment.get('preview_url') || status.getIn(['account', 'avatar_static'])} - blurhash={attachment.get('blurhash')} + backgroundColor={attachment.getIn(['meta', 'colors', 'background'])} + foregroundColor={attachment.getIn(['meta', 'colors', 'foreground'])} + accentColor={attachment.getIn(['meta', 'colors', 'accent'])} duration={attachment.getIn(['meta', 'original', 'duration'], 0)} width={this.props.cachedMediaWidth} height={110} diff --git a/app/javascript/mastodon/features/audio/index.js b/app/javascript/mastodon/features/audio/index.js index 99926e52ab2..686709ac389 100644 --- a/app/javascript/mastodon/features/audio/index.js +++ b/app/javascript/mastodon/features/audio/index.js @@ -5,131 +5,12 @@ import { formatTime } from 'mastodon/features/video'; import Icon from 'mastodon/components/icon'; import classNames from 'classnames'; import { throttle } from 'lodash'; -import { encode, decode } from 'blurhash'; import { getPointerPosition, fileNameFromURL } from 'mastodon/features/video'; import { debounce } from 'lodash'; -const digitCharacters = [ - '0', - '1', - '2', - '3', - '4', - '5', - '6', - '7', - '8', - '9', - 'A', - 'B', - 'C', - 'D', - 'E', - 'F', - 'G', - 'H', - 'I', - 'J', - 'K', - 'L', - 'M', - 'N', - 'O', - 'P', - 'Q', - 'R', - 'S', - 'T', - 'U', - 'V', - 'W', - 'X', - 'Y', - 'Z', - 'a', - 'b', - 'c', - 'd', - 'e', - 'f', - 'g', - 'h', - 'i', - 'j', - 'k', - 'l', - 'm', - 'n', - 'o', - 'p', - 'q', - 'r', - 's', - 't', - 'u', - 'v', - 'w', - 'x', - 'y', - 'z', - '#', - '$', - '%', - '*', - '+', - ',', - '-', - '.', - ':', - ';', - '=', - '?', - '@', - '[', - ']', - '^', - '_', - '{', - '|', - '}', - '~', -]; - -const decode83 = (str) => { - let value = 0; - let c, digit; - - for (let i = 0; i < str.length; i++) { - c = str[i]; - digit = digitCharacters.indexOf(c); - value = value * 83 + digit; - } - - return value; -}; - -const decodeRGB = int => ({ - r: Math.max(0, (int >> 16)), - g: Math.max(0, (int >> 8) & 255), - b: Math.max(0, (int & 255)), -}); - -const luma = ({ r, g, b }) => 0.2126 * r + 0.7152 * g + 0.0722 * b; - -const adjustColor = ({ r, g, b }, lumaThreshold = 100) => { - let delta; - - if (luma({ r, g, b }) >= lumaThreshold) { - delta = -80; - } else { - delta = 80; - } - - return { - r: r + delta, - g: g + delta, - b: b + delta, - }; +const hex2rgba = (hex, alpha = 1) => { + const [r, g, b] = hex.match(/\w\w/g).map(x => parseInt(x, 16)); + return `rgba(${r}, ${g}, ${b}, ${alpha})`; }; const messages = defineMessages({ @@ -157,7 +38,9 @@ class Audio extends React.PureComponent { fullscreen: PropTypes.bool, intl: PropTypes.object.isRequired, cacheWidth: PropTypes.func, - blurhash: PropTypes.string, + backgroundColor: PropTypes.string, + foregroundColor: PropTypes.string, + accentColor: PropTypes.string, }; state = { @@ -169,7 +52,6 @@ class Audio extends React.PureComponent { muted: false, volume: 0.5, dragging: false, - color: { r: 255, g: 255, b: 255 }, }; setPlayerRef = c => { @@ -207,10 +89,6 @@ class Audio extends React.PureComponent { } } - setBlurhashCanvasRef = c => { - this.blurhashCanvas = c; - } - setCanvasRef = c => { this.canvas = c; @@ -222,41 +100,13 @@ class Audio extends React.PureComponent { componentDidMount () { window.addEventListener('scroll', this.handleScroll); window.addEventListener('resize', this.handleResize, { passive: true }); - - if (!this.props.blurhash) { - const img = new Image(); - img.crossOrigin = 'anonymous'; - img.onload = () => this.handlePosterLoad(img); - img.src = this.props.poster; - } else { - this._setColorScheme(); - this._decodeBlurhash(); - } } componentDidUpdate (prevProps, prevState) { - if (prevProps.poster !== this.props.poster && !this.props.blurhash) { - const img = new Image(); - img.crossOrigin = 'anonymous'; - img.onload = () => this.handlePosterLoad(img); - img.src = this.props.poster; + if (prevProps.src !== this.props.src || this.state.width !== prevState.width || this.state.height !== prevState.height) { + this._clear(); + this._draw(); } - - if (prevState.blurhash !== this.state.blurhash || prevProps.blurhash !== this.props.blurhash) { - this._setColorScheme(); - this._decodeBlurhash(); - } - - this._clear(); - this._draw(); - } - - _decodeBlurhash () { - const context = this.blurhashCanvas.getContext('2d'); - const pixels = decode(this.props.blurhash || this.state.blurhash, 32, 32); - const outputImageData = new ImageData(pixels, 32, 32); - - context.putImageData(outputImageData, 0, 0); } componentWillUnmount () { @@ -425,31 +275,6 @@ class Audio extends React.PureComponent { this.analyser = analyser; } - handlePosterLoad = image => { - const canvas = document.createElement('canvas'); - const context = canvas.getContext('2d'); - - canvas.width = image.width; - canvas.height = image.height; - - context.drawImage(image, 0, 0); - - const inputImageData = context.getImageData(0, 0, image.width, image.height); - const blurhash = encode(inputImageData.data, image.width, image.height, 4, 4); - - this.setState({ blurhash }); - } - - _setColorScheme () { - const blurhash = this.props.blurhash || this.state.blurhash; - const averageColor = decodeRGB(decode83(blurhash.slice(2, 6))); - - this.setState({ - color: adjustColor(averageColor), - darkText: luma(averageColor) >= 165, - }); - } - handleDownload = () => { fetch(this.props.src).then(res => res.blob()).then(blob => { const element = document.createElement('a'); @@ -609,8 +434,8 @@ class Audio extends React.PureComponent { const gradient = this.canvasContext.createLinearGradient(dx1, dy1, dx2, dy2); - const mainColor = `rgb(${this.state.color.r}, ${this.state.color.g}, ${this.state.color.b})`; - const lastColor = `rgba(${this.state.color.r}, ${this.state.color.g}, ${this.state.color.b}, 0)`; + const mainColor = this._getAccentColor(); + const lastColor = hex2rgba(mainColor, 0); gradient.addColorStop(0, mainColor); gradient.addColorStop(0.6, mainColor); @@ -632,17 +457,25 @@ class Audio extends React.PureComponent { return Math.floor(this._getRadius() + (PADDING * this._getScaleCoefficient())); } - _getColor () { - return `rgb(${this.state.color.r}, ${this.state.color.g}, ${this.state.color.b})`; + _getAccentColor () { + return this.props.accentColor || '#ffffff'; + } + + _getBackgroundColor () { + return this.props.backgroundColor || '#000000'; + } + + _getForegroundColor () { + return this.props.foregroundColor || '#ffffff'; } render () { const { src, intl, alt, editable } = this.props; - const { paused, muted, volume, currentTime, duration, buffer, darkText, dragging } = this.state; + const { paused, muted, volume, currentTime, duration, buffer, dragging } = this.state; const progress = (currentTime / duration) * 100; return ( -
+