Add audio player (#11644)

app/javascript/mastodon/components/status.js

@@ -12,7 +12,7 @@ import AttachmentList from './attachment_list';
 import Card from '../features/status/components/card';
 import { injectIntl, FormattedMessage } from 'react-intl';
 import ImmutablePureComponent from 'react-immutable-pure-component';
-import { MediaGallery, Video } from '../features/ui/util/async-components';
+import { MediaGallery, Video, Audio } from '../features/ui/util/async-components';
 import { HotKeys } from 'react-hotkeys';
 import classNames from 'classnames';
 import Icon from 'mastodon/components/icon';
@@ -199,11 +199,15 @@ class Status extends ImmutablePureComponent {
   };
 
   renderLoadingMediaGallery () {
-    return <div className='media_gallery' style={{ height: '110px' }} />;
+    return <div className='media-gallery' style={{ height: '110px' }} />;
   }
 
   renderLoadingVideoPlayer () {
-    return <div className='media-spoiler-video' style={{ height: '110px' }} />;
+    return <div className='video-player' style={{ height: '110px' }} />;
+  }
+
+  renderLoadingAudioPlayer () {
+    return <div className='audio-player' style={{ height: '110px' }} />;
   }
 
   handleOpenVideo = (media, startTime) => {
@@ -348,7 +352,22 @@ class Status extends ImmutablePureComponent {
             media={status.get('media_attachments')}
           />
         );
-      } else if (['video', 'audio'].includes(status.getIn(['media_attachments', 0, 'type']))) {
+      } else if (status.getIn(['media_attachments', 0, 'type']) === 'audio') {
+        const attachment = status.getIn(['media_attachments', 0]);
+
+        media = (
+          <Bundle fetchComponent={Audio} loading={this.renderLoadingAudioPlayer} >
+            {Component => (
+              <Component
+                src={attachment.get('url')}
+                alt={attachment.get('description')}
+                duration={attachment.getIn(['meta', 'original', 'duration'], 0)}
+                height={110}
+              />
+            )}
+          </Bundle>
+        );
+      } else if (status.getIn(['media_attachments', 0, 'type']) === 'video') {
         const attachment = status.getIn(['media_attachments', 0]);
 
         media = (

app/javascript/mastodon/containers/media_container.js

@@ -8,6 +8,7 @@ import Video from '../features/video';
 import Card from '../features/status/components/card';
 import Poll from 'mastodon/components/poll';
 import Hashtag from 'mastodon/components/hashtag';
+import Audio from 'mastodon/features/audio';
 import ModalRoot from '../components/modal_root';
 import { getScrollbarWidth } from '../features/ui/components/modal_root';
 import MediaModal from '../features/ui/components/media_modal';
@@ -16,7 +17,7 @@ import { List as ImmutableList, fromJS } from 'immutable';
 const { localeData, messages } = getLocale();
 addLocaleData(localeData);
 
-const MEDIA_COMPONENTS = { MediaGallery, Video, Card, Poll, Hashtag };
+const MEDIA_COMPONENTS = { MediaGallery, Video, Card, Poll, Hashtag, Audio };
 
 export default class MediaContainer extends PureComponent {
 

app/javascript/mastodon/features/audio/index.js

@@ -0,0 +1,219 @@
+import React from 'react';
+import PropTypes from 'prop-types';
+import WaveSurfer from 'wavesurfer.js';
+import { defineMessages, injectIntl } from 'react-intl';
+import { formatTime } from 'mastodon/features/video';
+import Icon from 'mastodon/components/icon';
+import classNames from 'classnames';
+import { throttle } from 'lodash';
+
+const messages = defineMessages({
+  play: { id: 'video.play', defaultMessage: 'Play' },
+  pause: { id: 'video.pause', defaultMessage: 'Pause' },
+  mute: { id: 'video.mute', defaultMessage: 'Mute sound' },
+  unmute: { id: 'video.unmute', defaultMessage: 'Unmute sound' },
+});
+
+const arrayOf = (length, fill) => (new Array(length)).fill(fill);
+
+export default @injectIntl
+class Audio extends React.PureComponent {
+
+  static propTypes = {
+    src: PropTypes.string.isRequired,
+    alt: PropTypes.string,
+    duration: PropTypes.number,
+    height: PropTypes.number,
+    preload: PropTypes.bool,
+    editable: PropTypes.bool,
+    intl: PropTypes.object.isRequired,
+  };
+
+  state = {
+    currentTime: 0,
+    duration: null,
+    paused: true,
+    muted: false,
+    volume: 0.5,
+  };
+
+  // hard coded in components.scss
+  // any way to get ::before values programatically?
+
+  volWidth = 50;
+
+  volOffset = 70;
+
+  volHandleOffset = v => {
+    const offset = v * this.volWidth + this.volOffset;
+    return (offset > 110) ? 110 : offset;
+  }
+
+  setVolumeRef = c => {
+    this.volume = c;
+  }
+
+  setWaveformRef = c => {
+    this.waveform = c;
+  }
+
+  componentDidMount () {
+    if (this.waveform) {
+      this._updateWaveform();
+    }
+  }
+
+  componentDidUpdate (prevProps) {
+    if (this.waveform && prevProps.src !== this.props.src) {
+      this._updateWaveform();
+    }
+  }
+
+  componentWillUnmount () {
+    if (this.wavesurfer) {
+      this.wavesurfer.destroy();
+      this.wavesurfer = null;
+    }
+  }
+
+  _updateWaveform () {
+    const { src, height, duration, preload } = this.props;
+
+    const progressColor = window.getComputedStyle(document.querySelector('.audio-player__progress-placeholder')).getPropertyValue('background-color');
+    const waveColor     = window.getComputedStyle(document.querySelector('.audio-player__wave-placeholder')).getPropertyValue('background-color');
+
+    if (this.wavesurfer) {
+      this.wavesurfer.destroy();
+    }
+
+    const wavesurfer = WaveSurfer.create({
+      container: this.waveform,
+      height,
+      barWidth: 3,
+      cursorWidth: 0,
+      progressColor,
+      waveColor,
+      forceDecode: true,
+    });
+
+    wavesurfer.setVolume(this.state.volume);
+
+    if (preload) {
+      wavesurfer.load(src);
+    } else {
+      wavesurfer.load(src, arrayOf(1, 0.5), null, duration);
+    }
+
+    wavesurfer.on('ready', () => this.setState({ duration: Math.floor(wavesurfer.getDuration()) }));
+    wavesurfer.on('audioprocess', () => this.setState({ currentTime: Math.floor(wavesurfer.getCurrentTime()) }));
+    wavesurfer.on('pause', () => this.setState({ paused: true }));
+    wavesurfer.on('play', () => this.setState({ paused: false }));
+    wavesurfer.on('volume', volume => this.setState({ volume }));
+    wavesurfer.on('mute', muted => this.setState({ muted }));
+
+    this.wavesurfer = wavesurfer;
+  }
+
+  togglePlay = () => {
+    if (this.state.paused) {
+      if (!this.props.preload) {
+        this.wavesurfer.createBackend();
+        this.wavesurfer.createPeakCache();
+        this.wavesurfer.load(this.props.src);
+      }
+
+      this.wavesurfer.play();
+    } else {
+      this.wavesurfer.pause();
+    }
+  }
+
+  toggleMute = () => {
+    this.wavesurfer.setMute(!this.state.muted);
+  }
+
+  handleVolumeMouseDown = e => {
+    document.addEventListener('mousemove', this.handleMouseVolSlide, true);
+    document.addEventListener('mouseup', this.handleVolumeMouseUp, true);
+    document.addEventListener('touchmove', this.handleMouseVolSlide, true);
+    document.addEventListener('touchend', this.handleVolumeMouseUp, true);
+
+    this.handleMouseVolSlide(e);
+
+    e.preventDefault();
+    e.stopPropagation();
+  }
+
+  handleVolumeMouseUp = () => {
+    document.removeEventListener('mousemove', this.handleMouseVolSlide, true);
+    document.removeEventListener('mouseup', this.handleVolumeMouseUp, true);
+    document.removeEventListener('touchmove', this.handleMouseVolSlide, true);
+    document.removeEventListener('touchend', this.handleVolumeMouseUp, true);
+  }
+
+  handleMouseVolSlide = throttle(e => {
+    const rect = this.volume.getBoundingClientRect();
+    const x    = (e.clientX - rect.left) / this.volWidth; // x position within the element.
+
+    if(!isNaN(x)) {
+      let slideamt = x;
+
+      if (x > 1) {
+        slideamt = 1;
+      } else if(x < 0) {
+        slideamt = 0;
+      }
+
+      this.wavesurfer.setVolume(slideamt);
+    }
+  }, 60);
+
+  render () {
+    const { height, intl, alt, editable } = this.props;
+    const { paused, muted, volume, currentTime } = this.state;
+
+    const volumeWidth     = muted ? 0 : volume * this.volWidth;
+    const volumeHandleLoc = muted ? this.volHandleOffset(0) : this.volHandleOffset(volume);
+
+    return (
+      <div className={classNames('audio-player', { editable })}>
+        <div className='audio-player__progress-placeholder' style={{ display: 'none' }} />
+        <div className='audio-player__wave-placeholder' style={{ display: 'none' }} />
+
+        <div
+          className='audio-player__waveform'
+          aria-label={alt}
+          title={alt}
+          style={{ height }}
+          ref={this.setWaveformRef}
+        />
+
+        <div className='video-player__controls active'>
+          <div className='video-player__buttons-bar'>
+            <div className='video-player__buttons left'>
+              <button type='button' aria-label={intl.formatMessage(paused ? messages.play : messages.pause)} onClick={this.togglePlay}><Icon id={paused ? 'play' : 'pause'} fixedWidth /></button>
+              <button type='button' aria-label={intl.formatMessage(muted ? messages.unmute : messages.mute)} onClick={this.toggleMute}><Icon id={muted ? 'volume-off' : 'volume-up'} fixedWidth /></button>
+
+              <div className='video-player__volume' onMouseDown={this.handleVolumeMouseDown} ref={this.setVolumeRef}>
+                <div className='video-player__volume__current' style={{ width: `${volumeWidth}px` }} />
+
+                <span
+                  className={classNames('video-player__volume__handle')}
+                  tabIndex='0'
+                  style={{ left: `${volumeHandleLoc}px` }}
+                />
+              </div>
+
+              <span>
+                <span className='video-player__time-current'>{formatTime(currentTime)}</span>
+                <span className='video-player__time-sep'>/</span>
+                <span className='video-player__time-total'>{formatTime(this.state.duration || Math.floor(this.props.duration))}</span>
+              </span>
+            </div>
+          </div>
+        </div>
+      </div>
+    );
+  }
+
+}

app/javascript/mastodon/features/status/components/detailed_status.js

@@ -10,6 +10,7 @@ import { FormattedDate, FormattedNumber } from 'react-intl';
 import Card from './card';
 import ImmutablePureComponent from 'react-immutable-pure-component';
 import Video from '../../video';
+import Audio from '../../audio';
 import scheduleIdleTask from '../../ui/util/schedule_idle_task';
 import classNames from 'classnames';
 import Icon from 'mastodon/components/icon';
@@ -107,7 +108,19 @@ export default class DetailedStatus extends ImmutablePureComponent {
     }
 
     if (status.get('media_attachments').size > 0) {
-      if (['video', 'audio'].includes(status.getIn(['media_attachments', 0, 'type']))) {
+      if (status.getIn(['media_attachments', 0, 'type']) === 'audio') {
+        const attachment = status.getIn(['media_attachments', 0]);
+
+        media = (
+          <Audio
+            src={attachment.get('url')}
+            alt={attachment.get('description')}
+            duration={attachment.getIn(['meta', 'original', 'duration'], 0)}
+            height={150}
+            preload
+          />
+        );
+      } else if (status.getIn(['media_attachments', 0, 'type']) === 'video') {
         const attachment = status.getIn(['media_attachments', 0]);
 
         media = (

app/javascript/mastodon/features/ui/components/focal_point_modal.js

@@ -10,6 +10,7 @@ import { FormattedMessage, defineMessages, injectIntl } from 'react-intl';
 import IconButton from 'mastodon/components/icon_button';
 import Button from 'mastodon/components/button';
 import Video from 'mastodon/features/video';
+import Audio from 'mastodon/features/audio';
 import Textarea from 'react-textarea-autosize';
 import UploadProgress from 'mastodon/features/compose/components/upload_progress';
 import CharacterCounter from 'mastodon/features/compose/components/character_counter';
@@ -244,12 +245,23 @@ class FocalPointModal extends ImmutablePureComponent {
               </div>
             )}
 
-            {['audio', 'video'].includes(media.get('type')) && (
+            {media.get('type') === 'video' && (
               <Video
                 preview={media.get('preview_url')}
                 blurhash={media.get('blurhash')}
                 src={media.get('url')}
                 detailed
+                inline
+                editable
+              />
+            )}
+
+            {media.get('type') === 'audio' && (
+              <Audio
+                src={media.get('url')}
+                duration={media.getIn(['meta', 'original', 'duration'], 0)}
+                height={150}
+                preload
                 editable
               />
             )}

app/javascript/mastodon/features/ui/util/async-components.js

@@ -137,3 +137,7 @@ export function Search () {
 export function Tesseract () {
   return import(/*webpackChunkName: "tesseract" */'tesseract.js');
 }
+
+export function Audio () {
+  return import(/* webpackChunkName: "features/audio" */'../../audio');
+}

app/javascript/mastodon/features/video/index.js

@@ -21,7 +21,7 @@ const messages = defineMessages({
   exit_fullscreen: { id: 'video.exit_fullscreen', defaultMessage: 'Exit full screen' },
 });
 
-const formatTime = secondsNum => {
+export const formatTime = secondsNum => {
   let hours   = Math.floor(secondsNum / 3600);
   let minutes = Math.floor((secondsNum - (hours * 3600)) / 60);
   let seconds = secondsNum - (hours * 3600) - (minutes * 60);

app/javascript/styles/mastodon/components.scss

@@ -948,7 +948,8 @@
   opacity: 1;
   animation: fade 150ms linear;
 
-  .video-player {
+  .video-player,
+  .audio-player {
     margin-top: 8px;
   }
 
@@ -1043,7 +1044,8 @@
       white-space: normal;
     }
 
-    .video-player {
+    .video-player,
+    .audio-player {
       margin-top: 8px;
       max-width: 250px;
     }
@@ -1154,7 +1156,8 @@
     }
   }
 
-  .video-player {
+  .video-player,
+  .audio-player {
     margin-top: 8px;
   }
 }
@@ -2130,7 +2133,8 @@ a.account__display-name {
       padding: 15px;
 
       .media-gallery,
-      .video-player {
+      .video-player,
+      .audio-player {
         margin-top: 15px;
       }
     }
@@ -2172,7 +2176,8 @@ a.account__display-name {
 
       .media-gallery,
       &__action-bar,
-      .video-player {
+      .video-player,
+      .audio-player {
         margin-top: 10px;
       }
     }
@@ -5043,15 +5048,50 @@ a.status-card.compact:hover {
 
 }
 
+.audio-player {
+  box-sizing: border-box;
+  position: relative;
+  background: darken($ui-base-color, 8%);
+  border-radius: 4px;
+  padding-bottom: 44px;
+
+  &.editable {
+    border-radius: 0;
+    height: 100%;
+  }
+
+  &__waveform {
+    padding: 15px 0;
+  }
+
+  &__progress-placeholder {
+    background-color: rgba(lighten($ui-highlight-color, 8%), 0.5);
+  }
+
+  &__wave-placeholder {
+    background-color: lighten($ui-base-color, 16%);
+  }
+
+  .video-player__controls {
+    padding: 0 15px;
+    padding-top: 10px;
+    background: darken($ui-base-color, 8%);
+    border-top: 1px solid lighten($ui-base-color, 4%);
+    border-radius: 0 0 4px 4px;
+  }
+}
+
 .video-player {
   overflow: hidden;
   position: relative;
   background: $base-shadow-color;
   max-width: 100%;
   border-radius: 4px;
+  box-sizing: border-box;
 
   &.editable {
     border-radius: 0;
+    height: 100% !important;
   }
 
   &:focus {

app/views/statuses/_detailed_status.html.haml

@@ -27,10 +27,14 @@
           = render partial: 'statuses/poll', locals: { status: status, poll: status.preloadable_poll, autoplay: autoplay }
 
   - if !status.media_attachments.empty?
-    - if status.media_attachments.first.audio_or_video?
+    - if status.media_attachments.first.video?
       - video = status.media_attachments.first
       = react_component :video, src: video.file.url(:original), preview: video.file.url(:small), blurhash: video.blurhash, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, width: 670, height: 380, detailed: true, inline: true, alt: video.description do
         = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments }
+    - elsif status.media_attachments.first.audio?
+      - audio = status.media_attachments.first
+      = react_component :audio, src: audio.file.url(:original), height: 130, alt: audio.description, preload: true, duration: audio.meta.dig(:original, :duration) do
+        = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments }
     - else
       = react_component :media_gallery, height: 380, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, standalone: true, 'autoPlayGif': current_account&.user&.setting_auto_play_gif || autoplay, 'reduceMotion': current_account&.user&.setting_reduce_motion, media: status.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json } do
         = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments }

app/views/statuses/_simple_status.html.haml

@@ -31,10 +31,14 @@
           = render partial: 'statuses/poll', locals: { status: status, poll: status.preloadable_poll, autoplay: autoplay }
 
   - if !status.media_attachments.empty?
-    - if status.media_attachments.first.audio_or_video?
+    - if status.media_attachments.first.video?
       - video = status.media_attachments.first
       = react_component :video, src: video.file.url(:original), preview: video.file.url(:small), blurhash: video.blurhash, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, width: 610, height: 343, inline: true, alt: video.description do
         = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments }
+    - elsif status.media_attachments.first.audio?
+      - audio = status.media_attachments.first
+      = react_component :audio, src: audio.file.url(:original), height: 110, alt: audio.description, duration: audio.meta.dig(:original, :duration) do
+        = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments }
     - else
       = react_component :media_gallery, height: 343, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, 'autoPlayGif': current_account&.user&.setting_auto_play_gif || autoplay, media: status.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json } do
         = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments }

package.json

@@ -160,6 +160,7 @@
     "throng": "^4.0.0",
     "tiny-queue": "^0.2.1",
     "uuid": "^3.1.0",
+    "wavesurfer.js": "^3.0.0",
     "webpack": "^4.35.3",
     "webpack-assets-manifest": "^3.1.1",
     "webpack-bundle-analyzer": "^3.3.2",

yarn.lock

@@ -10466,6 +10466,11 @@ watchpack@^1.5.0:
     graceful-fs "^4.1.2"
     neo-async "^2.5.0"
 
+wavesurfer.js@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/wavesurfer.js/-/wavesurfer.js-3.0.0.tgz#35f36d76d59c749dca453cf4e10ee0ec49f454f8"
+  integrity sha512-DANu206c6gb9pSUbYFevsSiXMy8+Ri+CNtqm0UsouUdsn9fVQRtYs8uxzBtXK+rUPlIc6FlO54DU8uWeW3lDzw==
+
 wbuf@^1.1.0, wbuf@^1.7.3:
   version "1.7.3"
   resolved "https://registry.yarnpkg.com/wbuf/-/wbuf-1.7.3.tgz#c1d8d149316d3ea852848895cb6a0bfe887b87df"