From 8eb97e8ca2e731eba7f5be2d80e7e5254929814f Mon Sep 17 00:00:00 2001
From: Yamagishi Kazutoshi <ykzts@desire.sh>
Date: Sun, 15 Nov 2020 03:19:23 +0900
Subject: [PATCH 01/31] Apply label automatically when issue creation from
 template (#15154)

---
 .github/ISSUE_TEMPLATE/bug_report.md      | 2 +-
 .github/ISSUE_TEMPLATE/feature_request.md | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
index f49964bd9f5..8ae4bb88253 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -1,7 +1,7 @@
 ---
 name: Bug Report
 about: If something isn't working as expected
-
+labels: bug
 ---
 
 <!-- Make sure that you are submitting a new bug that was not previously reported or already fixed -->
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
index 3890729e22f..ff92c0316e0 100644
--- a/.github/ISSUE_TEMPLATE/feature_request.md
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -1,7 +1,6 @@
 ---
 name: Feature Request
 about: I have a suggestion
-
 ---
 
 <!-- Please use a concise and distinct title for the issue -->

From 04a079e7230bc4e73b897e3c4462011cddf36474 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 15 Nov 2020 14:24:54 +0100
Subject: [PATCH 02/31] Add hotkeys for audio/video control (#15158)

Fix #14515
---
 .../mastodon/features/audio/index.js          | 51 +++++++++++-
 .../mastodon/features/video/index.js          | 78 +++++++++++++++++++
 2 files changed, 128 insertions(+), 1 deletion(-)

diff --git a/app/javascript/mastodon/features/audio/index.js b/app/javascript/mastodon/features/audio/index.js
index 434148e8e79..c47f55dd1ea 100644
--- a/app/javascript/mastodon/features/audio/index.js
+++ b/app/javascript/mastodon/features/audio/index.js
@@ -386,13 +386,59 @@ class Audio extends React.PureComponent {
     return this.props.foregroundColor || '#ffffff';
   }
 
+  seekBy (time) {
+    const currentTime = this.audio.currentTime + time;
+
+    if (!isNaN(currentTime)) {
+      this.setState({ currentTime }, () => {
+        this.audio.currentTime = currentTime;
+      });
+    }
+  }
+
+  handleAudioKeyDown = e => {
+    // On the audio element or the seek bar, we can safely use the space bar
+    // for playback control because there are no buttons to press
+
+    if (e.key === ' ') {
+      e.preventDefault();
+      e.stopPropagation();
+      this.togglePlay();
+    }
+  }
+
+  handleKeyDown = e => {
+    switch(e.key) {
+    case 'k':
+      e.preventDefault();
+      e.stopPropagation();
+      this.togglePlay();
+      break;
+    case 'm':
+      e.preventDefault();
+      e.stopPropagation();
+      this.toggleMute();
+      break;
+    case 'j':
+      e.preventDefault();
+      e.stopPropagation();
+      this.seekBy(-10);
+      break;
+    case 'l':
+      e.preventDefault();
+      e.stopPropagation();
+      this.seekBy(10);
+      break;
+    }
+  }
+
   render () {
     const { src, intl, alt, editable, autoPlay } = this.props;
     const { paused, muted, volume, currentTime, duration, buffer, dragging } = this.state;
     const progress = Math.min((currentTime / duration) * 100, 100);
 
     return (
-      <div className={classNames('audio-player', { editable })} ref={this.setPlayerRef} style={{ backgroundColor: this._getBackgroundColor(), color: this._getForegroundColor(), width: '100%', height: this.props.fullscreen ? '100%' : (this.state.height || this.props.height) }} onMouseEnter={this.handleMouseEnter} onMouseLeave={this.handleMouseLeave}>
+      <div className={classNames('audio-player', { editable })} ref={this.setPlayerRef} style={{ backgroundColor: this._getBackgroundColor(), color: this._getForegroundColor(), width: '100%', height: this.props.fullscreen ? '100%' : (this.state.height || this.props.height) }} onMouseEnter={this.handleMouseEnter} onMouseLeave={this.handleMouseLeave} tabIndex='0' onKeyDown={this.handleKeyDown}>
         <audio
           src={src}
           ref={this.setAudioRef}
@@ -406,12 +452,14 @@ class Audio extends React.PureComponent {
 
         <canvas
           role='button'
+          tabIndex='0'
           className='audio-player__canvas'
           width={this.state.width}
           height={this.state.height}
           style={{ width: '100%', position: 'absolute', top: 0, left: 0 }}
           ref={this.setCanvasRef}
           onClick={this.togglePlay}
+          onKeyDown={this.handleAudioKeyDown}
           title={alt}
           aria-label={alt}
         />
@@ -432,6 +480,7 @@ class Audio extends React.PureComponent {
             className={classNames('video-player__seek__handle', { active: dragging })}
             tabIndex='0'
             style={{ left: `${progress}%`, backgroundColor: this._getAccentColor() }}
+            onKeyDown={this.handleAudioKeyDown}
           />
         </div>
 
diff --git a/app/javascript/mastodon/features/video/index.js b/app/javascript/mastodon/features/video/index.js
index 229a9214084..e6c6f4b679e 100644
--- a/app/javascript/mastodon/features/video/index.js
+++ b/app/javascript/mastodon/features/video/index.js
@@ -266,6 +266,81 @@ class Video extends React.PureComponent {
     }
   }, 15);
 
+  seekBy (time) {
+    const currentTime = this.video.currentTime + time;
+
+    if (!isNaN(currentTime)) {
+      this.setState({ currentTime }, () => {
+        this.video.currentTime = currentTime;
+      });
+    }
+  }
+
+  handleVideoKeyDown = e => {
+    // On the video element or the seek bar, we can safely use the space bar
+    // for playback control because there are no buttons to press
+
+    if (e.key === ' ') {
+      e.preventDefault();
+      e.stopPropagation();
+      this.togglePlay();
+    }
+  }
+
+  handleKeyDown = e => {
+    const frameTime = 1 / 25;
+
+    switch(e.key) {
+    case 'k':
+      e.preventDefault();
+      e.stopPropagation();
+      this.togglePlay();
+      break;
+    case 'm':
+      e.preventDefault();
+      e.stopPropagation();
+      this.toggleMute();
+      break;
+    case 'f':
+      e.preventDefault();
+      e.stopPropagation();
+      this.toggleFullscreen();
+      break;
+    case 'j':
+      e.preventDefault();
+      e.stopPropagation();
+      this.seekBy(-10);
+      break;
+    case 'l':
+      e.preventDefault();
+      e.stopPropagation();
+      this.seekBy(10);
+      break;
+    case ',':
+      e.preventDefault();
+      e.stopPropagation();
+      this.seekBy(-frameTime);
+      break;
+    case '.':
+      e.preventDefault();
+      e.stopPropagation();
+      this.seekBy(frameTime);
+      break;
+    }
+
+    // If we are in fullscreen mode, we don't want any hotkeys
+    // interacting with the UI that's not visible
+
+    if (this.state.fullscreen) {
+      e.preventDefault();
+      e.stopPropagation();
+
+      if (e.key === 'Escape') {
+        exitFullscreen();
+      }
+    }
+  }
+
   togglePlay = () => {
     if (this.state.paused) {
       this.setState({ paused: false }, () => this.video.play());
@@ -484,6 +559,7 @@ class Video extends React.PureComponent {
         onMouseEnter={this.handleMouseEnter}
         onMouseLeave={this.handleMouseLeave}
         onClick={this.handleClickRoot}
+        onKeyDown={this.handleKeyDown}
         tabIndex={0}
       >
         <Blurhash
@@ -507,6 +583,7 @@ class Video extends React.PureComponent {
           height={height}
           volume={volume}
           onClick={this.togglePlay}
+          onKeyDown={this.handleVideoKeyDown}
           onPlay={this.handlePlay}
           onPause={this.handlePause}
           onLoadedData={this.handleLoadedData}
@@ -529,6 +606,7 @@ class Video extends React.PureComponent {
               className={classNames('video-player__seek__handle', { active: dragging })}
               tabIndex='0'
               style={{ left: `${progress}%` }}
+              onKeyDown={this.handleVideoKeyDown}
             />
           </div>
 

From 18ca4e0e9a3f74a6f21d329882b429f8f5227b0f Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 16 Nov 2020 05:16:39 +0100
Subject: [PATCH 03/31] Fix pop-out player appearing on mobile screens in web
 UI (#15157)

Fix #15092
---
 app/javascript/mastodon/actions/app.js        |  7 ++
 app/javascript/mastodon/components/status.js  | 15 +--
 .../mastodon/containers/status_container.js   |  6 +-
 app/javascript/mastodon/features/ui/index.js  | 91 ++++++++-----------
 app/javascript/mastodon/is_mobile.js          | 25 +++--
 app/javascript/mastodon/reducers/meta.js      |  7 +-
 6 files changed, 79 insertions(+), 72 deletions(-)

diff --git a/app/javascript/mastodon/actions/app.js b/app/javascript/mastodon/actions/app.js
index 414968f7de4..c817c87080a 100644
--- a/app/javascript/mastodon/actions/app.js
+++ b/app/javascript/mastodon/actions/app.js
@@ -8,3 +8,10 @@ export const focusApp = () => ({
 export const unfocusApp = () => ({
   type: APP_UNFOCUS,
 });
+
+export const APP_LAYOUT_CHANGE = 'APP_LAYOUT_CHANGE';
+
+export const changeLayout = layout => ({
+  type: APP_LAYOUT_CHANGE,
+  layout,
+});
diff --git a/app/javascript/mastodon/components/status.js b/app/javascript/mastodon/components/status.js
index be4f0bcca3f..f4ed25f1eb6 100644
--- a/app/javascript/mastodon/components/status.js
+++ b/app/javascript/mastodon/components/status.js
@@ -97,7 +97,10 @@ class Status extends ImmutablePureComponent {
     cachedMediaWidth: PropTypes.number,
     scrollKey: PropTypes.string,
     deployPictureInPicture: PropTypes.func,
-    usingPiP: PropTypes.bool,
+    pictureInPicture: PropTypes.shape({
+      inUse: PropTypes.bool,
+      available: PropTypes.bool,
+    }),
   };
 
   // Avoid checking props that are functions (and whose equality will always
@@ -108,7 +111,7 @@ class Status extends ImmutablePureComponent {
     'muted',
     'hidden',
     'unread',
-    'usingPiP',
+    'pictureInPicture',
   ];
 
   state = {
@@ -277,7 +280,7 @@ class Status extends ImmutablePureComponent {
     let media = null;
     let statusAvatar, prepend, rebloggedByText;
 
-    const { intl, hidden, featured, otherAccounts, unread, showThread, scrollKey, usingPiP } = this.props;
+    const { intl, hidden, featured, otherAccounts, unread, showThread, scrollKey, pictureInPicture } = this.props;
 
     let { status, account, ...other } = this.props;
 
@@ -348,7 +351,7 @@ class Status extends ImmutablePureComponent {
       status  = status.get('reblog');
     }
 
-    if (usingPiP) {
+    if (pictureInPicture.inUse) {
       media = <PictureInPicturePlaceholder width={this.props.cachedMediaWidth} />;
     } else if (status.get('media_attachments').size > 0) {
       if (this.props.muted) {
@@ -375,7 +378,7 @@ class Status extends ImmutablePureComponent {
                 width={this.props.cachedMediaWidth}
                 height={110}
                 cacheWidth={this.props.cacheMediaWidth}
-                deployPictureInPicture={this.handleDeployPictureInPicture}
+                deployPictureInPicture={pictureInPicture.available ? this.handleDeployPictureInPicture : undefined}
               />
             )}
           </Bundle>
@@ -397,7 +400,7 @@ class Status extends ImmutablePureComponent {
                 sensitive={status.get('sensitive')}
                 onOpenVideo={this.handleOpenVideo}
                 cacheWidth={this.props.cacheMediaWidth}
-                deployPictureInPicture={this.handleDeployPictureInPicture}
+                deployPictureInPicture={pictureInPicture.available ? this.handleDeployPictureInPicture : undefined}
                 visible={this.state.showMedia}
                 onToggleVisibility={this.handleToggleMediaVisibility}
               />
diff --git a/app/javascript/mastodon/containers/status_container.js b/app/javascript/mastodon/containers/status_container.js
index 7bfd66d3eaa..fe81981a780 100644
--- a/app/javascript/mastodon/containers/status_container.js
+++ b/app/javascript/mastodon/containers/status_container.js
@@ -57,7 +57,11 @@ const makeMapStateToProps = () => {
 
   const mapStateToProps = (state, props) => ({
     status: getStatus(state, props),
-    usingPiP: state.get('picture_in_picture').statusId === props.id,
+
+    pictureInPicture: {
+      inUse: state.getIn(['meta', 'layout']) !== 'mobile' && state.get('picture_in_picture').statusId === props.id,
+      available: state.getIn(['meta', 'layout']) !== 'mobile',
+    },
   });
 
   return mapStateToProps;
diff --git a/app/javascript/mastodon/features/ui/index.js b/app/javascript/mastodon/features/ui/index.js
index c6df49a5fb7..507ac1df146 100644
--- a/app/javascript/mastodon/features/ui/index.js
+++ b/app/javascript/mastodon/features/ui/index.js
@@ -8,14 +8,14 @@ import PropTypes from 'prop-types';
 import NotificationsContainer from './containers/notifications_container';
 import LoadingBarContainer from './containers/loading_bar_container';
 import ModalContainer from './containers/modal_container';
-import { isMobile } from '../../is_mobile';
+import { layoutFromWindow } from 'mastodon/is_mobile';
 import { debounce } from 'lodash';
 import { uploadCompose, resetCompose, changeComposeSpoilerness } from '../../actions/compose';
 import { expandHomeTimeline } from '../../actions/timelines';
 import { expandNotifications } from '../../actions/notifications';
 import { fetchFilters } from '../../actions/filters';
 import { clearHeight } from '../../actions/height_cache';
-import { focusApp, unfocusApp } from 'mastodon/actions/app';
+import { focusApp, unfocusApp, changeLayout } from 'mastodon/actions/app';
 import { synchronouslySubmitMarkers, submitMarkers, fetchMarkers } from 'mastodon/actions/markers';
 import { WrappedSwitch, WrappedRoute } from './util/react_router_helpers';
 import UploadArea from './components/upload_area';
@@ -52,7 +52,7 @@ import {
   Search,
   Directory,
 } from './util/async-components';
-import { me, forceSingleColumn } from '../../initial_state';
+import { me } from '../../initial_state';
 import { previewState as previewMediaState } from './components/media_modal';
 import { previewState as previewVideoState } from './components/video_modal';
 
@@ -65,6 +65,7 @@ const messages = defineMessages({
 });
 
 const mapStateToProps = state => ({
+  layout: state.getIn(['meta', 'layout']),
   isComposing: state.getIn(['compose', 'is_composing']),
   hasComposingText: state.getIn(['compose', 'text']).trim().length !== 0,
   hasMediaAttachments: state.getIn(['compose', 'media_attachments']).size > 0,
@@ -110,17 +111,11 @@ class SwitchingColumnsArea extends React.PureComponent {
   static propTypes = {
     children: PropTypes.node,
     location: PropTypes.object,
-    onLayoutChange: PropTypes.func.isRequired,
-  };
-
-  state = {
-    mobile: isMobile(window.innerWidth),
+    mobile: PropTypes.bool,
   };
 
   componentWillMount () {
-    window.addEventListener('resize', this.handleResize, { passive: true });
-
-    if (this.state.mobile || forceSingleColumn) {
+    if (this.props.mobile) {
       document.body.classList.toggle('layout-single-column', true);
       document.body.classList.toggle('layout-multiple-columns', false);
     } else {
@@ -129,44 +124,21 @@ class SwitchingColumnsArea extends React.PureComponent {
     }
   }
 
-  componentDidUpdate (prevProps, prevState) {
+  componentDidUpdate (prevProps) {
     if (![this.props.location.pathname, '/'].includes(prevProps.location.pathname)) {
       this.node.handleChildrenContentChange();
     }
 
-    if (prevState.mobile !== this.state.mobile && !forceSingleColumn) {
-      document.body.classList.toggle('layout-single-column', this.state.mobile);
-      document.body.classList.toggle('layout-multiple-columns', !this.state.mobile);
+    if (prevProps.mobile !== this.props.mobile) {
+      document.body.classList.toggle('layout-single-column', this.props.mobile);
+      document.body.classList.toggle('layout-multiple-columns', !this.props.mobile);
     }
   }
 
-  componentWillUnmount () {
-    window.removeEventListener('resize', this.handleResize);
-  }
-
   shouldUpdateScroll (_, { location }) {
     return location.state !== previewMediaState && location.state !== previewVideoState;
   }
 
-  handleLayoutChange = debounce(() => {
-    // The cached heights are no longer accurate, invalidate
-    this.props.onLayoutChange();
-  }, 500, {
-    trailing: true,
-  })
-
-  handleResize = () => {
-    const mobile = isMobile(window.innerWidth);
-
-    if (mobile !== this.state.mobile) {
-      this.handleLayoutChange.cancel();
-      this.props.onLayoutChange();
-      this.setState({ mobile });
-    } else {
-      this.handleLayoutChange();
-    }
-  }
-
   setRef = c => {
     if (c) {
       this.node = c.getWrappedInstance();
@@ -174,13 +146,11 @@ class SwitchingColumnsArea extends React.PureComponent {
   }
 
   render () {
-    const { children } = this.props;
-    const { mobile } = this.state;
-    const singleColumn = forceSingleColumn || mobile;
-    const redirect = singleColumn ? <Redirect from='/' to='/timelines/home' exact /> : <Redirect from='/' to='/getting-started' exact />;
+    const { children, mobile } = this.props;
+    const redirect = mobile ? <Redirect from='/' to='/timelines/home' exact /> : <Redirect from='/' to='/getting-started' exact />;
 
     return (
-      <ColumnsAreaContainer ref={this.setRef} singleColumn={singleColumn}>
+      <ColumnsAreaContainer ref={this.setRef} singleColumn={mobile}>
         <WrappedSwitch>
           {redirect}
           <WrappedRoute path='/getting-started' component={GettingStarted} content={children} />
@@ -244,6 +214,7 @@ class UI extends React.PureComponent {
     location: PropTypes.object,
     intl: PropTypes.object.isRequired,
     dropdownMenuIsOpen: PropTypes.bool,
+    layout: PropTypes.string.isRequired,
   };
 
   state = {
@@ -273,11 +244,6 @@ class UI extends React.PureComponent {
     this.props.dispatch(unfocusApp());
   }
 
-  handleLayoutChange = () => {
-    // The cached heights are no longer accurate, invalidate
-    this.props.dispatch(clearHeight());
-  }
-
   handleDragEnter = (e) => {
     e.preventDefault();
 
@@ -351,10 +317,28 @@ class UI extends React.PureComponent {
     }
   }
 
-  componentWillMount () {
+  handleLayoutChange = debounce(() => {
+    this.props.dispatch(clearHeight()); // The cached heights are no longer accurate, invalidate
+  }, 500, {
+    trailing: true,
+  });
+
+  handleResize = () => {
+    const layout = layoutFromWindow();
+
+    if (layout !== this.props.layout) {
+      this.handleLayoutChange.cancel();
+      this.props.dispatch(changeLayout(layout));
+    } else {
+      this.handleLayoutChange();
+    }
+  }
+
+  componentDidMount () {
     window.addEventListener('focus', this.handleWindowFocus, false);
     window.addEventListener('blur', this.handleWindowBlur, false);
     window.addEventListener('beforeunload', this.handleBeforeUnload, false);
+    window.addEventListener('resize', this.handleResize, { passive: true });
 
     document.addEventListener('dragenter', this.handleDragEnter, false);
     document.addEventListener('dragover', this.handleDragOver, false);
@@ -371,9 +355,7 @@ class UI extends React.PureComponent {
     this.props.dispatch(expandNotifications());
 
     setTimeout(() => this.props.dispatch(fetchFilters()), 500);
-  }
 
-  componentDidMount () {
     this.hotkeys.__mousetrap__.stopCallback = (e, element) => {
       return ['TEXTAREA', 'SELECT', 'INPUT'].includes(element.tagName);
     };
@@ -383,6 +365,7 @@ class UI extends React.PureComponent {
     window.removeEventListener('focus', this.handleWindowFocus);
     window.removeEventListener('blur', this.handleWindowBlur);
     window.removeEventListener('beforeunload', this.handleBeforeUnload);
+    window.removeEventListener('resize', this.handleResize);
 
     document.removeEventListener('dragenter', this.handleDragEnter);
     document.removeEventListener('dragover', this.handleDragOver);
@@ -513,7 +496,7 @@ class UI extends React.PureComponent {
 
   render () {
     const { draggingOver } = this.state;
-    const { children, isComposing, location, dropdownMenuIsOpen } = this.props;
+    const { children, isComposing, location, dropdownMenuIsOpen, layout } = this.props;
 
     const handlers = {
       help: this.handleHotkeyToggleHelp,
@@ -540,11 +523,11 @@ class UI extends React.PureComponent {
     return (
       <HotKeys keyMap={keyMap} handlers={handlers} ref={this.setHotkeysRef} attach={window} focused>
         <div className={classNames('ui', { 'is-composing': isComposing })} ref={this.setRef} style={{ pointerEvents: dropdownMenuIsOpen ? 'none' : null }}>
-          <SwitchingColumnsArea location={location} onLayoutChange={this.handleLayoutChange}>
+          <SwitchingColumnsArea location={location} mobile={layout === 'mobile' || layout === 'single-column'}>
             {children}
           </SwitchingColumnsArea>
 
-          <PictureInPicture />
+          {layout !== 'mobile' && <PictureInPicture />}
           <NotificationsContainer />
           <LoadingBarContainer className='loading-bar' />
           <ModalContainer />
diff --git a/app/javascript/mastodon/is_mobile.js b/app/javascript/mastodon/is_mobile.js
index 5a8c3db085c..2926eb4b171 100644
--- a/app/javascript/mastodon/is_mobile.js
+++ b/app/javascript/mastodon/is_mobile.js
@@ -1,9 +1,18 @@
 import { supportsPassiveEvents } from 'detect-passive-events';
+import { forceSingleColumn } from 'mastodon/initial_state';
 
 const LAYOUT_BREAKPOINT = 630;
 
-export function isMobile(width) {
-  return width <= LAYOUT_BREAKPOINT;
+export const isMobile = width => width <= LAYOUT_BREAKPOINT;
+
+export const layoutFromWindow = () => {
+  if (isMobile(window.innerWidth)) {
+    return 'mobile';
+  } else if (forceSingleColumn) {
+    return 'single-column';
+  } else {
+    return 'multi-column';
+  }
 };
 
 const iOS = /iPad|iPhone|iPod/.test(navigator.userAgent) && !window.MSStream;
@@ -11,17 +20,13 @@ const iOS = /iPad|iPhone|iPod/.test(navigator.userAgent) && !window.MSStream;
 let userTouching = false;
 let listenerOptions = supportsPassiveEvents ? { passive: true } : false;
 
-function touchListener() {
+const touchListener = () => {
   userTouching = true;
   window.removeEventListener('touchstart', touchListener, listenerOptions);
-}
+};
 
 window.addEventListener('touchstart', touchListener, listenerOptions);
 
-export function isUserTouching() {
-  return userTouching;
-}
+export const isUserTouching = () => userTouching;
 
-export function isIOS() {
-  return iOS;
-};
+export const isIOS = () => iOS;
diff --git a/app/javascript/mastodon/reducers/meta.js b/app/javascript/mastodon/reducers/meta.js
index 36a5a1c3540..65becc44f8f 100644
--- a/app/javascript/mastodon/reducers/meta.js
+++ b/app/javascript/mastodon/reducers/meta.js
@@ -1,15 +1,20 @@
-import { STORE_HYDRATE } from '../actions/store';
+import { STORE_HYDRATE } from 'mastodon/actions/store';
+import { APP_LAYOUT_CHANGE } from 'mastodon/actions/app';
 import { Map as ImmutableMap } from 'immutable';
+import { layoutFromWindow } from 'mastodon/is_mobile';
 
 const initialState = ImmutableMap({
   streaming_api_base_url: null,
   access_token: null,
+  layout: layoutFromWindow(),
 });
 
 export default function meta(state = initialState, action) {
   switch(action.type) {
   case STORE_HYDRATE:
     return state.merge(action.state.get('meta'));
+  case APP_LAYOUT_CHANGE:
+    return state.set('layout', action.layout);
   default:
     return state;
   }

From d8d43a427a549cf063de3f6b3c22a08a06f53ffa Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Wed, 18 Nov 2020 18:01:58 +0100
Subject: [PATCH 04/31] Fix image uploads being random data when canvas read
 access is blocked (#15180)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fixes #15178

PR #11499 introduced a way to deal with all-white canvas, but newer
Firefox versions set random data instead.

This PR detects whether canvas operations are reliable by comparing
the results on a hardcoded 2×2 pixels image, and memoizing the result.

This should be both more reliable and faster than the previous check.
---
 app/javascript/mastodon/utils/resize_image.js | 50 +++++++++++++++----
 1 file changed, 41 insertions(+), 9 deletions(-)

diff --git a/app/javascript/mastodon/utils/resize_image.js b/app/javascript/mastodon/utils/resize_image.js
index 6c1cb61a253..8f14853796a 100644
--- a/app/javascript/mastodon/utils/resize_image.js
+++ b/app/javascript/mastodon/utils/resize_image.js
@@ -41,6 +41,45 @@ const dropOrientationIfNeeded = (orientation) => new Promise(resolve => {
   }
 });
 
+// Some browsers don't allow reading from a canvas and instead return all-white
+// or randomized data. Use a pre-defined image to check if reading the canvas
+// works.
+const checkCanvasReliability = () => new Promise((resolve, reject) => {
+  switch(_browser_quirks['canvas-read-unreliable']) {
+  case true:
+    reject('Canvas reading unreliable');
+    break;
+  case false:
+    resolve();
+    break;
+  default:
+    // 2×2 GIF with white, red, green and blue pixels
+    const testImageURL =
+      'data:image/gif;base64,R0lGODdhAgACAKEDAAAA//8AAAD/AP///ywAAAAAAgACAAACA1wEBQA7';
+    const refData =
+      [255, 255, 255, 255,  255, 0, 0, 255,  0, 255, 0, 255,  0, 0, 255, 255];
+    const img = new Image();
+    img.onload = () => {
+      const canvas  = document.createElement('canvas');
+      const context = canvas.getContext('2d');
+      context.drawImage(img, 0, 0, 2, 2);
+      const imageData = context.getImageData(0, 0, 2, 2);
+      if (imageData.data.every((x, i) => refData[i] === x)) {
+        _browser_quirks['canvas-read-unreliable'] = false;
+        resolve();
+      } else {
+        _browser_quirks['canvas-read-unreliable'] = true;
+        reject('Canvas reading unreliable');
+      }
+    };
+    img.onerror = () => {
+      _browser_quirks['canvas-read-unreliable'] = true;
+      reject('Failed to load test image');
+    };
+    img.src = testImageURL;
+  }
+});
+
 const getImageUrl = inputFile => new Promise((resolve, reject) => {
   if (window.URL && URL.createObjectURL) {
     try {
@@ -110,14 +149,6 @@ const processImage = (img, { width, height, orientation, type = 'image/png' }) =
 
   context.drawImage(img, 0, 0, width, height);
 
-  // The Tor Browser and maybe other browsers may prevent reading from canvas
-  // and return an all-white image instead. Assume reading failed if the resized
-  // image is perfectly white.
-  const imageData = context.getImageData(0, 0, width, height);
-  if (imageData.data.every(value => value === 255)) {
-    throw 'Failed to read from canvas';
-  }
-
   canvas.toBlob(resolve, type);
 });
 
@@ -127,7 +158,8 @@ const resizeImage = (img, type = 'image/png') => new Promise((resolve, reject) =
   const newWidth  = Math.round(Math.sqrt(MAX_IMAGE_PIXELS * (width / height)));
   const newHeight = Math.round(Math.sqrt(MAX_IMAGE_PIXELS * (height / width)));
 
-  getOrientation(img, type)
+  checkCanvasReliability()
+    .then(getOrientation(img, type))
     .then(orientation => processImage(img, {
       width: newWidth,
       height: newHeight,

From 966593393ede79e66d443a6b299c3aa35611e3ef Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Thu, 19 Nov 2020 00:23:46 +0100
Subject: [PATCH 05/31] Fix DMs not appearing into timelines (#15182)

Fixes #15179
---
 app/services/fan_out_on_write_service.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb
index 4f6c64de12e..b72bb82d3fa 100644
--- a/app/services/fan_out_on_write_service.rb
+++ b/app/services/fan_out_on_write_service.rb
@@ -9,6 +9,7 @@ class FanOutOnWriteService < BaseService
     deliver_to_self(status) if status.account.local?
 
     if status.direct_visibility?
+      deliver_to_mentioned_followers(status)
       deliver_to_own_conversation(status)
     elsif status.limited_visibility?
       deliver_to_mentioned_followers(status)

From feeaa6f50d125c4af6e0ecc7ea7269a9a8ee8785 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Nov 2020 10:22:24 +0900
Subject: [PATCH 06/31] Bump oj from 3.10.15 to 3.10.16 (#15175)

Bumps [oj](https://github.com/ohler55/oj) from 3.10.15 to 3.10.16.
- [Release notes](https://github.com/ohler55/oj/releases)
- [Changelog](https://github.com/ohler55/oj/blob/develop/CHANGELOG.md)
- [Commits](https://github.com/ohler55/oj/compare/v3.10.15...v3.10.16)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index 25e386f7eff..080dfae6b15 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -367,7 +367,7 @@ GEM
       concurrent-ruby (~> 1.0, >= 1.0.2)
       sidekiq (>= 3.5)
       statsd-ruby (~> 1.4, >= 1.4.0)
-    oj (3.10.15)
+    oj (3.10.16)
     omniauth (1.9.1)
       hashie (>= 3.4.6)
       rack (>= 1.6.2, < 3)

From beddf821d7c805cac24a0f364d539b2f7f44380b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Nov 2020 10:25:12 +0900
Subject: [PATCH 07/31] Bump json-ld from 3.1.4 to 3.1.5 (#15174)

Bumps [json-ld](https://github.com/ruby-rdf/json-ld) from 3.1.4 to 3.1.5.
- [Release notes](https://github.com/ruby-rdf/json-ld/releases)
- [Commits](https://github.com/ruby-rdf/json-ld/compare/3.1.4...3.1.5)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 Gemfile.lock | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index 080dfae6b15..29d61f5b59c 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -289,7 +289,7 @@ GEM
     jmespath (1.4.0)
     json (2.3.1)
     json-canonicalization (0.2.0)
-    json-ld (3.1.4)
+    json-ld (3.1.5)
       htmlentities (~> 4.3)
       json-canonicalization (~> 0.2)
       link_header (~> 0.0, >= 0.0.8)
@@ -473,7 +473,7 @@ GEM
       thor (>= 0.19.0, < 2.0)
     rainbow (3.0.0)
     rake (13.0.1)
-    rdf (3.1.6)
+    rdf (3.1.7)
       hamster (~> 3.0)
       link_header (~> 0.0, >= 0.0.8)
     rdf-normalize (0.4.0)

From 78a395672909674f12ff40cb1f029b782284aeda Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Nov 2020 10:28:31 +0900
Subject: [PATCH 08/31] Bump aws-sdk-s3 from 1.84.0 to 1.84.1 (#15173)

Bumps [aws-sdk-s3](https://github.com/aws/aws-sdk-ruby) from 1.84.0 to 1.84.1.
- [Release notes](https://github.com/aws/aws-sdk-ruby/releases)
- [Changelog](https://github.com/aws/aws-sdk-ruby/blob/master/gems/aws-sdk-s3/CHANGELOG.md)
- [Commits](https://github.com/aws/aws-sdk-ruby/commits)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 Gemfile.lock | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index 29d61f5b59c..0cca4fb3ac0 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -79,7 +79,7 @@ GEM
       cocaine (~> 0.5.3)
     awrence (1.1.1)
     aws-eventstream (1.1.0)
-    aws-partitions (1.390.0)
+    aws-partitions (1.393.0)
     aws-sdk-core (3.109.2)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
@@ -88,7 +88,7 @@ GEM
     aws-sdk-kms (1.39.0)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.84.0)
+    aws-sdk-s3 (1.84.1)
       aws-sdk-core (~> 3, >= 3.109.0)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.1)

From d942eb23feb5cb31c852d933b6ebb43a150d36b6 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Nov 2020 10:32:00 +0900
Subject: [PATCH 09/31] Bump compression-webpack-plugin from 6.1.0 to 6.1.1
 (#15161)

Bumps [compression-webpack-plugin](https://github.com/webpack-contrib/compression-webpack-plugin) from 6.1.0 to 6.1.1.
- [Release notes](https://github.com/webpack-contrib/compression-webpack-plugin/releases)
- [Changelog](https://github.com/webpack-contrib/compression-webpack-plugin/blob/master/CHANGELOG.md)
- [Commits](https://github.com/webpack-contrib/compression-webpack-plugin/compare/v6.1.0...v6.1.1)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 package.json | 2 +-
 yarn.lock    | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/package.json b/package.json
index 2375a9ecd0a..f8f8ddda430 100644
--- a/package.json
+++ b/package.json
@@ -83,7 +83,7 @@
     "babel-runtime": "^6.26.0",
     "blurhash": "^1.1.3",
     "classnames": "^2.2.5",
-    "compression-webpack-plugin": "^6.1.0",
+    "compression-webpack-plugin": "^6.1.1",
     "cross-env": "^7.0.2",
     "css-loader": "^5.0.1",
     "cssnano": "^4.1.10",
diff --git a/yarn.lock b/yarn.lock
index d625b2dfe52..0e5182eca56 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -3005,10 +3005,10 @@ compressible@~2.0.16:
   dependencies:
     mime-db ">= 1.43.0 < 2"
 
-compression-webpack-plugin@^6.1.0:
-  version "6.1.0"
-  resolved "https://registry.yarnpkg.com/compression-webpack-plugin/-/compression-webpack-plugin-6.1.0.tgz#ef88a4c35e240aa14bec6cccc0582ed47e148605"
-  integrity sha512-RK/bBW3JwQpb7tH91trro7ulNa0ynSTPxQO48rn/oS1Y2nGUYuX6CWIOqbhUF2+b+2clqJeDGIYYckvg6WKabA==
+compression-webpack-plugin@^6.1.1:
+  version "6.1.1"
+  resolved "https://registry.yarnpkg.com/compression-webpack-plugin/-/compression-webpack-plugin-6.1.1.tgz#ae8e4b2ffdb7396bb776e66918d751a20d8ccf0e"
+  integrity sha512-BEHft9M6lwOqVIQFMS/YJGmeCYXVOakC5KzQk05TFpMBlODByh1qNsZCWjUBxCQhUP9x0WfGidxTbGkjbWO/TQ==
   dependencies:
     cacache "^15.0.5"
     find-cache-dir "^3.3.1"

From 3d98741639e55838619ef3e2801438bd3b83fcd5 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Nov 2020 10:34:31 +0900
Subject: [PATCH 10/31] Bump @testing-library/jest-dom from 5.11.5 to 5.11.6
 (#15162)

Bumps [@testing-library/jest-dom](https://github.com/testing-library/jest-dom) from 5.11.5 to 5.11.6.
- [Release notes](https://github.com/testing-library/jest-dom/releases)
- [Changelog](https://github.com/testing-library/jest-dom/blob/master/CHANGELOG.md)
- [Commits](https://github.com/testing-library/jest-dom/compare/v5.11.5...v5.11.6)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 package.json | 2 +-
 yarn.lock    | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/package.json b/package.json
index f8f8ddda430..48e3fa8d407 100644
--- a/package.json
+++ b/package.json
@@ -170,7 +170,7 @@
     "wicg-inert": "^3.1.0"
   },
   "devDependencies": {
-    "@testing-library/jest-dom": "^5.11.5",
+    "@testing-library/jest-dom": "^5.11.6",
     "@testing-library/react": "^11.1.1",
     "babel-eslint": "^10.1.0",
     "babel-jest": "^26.6.3",
diff --git a/yarn.lock b/yarn.lock
index 0e5182eca56..8608cdbf536 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1355,10 +1355,10 @@
     lz-string "^1.4.4"
     pretty-format "^26.4.2"
 
-"@testing-library/jest-dom@^5.11.5":
-  version "5.11.5"
-  resolved "https://registry.yarnpkg.com/@testing-library/jest-dom/-/jest-dom-5.11.5.tgz#44010f37f4b1e15f9d433963b515db0b05182fc8"
-  integrity sha512-XI+ClHR864i6p2kRCEyhvpVejuer+ObVUF4cjCvRSF88eOMIfqw7RoS9+qoRhyigGswMfT64L6Nt0Ufotxbwtg==
+"@testing-library/jest-dom@^5.11.6":
+  version "5.11.6"
+  resolved "https://registry.yarnpkg.com/@testing-library/jest-dom/-/jest-dom-5.11.6.tgz#782940e82e5cd17bc0a36f15156ba16f3570ac81"
+  integrity sha512-cVZyUNRWwUKI0++yepYpYX7uhrP398I+tGz4zOlLVlUYnZS+Svuxv4fwLeCIy7TnBYKXUaOlQr3vopxL8ZfEnA==
   dependencies:
     "@babel/runtime" "^7.9.2"
     "@types/testing-library__jest-dom" "^5.9.1"

From 9b188a65bee0e61fdde8c2932c096ad4ca2afb23 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Nov 2020 10:42:05 +0900
Subject: [PATCH 11/31] Bump webmock from 3.9.5 to 3.10.0 (#15172)

Bumps [webmock](https://github.com/bblimke/webmock) from 3.9.5 to 3.10.0.
- [Release notes](https://github.com/bblimke/webmock/releases)
- [Changelog](https://github.com/bblimke/webmock/blob/master/CHANGELOG.md)
- [Commits](https://github.com/bblimke/webmock/compare/v3.9.5...v3.10.0)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 Gemfile      | 2 +-
 Gemfile.lock | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Gemfile b/Gemfile
index 7d21d41d857..26bc6fd524c 100644
--- a/Gemfile
+++ b/Gemfile
@@ -125,7 +125,7 @@ group :test do
   gem 'rails-controller-testing', '~> 1.0'
   gem 'rspec-sidekiq', '~> 3.1'
   gem 'simplecov', '~> 0.19', require: false
-  gem 'webmock', '~> 3.9'
+  gem 'webmock', '~> 3.10'
   gem 'parallel_tests', '~> 3.3'
   gem 'rspec_junit_formatter', '~> 0.4'
 end
diff --git a/Gemfile.lock b/Gemfile.lock
index 0cca4fb3ac0..c385cf4931e 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -649,7 +649,7 @@ GEM
       safety_net_attestation (~> 0.4.0)
       securecompare (~> 1.0)
       tpm-key_attestation (~> 0.9.0)
-    webmock (3.9.5)
+    webmock (3.10.0)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
@@ -798,7 +798,7 @@ DEPENDENCIES
   twitter-text (~> 1.14)
   tzinfo-data (~> 1.2020)
   webauthn (~> 3.0.0.alpha1)
-  webmock (~> 3.9)
+  webmock (~> 3.10)
   webpacker (~> 5.2)
   webpush
   xorcist (~> 1.1)

From 34ced3da9f4f5fed15091f99b5764c0e864a8120 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Nov 2020 10:44:38 +0900
Subject: [PATCH 12/31] Bump yargs from 16.1.0 to 16.1.1 (#15163)

Bumps [yargs](https://github.com/yargs/yargs) from 16.1.0 to 16.1.1.
- [Release notes](https://github.com/yargs/yargs/releases)
- [Changelog](https://github.com/yargs/yargs/blob/master/CHANGELOG.md)
- [Commits](https://github.com/yargs/yargs/compare/v16.1.0...v16.1.1)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 package.json |  2 +-
 yarn.lock    | 18 +++++++++---------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/package.json b/package.json
index 48e3fa8d407..2d60ed432bf 100644
--- a/package.json
+++ b/package.json
@@ -185,7 +185,7 @@
     "react-test-renderer": "^16.14.0",
     "sass-lint": "^1.13.1",
     "webpack-dev-server": "^3.11.0",
-    "yargs": "^16.1.0"
+    "yargs": "^16.1.1"
   },
   "resolutions": {
     "kind-of": "^6.0.3"
diff --git a/yarn.lock b/yarn.lock
index 8608cdbf536..ee3bde61a35 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -11284,10 +11284,10 @@ y18n@^4.0.0:
   resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.0.tgz#95ef94f85ecc81d007c264e190a120f0a3c8566b"
   integrity sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==
 
-y18n@^5.0.2:
-  version "5.0.4"
-  resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.4.tgz#0ab2db89dd5873b5ec4682d8e703e833373ea897"
-  integrity sha512-deLOfD+RvFgrpAmSZgfGdWYE+OKyHcVHaRQ7NphG/63scpRvTHHeQMAxGGvaLVGJ+HYVcCXlzcTK0ZehFf+eHQ==
+y18n@^5.0.5:
+  version "5.0.5"
+  resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.5.tgz#8769ec08d03b1ea2df2500acef561743bbb9ab18"
+  integrity sha512-hsRUr4FFrvhhRH12wOdfs38Gy7k2FFzB9qgN9v3aLykRq0dRcdcpz5C9FxdS2NuhOrI/628b/KSTJ3rwHysYSg==
 
 yallist@^3.0.2:
   version "3.1.1"
@@ -11358,17 +11358,17 @@ yargs@^15.4.1:
     y18n "^4.0.0"
     yargs-parser "^18.1.2"
 
-yargs@^16.1.0:
-  version "16.1.0"
-  resolved "https://registry.yarnpkg.com/yargs/-/yargs-16.1.0.tgz#fc333fe4791660eace5a894b39d42f851cd48f2a"
-  integrity sha512-upWFJOmDdHN0syLuESuvXDmrRcWd1QafJolHskzaw79uZa7/x53gxQKiR07W59GWY1tFhhU/Th9DrtSfpS782g==
+yargs@^16.1.1:
+  version "16.1.1"
+  resolved "https://registry.yarnpkg.com/yargs/-/yargs-16.1.1.tgz#5a4a095bd1ca806b0a50d0c03611d38034d219a1"
+  integrity sha512-hAD1RcFP/wfgfxgMVswPE+z3tlPFtxG8/yWUrG2i17sTWGCGqWnxKcLTF4cUKDUK8fzokwsmO9H0TDkRbMHy8w==
   dependencies:
     cliui "^7.0.2"
     escalade "^3.1.1"
     get-caller-file "^2.0.5"
     require-directory "^2.1.1"
     string-width "^4.2.0"
-    y18n "^5.0.2"
+    y18n "^5.0.5"
     yargs-parser "^20.2.2"
 
 zlibjs@^0.3.1:

From a3f3a9113dd02cccfe2af6142892df176b3ac15b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Nov 2020 10:45:57 +0900
Subject: [PATCH 13/31] Bump bootsnap from 1.5.0 to 1.5.1 (#15171)

Bumps [bootsnap](https://github.com/Shopify/bootsnap) from 1.5.0 to 1.5.1.
- [Release notes](https://github.com/Shopify/bootsnap/releases)
- [Changelog](https://github.com/Shopify/bootsnap/blob/master/CHANGELOG.md)
- [Commits](https://github.com/Shopify/bootsnap/compare/v1.5.0...v1.5.1)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index c385cf4931e..eaf6ebf1d57 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -104,7 +104,7 @@ GEM
       debug_inspector (>= 0.0.1)
     blurhash (0.1.4)
       ffi (~> 1.10.0)
-    bootsnap (1.5.0)
+    bootsnap (1.5.1)
       msgpack (~> 1.0)
     brakeman (4.10.0)
     browser (4.2.0)

From e77bd9fc8acae99fd30f99616be5eea183204782 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Nov 2020 10:56:04 +0900
Subject: [PATCH 14/31] Bump mini-css-extract-plugin from 1.3.0 to 1.3.1
 (#15168)

Bumps [mini-css-extract-plugin](https://github.com/webpack-contrib/mini-css-extract-plugin) from 1.3.0 to 1.3.1.
- [Release notes](https://github.com/webpack-contrib/mini-css-extract-plugin/releases)
- [Changelog](https://github.com/webpack-contrib/mini-css-extract-plugin/blob/master/CHANGELOG.md)
- [Commits](https://github.com/webpack-contrib/mini-css-extract-plugin/compare/v1.3.0...v1.3.1)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 package.json | 2 +-
 yarn.lock    | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/package.json b/package.json
index 2d60ed432bf..b9c7d233aab 100644
--- a/package.json
+++ b/package.json
@@ -110,7 +110,7 @@
     "lodash": "^4.17.19",
     "mark-loader": "^0.1.6",
     "marky": "^1.2.1",
-    "mini-css-extract-plugin": "^1.3.0",
+    "mini-css-extract-plugin": "^1.3.1",
     "mkdirp": "^1.0.4",
     "npmlog": "^4.1.2",
     "object-assign": "^4.1.1",
diff --git a/yarn.lock b/yarn.lock
index ee3bde61a35..a6533541177 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -7018,10 +7018,10 @@ min-indent@^1.0.0:
   resolved "https://registry.yarnpkg.com/min-indent/-/min-indent-1.0.1.tgz#a63f681673b30571fbe8bc25686ae746eefa9869"
   integrity sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==
 
-mini-css-extract-plugin@^1.3.0:
-  version "1.3.0"
-  resolved "https://registry.yarnpkg.com/mini-css-extract-plugin/-/mini-css-extract-plugin-1.3.0.tgz#bbcba978b68c39f0a9c75822cfb2874f9cf6b018"
-  integrity sha512-4DKmPwFd0XKlwoqvrkLi2X8Mlosh2ey/E/OVAucnPUdzGqrSWHgSqed/p4Ue2Q39JjIvcdSDgmZDO6mir5Ovmw==
+mini-css-extract-plugin@^1.3.1:
+  version "1.3.1"
+  resolved "https://registry.yarnpkg.com/mini-css-extract-plugin/-/mini-css-extract-plugin-1.3.1.tgz#1375c88b2bc2a9d197670a55761edcd1b5d72f21"
+  integrity sha512-jIOheqh9EU98rqj6ZaFTYNNDSFqdakNqaUZfkYwaXPjI9batmXVXX+K71NrqRAgtoGefELBMld1EQ7dqSAD5SQ==
   dependencies:
     loader-utils "^2.0.0"
     schema-utils "^3.0.0"

From 8d520d4427427292de174eef8171f440936e1d06 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Nov 2020 11:03:07 +0900
Subject: [PATCH 15/31] Bump sass-loader from 10.0.5 to 10.1.0 (#15164)

Bumps [sass-loader](https://github.com/webpack-contrib/sass-loader) from 10.0.5 to 10.1.0.
- [Release notes](https://github.com/webpack-contrib/sass-loader/releases)
- [Changelog](https://github.com/webpack-contrib/sass-loader/blob/master/CHANGELOG.md)
- [Commits](https://github.com/webpack-contrib/sass-loader/compare/v10.0.5...v10.1.0)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 package.json | 2 +-
 yarn.lock    | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/package.json b/package.json
index b9c7d233aab..f6fb2528f61 100644
--- a/package.json
+++ b/package.json
@@ -153,7 +153,7 @@
     "reselect": "^4.0.0",
     "rimraf": "^3.0.2",
     "sass": "^1.29.0",
-    "sass-loader": "^10.0.5",
+    "sass-loader": "^10.1.0",
     "stacktrace-js": "^2.0.2",
     "stringz": "^2.1.0",
     "substring-trie": "^1.0.2",
diff --git a/yarn.lock b/yarn.lock
index a6533541177..cecebe88b5e 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -9425,10 +9425,10 @@ sass-lint@^1.13.1:
     path-is-absolute "^1.0.0"
     util "^0.10.3"
 
-sass-loader@^10.0.5:
-  version "10.0.5"
-  resolved "https://registry.yarnpkg.com/sass-loader/-/sass-loader-10.0.5.tgz#f53505b5ddbedf43797470ceb34066ded82bb769"
-  integrity sha512-2LqoNPtKkZq/XbXNQ4C64GFEleSEHKv6NPSI+bMC/l+jpEXGJhiRYkAQToO24MR7NU4JRY2RpLpJ/gjo2Uf13w==
+sass-loader@^10.1.0:
+  version "10.1.0"
+  resolved "https://registry.yarnpkg.com/sass-loader/-/sass-loader-10.1.0.tgz#1727fcc0c32ab3eb197cda61d78adf4e9174a4b3"
+  integrity sha512-ZCKAlczLBbFd3aGAhowpYEy69Te3Z68cg8bnHHl6WnSCvnKpbM6pQrz957HWMa8LKVuhnD9uMplmMAHwGQtHeg==
   dependencies:
     klona "^2.0.4"
     loader-utils "^2.0.0"

From 60cc3a96d8d7fb3253dc93356009e52ccde2fc05 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Nov 2020 11:07:57 +0900
Subject: [PATCH 16/31] Bump webpack-merge from 5.3.0 to 5.4.0 (#15166)

Bumps [webpack-merge](https://github.com/survivejs/webpack-merge) from 5.3.0 to 5.4.0.
- [Release notes](https://github.com/survivejs/webpack-merge/releases)
- [Changelog](https://github.com/survivejs/webpack-merge/blob/develop/CHANGELOG.md)
- [Commits](https://github.com/survivejs/webpack-merge/compare/v5.3.0...v5.4.0)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 package.json | 2 +-
 yarn.lock    | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/package.json b/package.json
index f6fb2528f61..d31b0de3fab 100644
--- a/package.json
+++ b/package.json
@@ -166,7 +166,7 @@
     "webpack-assets-manifest": "^3.1.1",
     "webpack-bundle-analyzer": "^4.1.0",
     "webpack-cli": "^3.3.12",
-    "webpack-merge": "^5.3.0",
+    "webpack-merge": "^5.4.0",
     "wicg-inert": "^3.1.0"
   },
   "devDependencies": {
diff --git a/yarn.lock b/yarn.lock
index cecebe88b5e..b477aae3e8c 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -11061,10 +11061,10 @@ webpack-log@^2.0.0:
     ansi-colors "^3.0.0"
     uuid "^3.3.2"
 
-webpack-merge@^5.3.0:
-  version "5.3.0"
-  resolved "https://registry.yarnpkg.com/webpack-merge/-/webpack-merge-5.3.0.tgz#a80df44d35fabace680bf430a19fda9ec49ed8eb"
-  integrity sha512-4PtsBAWnmJULIJYviiPq4BxwAykbAgGMheyEVaemj2bJI54h+p/gnlbXZEH2EM0IYC3blOE1Qm6kzKlc06N1UQ==
+webpack-merge@^5.4.0:
+  version "5.4.0"
+  resolved "https://registry.yarnpkg.com/webpack-merge/-/webpack-merge-5.4.0.tgz#81bef0a7d23fc1e6c24b06ad8bf22ddeb533a3a3"
+  integrity sha512-/scBgu8LVPlHDgqH95Aw1xS+L+PHrpHKOwYVGFaNOQl4Q4wwwWDarwB1WdZAbLQ24SKhY3Awe7VZGYAdp+N+gQ==
   dependencies:
     clone-deep "^4.0.1"
     wildcard "^2.0.0"

From 29e76f994e3abf0146da1be3ddda9e0893f7efa5 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Nov 2020 11:08:20 +0900
Subject: [PATCH 17/31] Bump babel-loader from 8.1.0 to 8.2.1 (#15167)

Bumps [babel-loader](https://github.com/babel/babel-loader) from 8.1.0 to 8.2.1.
- [Release notes](https://github.com/babel/babel-loader/releases)
- [Changelog](https://github.com/babel/babel-loader/blob/main/CHANGELOG.md)
- [Commits](https://github.com/babel/babel-loader/compare/v8.1.0...v8.2.1)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 package.json |  2 +-
 yarn.lock    | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/package.json b/package.json
index d31b0de3fab..3129edeb2bb 100644
--- a/package.json
+++ b/package.json
@@ -75,7 +75,7 @@
     "arrow-key-navigation": "^1.2.0",
     "autoprefixer": "^9.8.6",
     "axios": "^0.21.0",
-    "babel-loader": "^8.1.0",
+    "babel-loader": "^8.2.1",
     "babel-plugin-lodash": "^3.3.4",
     "babel-plugin-preval": "^5.0.0",
     "babel-plugin-react-intl": "^6.2.0",
diff --git a/yarn.lock b/yarn.lock
index b477aae3e8c..a36877bb14b 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2148,14 +2148,14 @@ babel-jest@^26.6.3:
     graceful-fs "^4.2.4"
     slash "^3.0.0"
 
-babel-loader@^8.1.0:
-  version "8.1.0"
-  resolved "https://registry.yarnpkg.com/babel-loader/-/babel-loader-8.1.0.tgz#c611d5112bd5209abe8b9fa84c3e4da25275f1c3"
-  integrity sha512-7q7nC1tYOrqvUrN3LQK4GwSk/TQorZSOlO9C+RZDZpODgyN4ZlCqE5q9cDsyWOliN+aU9B4JX01xK9eJXowJLw==
+babel-loader@^8.2.1:
+  version "8.2.1"
+  resolved "https://registry.yarnpkg.com/babel-loader/-/babel-loader-8.2.1.tgz#e53313254677e86f27536f5071d807e01d24ec00"
+  integrity sha512-dMF8sb2KQ8kJl21GUjkW1HWmcsL39GOV5vnzjqrCzEPNY0S0UfMLnumidiwIajDSBmKhYf5iRW+HXaM4cvCKBw==
   dependencies:
     find-cache-dir "^2.1.0"
     loader-utils "^1.4.0"
-    mkdirp "^0.5.3"
+    make-dir "^2.1.0"
     pify "^4.0.1"
     schema-utils "^2.6.5"
 
@@ -6842,7 +6842,7 @@ lz-string@^1.4.4:
   resolved "https://registry.yarnpkg.com/lz-string/-/lz-string-1.4.4.tgz#c0d8eaf36059f705796e1e344811cf4c498d3a26"
   integrity sha1-wNjq82BZ9wV5bh40SBHPTEmNOiY=
 
-make-dir@^2.0.0:
+make-dir@^2.0.0, make-dir@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-2.1.0.tgz#5f0310e18b8be898cc07009295a30ae41e91e6f5"
   integrity sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==

From 1242e57c270e9ff356e8c175670d5dc3a10ad273 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Thu, 19 Nov 2020 17:37:49 +0100
Subject: [PATCH 18/31] Deal with collation-related index corruption (#14860)

* Add tootctl maintenance fix-duplicates

This tool goes through the database to detect and fix duplicates.
This operation is very slow and may cause data loss (of data that would be
inaccessible without intervention because of the existing index corruptions).
It tries its best to make sensible decisions, and asks the user in some cases.

* Add warning message in db:migrate hook

* Clear Rails cache after being done with database deduplication

Avoids followers hash cache being incorrect, among other things
---
 lib/cli.rb                      |   4 +
 lib/mastodon/maintenance_cli.rb | 610 ++++++++++++++++++++++++++++++++
 lib/tasks/db.rake               |  11 +
 3 files changed, 625 insertions(+)
 create mode 100644 lib/mastodon/maintenance_cli.rb

diff --git a/lib/cli.rb b/lib/cli.rb
index 2a4dd11b217..3f1658566b3 100644
--- a/lib/cli.rb
+++ b/lib/cli.rb
@@ -14,6 +14,7 @@ require_relative 'mastodon/cache_cli'
 require_relative 'mastodon/upgrade_cli'
 require_relative 'mastodon/email_domain_blocks_cli'
 require_relative 'mastodon/ip_blocks_cli'
+require_relative 'mastodon/maintenance_cli'
 require_relative 'mastodon/version'
 
 module Mastodon
@@ -61,6 +62,9 @@ module Mastodon
     desc 'ip_blocks SUBCOMMAND ...ARGS', 'Manage IP blocks'
     subcommand 'ip_blocks', Mastodon::IpBlocksCLI
 
+    desc 'maintenance SUBCOMMAND ...ARGS', 'Various maintenance utilities'
+    subcommand 'maintenance', Mastodon::MaintenanceCLI
+
     option :dry_run, type: :boolean
     desc 'self-destruct', 'Erase the server from the federation'
     long_desc <<~LONG_DESC
diff --git a/lib/mastodon/maintenance_cli.rb b/lib/mastodon/maintenance_cli.rb
new file mode 100644
index 00000000000..191a3b03fde
--- /dev/null
+++ b/lib/mastodon/maintenance_cli.rb
@@ -0,0 +1,610 @@
+# frozen_string_literal: true
+
+require 'tty-prompt'
+require_relative '../../config/boot'
+require_relative '../../config/environment'
+require_relative 'cli_helper'
+
+module Mastodon
+  class MaintenanceCLI < Thor
+    include CLIHelper
+
+    def self.exit_on_failure?
+      true
+    end
+
+    MIN_SUPPORTED_VERSION = 2019_10_01_213028
+    MAX_SUPPORTED_VERSION = 2020_10_17_234926
+
+    # Stubs to enjoy ActiveRecord queries while not depending on a particular
+    # version of the code/database
+
+    class Status < ApplicationRecord; end
+    class StatusPin < ApplicationRecord; end
+    class Poll < ApplicationRecord; end
+    class Report < ApplicationRecord; end
+    class Tombstone < ApplicationRecord; end
+    class Favourite < ApplicationRecord; end
+    class Follow < ApplicationRecord; end
+    class FollowRequest < ApplicationRecord; end
+    class Block < ApplicationRecord; end
+    class Mute < ApplicationRecord; end
+    class AccountIdentityProof < ApplicationRecord; end
+    class AccountModerationNote < ApplicationRecord; end
+    class AccountPin < ApplicationRecord; end
+    class ListAccount < ApplicationRecord; end
+    class PollVote < ApplicationRecord; end
+    class Mention < ApplicationRecord; end
+    class AccountDomainBlock < ApplicationRecord; end
+    class AnnouncementReaction < ApplicationRecord; end
+    class FeaturedTag < ApplicationRecord; end
+    class CustomEmoji < ApplicationRecord; end
+    class CustomEmojiCategory < ApplicationRecord; end
+    class Bookmark < ApplicationRecord; end
+    class WebauthnCredential < ApplicationRecord; end
+
+    class PreviewCard < ApplicationRecord
+      self.inheritance_column = false
+    end
+
+    class MediaAttachment < ApplicationRecord
+      self.inheritance_column = nil
+    end
+
+    class AccountStat < ApplicationRecord
+      belongs_to :account, inverse_of: :account_stat
+    end
+
+    class Account < ApplicationRecord
+      # Dummy class, to make migration possible across version changes
+      has_one :user, inverse_of: :account
+      has_one :account_stat, inverse_of: :account
+
+      scope :local, -> { where(domain: nil) }
+
+      def local?
+        domain.nil?
+      end
+
+      def acct
+        local? ? username : "#{username}@#{domain}"
+      end
+    end
+
+    class User < ApplicationRecord
+      belongs_to :account, inverse_of: :user
+    end
+
+    desc 'fix-duplicates', 'Fix duplicates in database and rebuild indexes'
+    long_desc <<~LONG_DESC
+      Delete or merge duplicate accounts, statuses, emojis, etc. and rebuild indexes.
+
+      This is useful if your database indexes are corrupted because of issues such as https://wiki.postgresql.org/wiki/Locale_data_changes
+
+      Mastodon has to be stopped to run this task, which will take a long time and may be destructive.
+    LONG_DESC
+    def fix_duplicates
+      @prompt = TTY::Prompt.new
+
+      if ActiveRecord::Migrator.current_version < MIN_SUPPORTED_VERSION
+        @prompt.warn 'Your version of the database schema is too old and is not supported by this script.'
+        @prompt.warn 'Please update to at least Mastodon 3.0.0 before running this script.'
+        exit(1)
+      elsif ActiveRecord::Migrator.current_version > MAX_SUPPORTED_VERSION
+        @prompt.warn 'Your version of the database schema is more recent than this script, this may cause unexpected errors.'
+        exit(1) unless @prompt.yes?('Continue anyway?')
+      end
+
+      @prompt.warn 'This task will take a long time to run and is potentially destructive.'
+      @prompt.warn 'Please make sure to stop Mastodon and have a backup.'
+      exit(1) unless @prompt.yes?('Continue?')
+
+      deduplicate_accounts!
+      deduplicate_users!
+      deduplicate_account_domain_blocks!
+      deduplicate_account_identity_proofs!
+      deduplicate_announcement_reactions!
+      deduplicate_conversations!
+      deduplicate_custom_emojis!
+      deduplicate_custom_emoji_categories!
+      deduplicate_domain_allows!
+      deduplicate_domain_blocks!
+      deduplicate_unavailable_domains!
+      deduplicate_email_domain_blocks!
+      deduplicate_media_attachments!
+      deduplicate_preview_cards!
+      deduplicate_statuses!
+      deduplicate_tags!
+      deduplicate_webauthn_credentials!
+
+      Rails.cache.clear
+
+      @prompt.say 'Finished!'
+    end
+
+    private
+
+    def deduplicate_accounts!
+      remove_index_if_exists!(:accounts, 'index_accounts_on_username_and_domain_lower')
+
+      @prompt.say 'Deduplicating accounts… for local accounts, you will be asked to chose which account to keep unchanged.'
+
+      find_duplicate_accounts.each do |row|
+        accounts = Account.where(id: row['ids'].split(',')).to_a
+
+        if accounts.first.local?
+          deduplicate_local_accounts!(accounts)
+        else
+          deduplicate_remote_accounts!(accounts)
+        end
+      end
+
+      @prompt.say 'Restoring index_accounts_on_username_and_domain_lower…'
+      if ActiveRecord::Migrator.current_version < 20200620164023
+        ActiveRecord::Base.connection.add_index :accounts, 'lower (username), lower(domain)', name: 'index_accounts_on_username_and_domain_lower', unique: true
+      else
+        ActiveRecord::Base.connection.add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true
+      end
+    end
+
+    def deduplicate_users!
+      remove_index_if_exists!(:users, 'index_users_on_confirmation_token')
+      remove_index_if_exists!(:users, 'index_users_on_email')
+      remove_index_if_exists!(:users, 'index_users_on_remember_token')
+      remove_index_if_exists!(:users, 'index_users_on_reset_password_token')
+
+      @prompt.say 'Deduplicating user records…'
+
+      # Deduplicating email
+      ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users GROUP BY email HAVING count(*) > 1").each do |row|
+        users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse
+        ref_user = users.shift
+        @prompt.warn "Multiple users registered with e-mail address #{ref_user.email}."
+        @prompt.warn "e-mail will be disabled for the following accounts: #{user.map(&:account).map(&:acct).join(', ')}"
+        @prompt.warn 'Please reach out to them and set another address with `tootctl account modify` or delete them.'
+
+        i = 0
+        users.each do |user|
+          user.update!(email: "#{i} " + user.email)
+        end
+      end
+
+      ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE confirmation_token IS NOT NULL GROUP BY confirmation_token HAVING count(*) > 1").each do |row|
+        users = User.where(id: row['ids'].split(',')).sort_by(&:created_at).reverse.drop(1)
+        @prompt.warn "Unsetting confirmation token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}"
+
+        users.each do |user|
+          user.update!(confirmation_token: nil)
+        end
+      end
+
+      ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE remember_token IS NOT NULL GROUP BY remember_token HAVING count(*) > 1").each do |row|
+        users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1)
+        @prompt.warn "Unsetting remember token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}"
+
+        users.each do |user|
+          user.update!(remember_token: nil)
+        end
+      end
+
+      ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE reset_password_token IS NOT NULL GROUP BY reset_password_token HAVING count(*) > 1").each do |row|
+        users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1)
+        @prompt.warn "Unsetting password reset token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}"
+
+        users.each do |user|
+          user.update!(reset_password_token: nil)
+        end
+      end
+
+      @prompt.say 'Restoring users indexes…'
+      ActiveRecord::Base.connection.add_index :users, ['confirmation_token'], name: 'index_users_on_confirmation_token', unique: true
+      ActiveRecord::Base.connection.add_index :users, ['email'], name: 'index_users_on_email', unique: true
+      ActiveRecord::Base.connection.add_index :users, ['remember_token'], name: 'index_users_on_remember_token', unique: true
+      ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true
+    end
+
+    def deduplicate_account_domain_blocks!
+      remove_index_if_exists!(:account_domain_blocks, 'index_account_domain_blocks_on_account_id_and_domain')
+
+      @prompt.say 'Removing duplicate account domain blocks…'
+      ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM account_domain_blocks GROUP BY account_id, domain HAVING count(*) > 1").each do |row|
+        AccountDomainBlock.where(id: row['ids'].split(',').drop(1)).delete_all
+      end
+
+      @prompt.say 'Restoring account domain blocks indexes…'
+      ActiveRecord::Base.connection.add_index :account_domain_blocks, ['account_id', 'domain'], name: 'index_account_domain_blocks_on_account_id_and_domain', unique: true
+    end
+
+    def deduplicate_account_identity_proofs!
+      remove_index_if_exists!(:account_identity_proofs, 'index_account_proofs_on_account_and_provider_and_username')
+
+      @prompt.say 'Removing duplicate account identity proofs…'
+      ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM account_identity_proofs GROUP BY account_id, provider, provider_username HAVING count(*) > 1").each do |row|
+        AccountIdentityProof.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
+      end
+
+      @prompt.say 'Restoring account identity proofs indexes…'
+      ActiveRecord::Base.connection.add_index :account_identity_proofs, ['account_id', 'provider', 'provider_username'], name: 'index_account_proofs_on_account_and_provider_and_username', unique: true
+    end
+
+    def deduplicate_announcement_reactions!
+      return unless ActiveRecord::Base.connection.table_exists?(:announcement_reactions)
+
+      remove_index_if_exists!(:announcement_reactions, 'index_announcement_reactions_on_account_id_and_announcement_id')
+
+      @prompt.say 'Removing duplicate account identity proofs…'
+      ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM announcement_reactions GROUP BY account_id, announcement_id, name HAVING count(*) > 1").each do |row|
+        AnnouncementReaction.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
+      end
+
+      @prompt.say 'Restoring announcement_reactions indexes…'
+      ActiveRecord::Base.connection.add_index :announcement_reactions, ['account_id', 'announcement_id', 'name'], name: 'index_announcement_reactions_on_account_id_and_announcement_id', unique: true
+    end
+
+    def deduplicate_conversations!
+      remove_index_if_exists!(:conversations, 'index_conversations_on_uri')
+
+      @prompt.say 'Deduplicating conversations…'
+      ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM conversations WHERE uri IS NOT NULL GROUP BY uri HAVING count(*) > 1").each do |row|
+        conversations = Conversation.where(id: row['ids'].split(',')).sort_by(&:id).reverse
+
+        ref_conversation = conversations.shift
+
+        conversations.each do |other|
+          merge_conversations!(ref_conversation, other)
+          other.destroy
+        end
+      end
+
+      @prompt.say 'Restoring conversations indexes…'
+      ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true
+    end
+
+    def deduplicate_custom_emojis!
+      remove_index_if_exists!(:custom_emojis, 'index_custom_emojis_on_shortcode_and_domain')
+
+      @prompt.say 'Deduplicating custom_emojis…'
+      ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM custom_emojis GROUP BY shortcode, domain HAVING count(*) > 1").each do |row|
+        emojis = CustomEmoji.where(id: row['ids'].split(',')).sort_by(&:id).reverse
+
+        ref_emoji = emojis.shift
+
+        emojis.each do |other|
+          merge_custom_emojis!(ref_emoji, other)
+          other.destroy
+        end
+      end
+
+      @prompt.say 'Restoring custom_emojis indexes…'
+      ActiveRecord::Base.connection.add_index :custom_emojis, ['shortcode', 'domain'], name: 'index_custom_emojis_on_shortcode_and_domain', unique: true
+    end
+
+    def deduplicate_custom_emoji_categories!
+      remove_index_if_exists!(:custom_emoji_categories, 'index_custom_emoji_categories_on_name')
+
+      @prompt.say 'Deduplicating custom_emoji_categories…'
+      ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM custom_emoji_categories GROUP BY name HAVING count(*) > 1").each do |row|
+        categories = CustomEmojiCategory.where(id: row['ids'].split(',')).sort_by(&:id).reverse
+
+        ref_category = categories.shift
+
+        categories.each do |other|
+          merge_custom_emoji_categories!(ref_category, other)
+          other.destroy
+        end
+      end
+
+      @prompt.say 'Restoring custom_emoji_categories indexes…'
+      ActiveRecord::Base.connection.add_index :custom_emoji_categories, ['name'], name: 'index_custom_emoji_categories_on_name', unique: true
+    end
+
+    def deduplicate_domain_allows!
+      remove_index_if_exists!(:domain_allows, 'index_domain_allows_on_domain')
+
+      @prompt.say 'Deduplicating domain_allows…'
+      ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM domain_allows GROUP BY domain HAVING count(*) > 1").each do |row|
+        DomainAllow.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
+      end
+
+      @prompt.say 'Restoring domain_allows indexes…'
+      ActiveRecord::Base.connection.add_index :domain_allows, ['domain'], name: 'index_domain_allows_on_domain', unique: true
+    end
+
+    def deduplicate_domain_blocks!
+      remove_index_if_exists!(:domain_blocks, 'index_domain_blocks_on_domain')
+
+      @prompt.say 'Deduplicating domain_allows…'
+      ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM domain_blocks GROUP BY domain HAVING count(*) > 1").each do |row|
+        domain_blocks = DomainBlock.where(id: row['ids'].split(',')).by_severity.reverse.to_a
+
+        reject_media = domain_blocks.any?(&:reject_media?)
+        reject_reports = domain_blocks.any?(&:reject_reports?)
+
+        reference_block = domain_blocks.shift
+
+        private_comment = domain_blocks.reduce(reference_block.private_comment.presence) { |a, b| a || b.private_comment.presence }
+        public_comment  = domain_blocks.reduce(reference_block.public_comment.presence)  { |a, b| a || b.public_comment.presence }
+
+        reference_block.update!(reject_media: reject_media, reject_reports: reject_reports, private_comment: private_comment, public_comment: public_comment)
+
+        domain_blocks.each(&:destroy)
+      end
+
+      @prompt.say 'Restoring domain_blocks indexes…'
+      ActiveRecord::Base.connection.add_index :domain_blocks, ['domain'], name: 'index_domain_blocks_on_domain', unique: true
+    end
+
+    def deduplicate_unavailable_domains!
+      return unless ActiveRecord::Base.connection.table_exists?(:unavailable_domains)
+
+      remove_index_if_exists!(:unavailable_domains, 'index_unavailable_domains_on_domain')
+
+      @prompt.say 'Deduplicating unavailable_domains…'
+      ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM unavailable_domains GROUP BY domain HAVING count(*) > 1").each do |row|
+        UnavailableDomain.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
+      end
+
+      @prompt.say 'Restoring domain_allows indexes…'
+      ActiveRecord::Base.connection.add_index :unavailable_domains, ['domain'], name: 'index_unavailable_domains_on_domain', unique: true
+    end
+
+    def deduplicate_email_domain_blocks!
+      remove_index_if_exists!(:email_domain_blocks, 'index_email_domain_blocks_on_domain')
+
+      @prompt.say 'Deduplicating email_domain_blocks…'
+      ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM email_domain_blocks GROUP BY domain HAVING count(*) > 1").each do |row|
+        domain_blocks = EmailDomainBlock.where(id: row['ids'].split(',')).sort_by { |b| b.parent.nil? ? 1 : 0 }.to_a
+        domain_blocks.drop(1).each(&:destroy)
+      end
+
+      @prompt.say 'Restoring email_domain_blocks indexes…'
+      ActiveRecord::Base.connection.add_index :email_domain_blocks, ['domain'], name: 'index_email_domain_blocks_on_domain', unique: true
+    end
+
+    def deduplicate_media_attachments!
+      remove_index_if_exists!(:media_attachments, 'index_media_attachments_on_shortcode')
+
+      @prompt.say 'Deduplicating media_attachments…'
+      ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM media_attachments WHERE shortcode IS NOT NULL GROUP BY shortcode HAVING count(*) > 1").each do |row|
+        MediaAttachment.where(id: row['ids'].split(',').drop(1)).update_all(shortcode: nil)
+      end
+
+      @prompt.say 'Restoring media_attachments indexes…'
+      ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true
+    end
+
+    def deduplicate_preview_cards!
+      remove_index_if_exists!(:preview_cards, 'index_preview_cards_on_url')
+
+      @prompt.say 'Deduplicating preview_cards…'
+      ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM preview_cards GROUP BY url HAVING count(*) > 1").each do |row|
+        PreviewCard.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
+      end
+
+      @prompt.say 'Restoring preview_cards indexes…'
+      ActiveRecord::Base.connection.add_index :preview_cards, ['url'], name: 'index_preview_cards_on_url', unique: true
+    end
+
+    def deduplicate_statuses!
+      remove_index_if_exists!(:statuses, 'index_statuses_on_uri')
+
+      @prompt.say 'Deduplicating statuses…'
+      ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM statuses WHERE uri IS NOT NULL GROUP BY uri HAVING count(*) > 1").each do |row|
+        statuses = Status.where(id: row['ids'].split(',')).sort_by(&:id)
+        ref_status = statuses.shift
+        statuses.each do |status|
+          merge_statuses!(ref_status, status) if status.account_id == ref_status.account_id
+          status.destroy
+        end
+      end
+
+      @prompt.say 'Restoring statuses indexes…'
+      ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true
+    end
+
+    def deduplicate_tags!
+      remove_index_if_exists!(:tags, 'index_tags_on_name_lower')
+
+      @prompt.say 'Deduplicating tags…'
+      ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM tags GROUP BY lower((name)::text) HAVING count(*) > 1").each do |row|
+        tags = Tag.where(id: row['ids'].split(',')).sort_by { |t| [t.usable?, t.trendable?, t.listable?].count(false) }
+        ref_tag = tags.shift
+        tags.each do |tag|
+          merge_tags!(ref_tag, tag)
+          tag.destroy
+        end
+      end
+
+      @prompt.say 'Restoring tags indexes…'
+      ActiveRecord::Base.connection.add_index :tags, 'lower((name)::text)', name: 'index_tags_on_name_lower', unique: true
+    end
+
+    def deduplicate_webauthn_credentials!
+      return unless ActiveRecord::Base.connection.table_exists?(:webauthn_credentials)
+
+      remove_index_if_exists!(:webauthn_credentials, 'index_webauthn_credentials_on_external_id')
+
+      @prompt.say 'Deduplicating webauthn_credentials…'
+      ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM webauthn_credentials GROUP BY external_id HAVING count(*) > 1").each do |row|
+        WebauthnCredential.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
+      end
+
+      @prompt.say 'Restoring webauthn_credentials indexes…'
+      ActiveRecord::Base.connection.add_index :webauthn_credentials, ['external_id'], name: 'index_webauthn_credentials_on_external_id', unique: true
+    end
+
+    def deduplicate_local_accounts!(accounts)
+      accounts = accounts.sort_by(&:id).reverse
+
+      @prompt.warn "Multiple local accounts were found for username '#{accounts.first.username}'."
+      @prompt.warn 'All those accounts are distinct accounts but only the most recently-created one is fully-functionnal.'
+
+      accounts.each_with_index do |account, idx|
+        @prompt.say '%2d. %s: created at: %s; updated at: %s; last logged in at: %s; statuses: %5d; last status at: %s' % [idx, account.username, account.created_at, account.updated_at, account.user&.last_sign_in_at&.to_s || 'N/A', account.account_stat&.statuses_count || 0, account.account_stat&.last_status_at || 'N/A']
+      end
+
+      @prompt.say 'Please chose the one to keep unchanged, other ones will be automatically renamed.'
+
+      ref_id = @prompt.ask('Account to keep unchanged:') do |q|
+        q.required true
+        q.default 0
+        q.convert :int
+      end
+
+      accounts.delete_at(ref_id)
+
+      i = 0
+      accounts.each do |account|
+        i += 1
+        username = account.username + "_#{i}"
+
+        while Account.local.exists?(username: username)
+          i += 1
+          username = account.username + "_#{i}"
+        end
+
+        account.update!(username: username)
+      end
+    end
+
+    def deduplicate_remote_accounts!(accounts)
+      accounts = accounts.sort_by(&:updated_at).reverse
+
+      reference_account = accounts.shift
+
+      accounts.each do |other_account|
+        if other_account.public_key == reference_account.public_key
+          # The accounts definitely point to the same resource, so
+          # it's safe to re-attribute content and relationships
+          merge_accounts!(reference_account, other_account)
+        end
+
+        other_account.destroy
+      end
+    end
+
+    def merge_accounts!(main_account, duplicate_account)
+      # Since it's the same remote resource, the remote resource likely
+      # already believes we are following/blocking, so it's safe to
+      # re-attribute the relationships too. However, during the presence
+      # of the index bug users could have *also* followed the reference
+      # account already, therefore mass update will not work and we need
+      # to check for (and skip past) uniqueness errors
+      owned_classes = [
+        Status, StatusPin, MediaAttachment, Poll, Report, Tombstone, Favourite,
+        Follow, FollowRequest, Block, Mute, AccountIdentityProof,
+        AccountModerationNote, AccountPin, AccountStat, ListAccount,
+        PollVote, Mention
+      ]
+      owned_classes.each do |klass|
+        klass.where(account_id: duplicate_account.id).find_each do |record|
+          begin
+            record.update_attribute(:account_id, main_account.id)
+          rescue ActiveRecord::RecordNotUnique
+            next
+          end
+        end
+      end
+
+      target_classes = [Follow, FollowRequest, Block, Mute, AccountModerationNote, AccountPin]
+      target_classes.each do |klass|
+        klass.where(target_account_id: duplicate_account.id).find_each do |record|
+          begin
+            record.update_attribute(:target_account_id, main_account.id)
+          rescue ActiveRecord::RecordNotUnique
+            next
+          end
+        end
+      end
+    end
+
+    def merge_conversations!(main_conv, duplicate_conv)
+      owned_classes = [ConversationMute, AccountConversation]
+      owned_classes.each do |klass|
+        klass.where(conversation_id: duplicate_conv.id).find_each do |record|
+          begin
+            record.update_attribute(:account_id, main_conv.id)
+          rescue ActiveRecord::RecordNotUnique
+            next
+          end
+        end
+      end
+    end
+
+    def merge_custom_emojis!(main_emoji, duplicate_emoji)
+      owned_classes = [AnnouncementReaction]
+      owned_classes.each do |klass|
+        klass.where(custom_emoji_id: duplicate_emoji.id).update_all(custom_emoji_id: main_emoji.id)
+      end
+    end
+
+    def merge_custom_emoji_categories!(main_category, duplicate_category)
+      owned_classes = [CustomEmoji]
+      owned_classes.each do |klass|
+        klass.where(category_id: duplicate_category.id).update_all(category_id: main_category.id)
+      end
+    end
+
+    def merge_statuses!(main_status, duplicate_status)
+      owned_classes = [Favourite, Mention, Poll]
+      owned_classes << Bookmark if ActiveRecord::Base.connection.table_exists?(:bookmarks)
+      owned_classes.each do |klass|
+        klass.where(status_id: duplicate_status.id).find_each do |record|
+          begin
+            record.update_attribute(:status_id, main_status.id)
+          rescue ActiveRecord::RecordNotUnique
+            next
+          end
+        end
+      end
+
+      StatusPin.where(account_id: main_status.account_id, status_id: duplicate_status.id).find_each do |record|
+        begin
+          record.update_attribute(:status_id, main_status.id)
+        rescue ActiveRecord::RecordNotUnique
+          next
+        end
+      end
+
+      Status.where(in_reply_to_id: duplicate_status.id).find_each do |record|
+        begin
+          record.update_attribute(:in_reply_to_id, main_status.id)
+        rescue ActiveRecord::RecordNotUnique
+          next
+        end
+      end
+
+      Status.where(reblog_of_id: duplicate_status.id).find_each do |record|
+        begin
+          record.update_attribute(:reblog_of_id, main_status.id)
+        rescue ActiveRecord::RecordNotUnique
+          next
+        end
+      end
+    end
+
+    def merge_tags!(main_tag, duplicate_tag)
+      [FeaturedTag].each do |klass|
+        klass.where(tag_id: duplicate_tag.id).find_each do |record|
+          begin
+            record.update_attribute(:tag_id, main_tag.id)
+          rescue ActiveRecord::RecordNotUnique
+            next
+          end
+        end
+      end
+    end
+
+    def find_duplicate_accounts
+      ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM accounts GROUP BY lower(username), COALESCE(lower(domain), '') HAVING count(*) > 1")
+    end
+
+    def remove_index_if_exists!(table, name)
+      ActiveRecord::Base.connection.remove_index(table, name: name)
+    rescue ArgumentError
+      nil
+    rescue ActiveRecord::StatementInvalid
+      nil
+    end
+  end
+end
diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake
index b76e90131f2..199155107b1 100644
--- a/lib/tasks/db.rake
+++ b/lib/tasks/db.rake
@@ -48,6 +48,17 @@ namespace :db do
     end
   end
 
+  task :post_migration_hook do
+    at_exit do
+      unless %w(C POSIX).include?(ActiveRecord::Base.connection.execute('SELECT datcollate FROM pg_database WHERE datname = current_database();').first['datcollate'])
+        Rails.logger.warn 'WARNING: Your database is using an unsafe collation setting, which might result in index corruption.'
+        Rails.logger.warn 'WARNING: See https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/#am-i-affected'
+      end
+    end
+  end
+
+  Rake::Task['db:migrate'].enhance(['db:post_migration_hook'])
+
   # Before we load the schema, define the timestamp_id function.
   # Idiomatically, we might do this in a migration, but then it
   # wouldn't end up in schema.rb, so we'd need to figure out a way to

From df1653174be233f2737d8ec281325dee54011947 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 19 Nov 2020 17:38:06 +0100
Subject: [PATCH 19/31] Add cache buster feature for media files (#15155)

Nginx can be configured to bypass proxy cache when a special header
is in the request. If the response is cacheable, it will replace
the cache for that request. Proxy caching of media files is
desirable when using object storage as a way of minimizing bandwidth
costs, but has the drawback of leaving deleted media files for
a configured amount of cache time. A cache buster can make those
media files immediately unavailable. This especially makes sense
when suspending and unsuspending an account.
---
 app/lib/cache_buster.rb                   | 28 +++++++++++++++++++++++
 app/services/suspend_account_service.rb   |  2 ++
 app/services/unsuspend_account_service.rb |  2 ++
 app/workers/cache_buster_worker.rb        | 18 +++++++++++++++
 config/initializers/cache_buster.rb       | 10 ++++++++
 config/initializers/paperclip.rb          |  1 -
 6 files changed, 60 insertions(+), 1 deletion(-)
 create mode 100644 app/lib/cache_buster.rb
 create mode 100644 app/workers/cache_buster_worker.rb
 create mode 100644 config/initializers/cache_buster.rb

diff --git a/app/lib/cache_buster.rb b/app/lib/cache_buster.rb
new file mode 100644
index 00000000000..035611518eb
--- /dev/null
+++ b/app/lib/cache_buster.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+class CacheBuster
+  def initialize(options = {})
+    @secret_header = options[:secret_header] || 'Secret-Header'
+    @secret        = options[:secret] || 'True'
+  end
+
+  def bust(url)
+    site = Addressable::URI.parse(url).normalized_site
+
+    request_pool.with(site) do |http_client|
+      build_request(url, http_client).perform
+    end
+  end
+
+  private
+
+  def request_pool
+    RequestPool.current
+  end
+
+  def build_request(url, http_client)
+    Request.new(:get, url, http_client: http_client).tap do |request|
+      request.add_headers(@secret_header => @secret)
+    end
+  end
+end
diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb
index 7c70a6021b8..19d65280d97 100644
--- a/app/services/suspend_account_service.rb
+++ b/app/services/suspend_account_service.rb
@@ -78,6 +78,8 @@ class SuspendAccountService < BaseService
               Rails.logger.warn "Tried to change permission on non-existent file #{attachment.path(style)}"
             end
           end
+
+          CacheBusterWorker.perform_async(attachment.path(style)) if Rails.configuration.x.cache_buster_enabled
         end
       end
     end
diff --git a/app/services/unsuspend_account_service.rb b/app/services/unsuspend_account_service.rb
index a81d1ac4f70..f07a3f053b4 100644
--- a/app/services/unsuspend_account_service.rb
+++ b/app/services/unsuspend_account_service.rb
@@ -69,6 +69,8 @@ class UnsuspendAccountService < BaseService
               Rails.logger.warn "Tried to change permission on non-existent file #{attachment.path(style)}"
             end
           end
+
+          CacheBusterWorker.perform_async(attachment.path(style)) if Rails.configuration.x.cache_buster_enabled
         end
       end
     end
diff --git a/app/workers/cache_buster_worker.rb b/app/workers/cache_buster_worker.rb
new file mode 100644
index 00000000000..5ad0a44cb7a
--- /dev/null
+++ b/app/workers/cache_buster_worker.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class CacheBusterWorker
+  include Sidekiq::Worker
+  include RoutingHelper
+
+  sidekiq_options queue: 'pull'
+
+  def perform(path)
+    cache_buster.bust(full_asset_url(path))
+  end
+
+  private
+
+  def cache_buster
+    CacheBuster.new(Rails.configuration.x.cache_buster)
+  end
+end
diff --git a/config/initializers/cache_buster.rb b/config/initializers/cache_buster.rb
new file mode 100644
index 00000000000..227e450f35c
--- /dev/null
+++ b/config/initializers/cache_buster.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+Rails.application.configure do
+  config.x.cache_buster_enabled = ENV['CACHE_BUSTER_ENABLED'] == 'true'
+
+  config.x.cache_buster = {
+    secret_header: ENV['CACHE_BUSTER_SECRET_HEADER'],
+    secret: ENV['CACHE_BUSTER_SECRET'],
+  }
+end
diff --git a/config/initializers/paperclip.rb b/config/initializers/paperclip.rb
index b841d52203c..25adcd8d634 100644
--- a/config/initializers/paperclip.rb
+++ b/config/initializers/paperclip.rb
@@ -107,7 +107,6 @@ elsif ENV['SWIFT_ENABLED'] == 'true'
 else
   Paperclip::Attachment.default_options.merge!(
     storage: :filesystem,
-    use_timestamp: true,
     path: File.join(ENV.fetch('PAPERCLIP_ROOT_PATH', File.join(':rails_root', 'public', 'system')), ':prefix_path:class', ':attachment', ':id_partition', ':style', ':filename'),
     url: ENV.fetch('PAPERCLIP_ROOT_URL', '/system') + '/:prefix_url:class/:attachment/:id_partition/:style/:filename',
   )

From 2f6831f3187af0dfd9cacd7d4cba5664be7a9cbd Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Thu, 19 Nov 2020 17:39:47 +0100
Subject: [PATCH 20/31] Fix sending spurious Rejects when processing remote
 account deletion (#15104)

* Fix sending spurious Rejects when processing remote account deletion

* Make skip_side_effects imply skip_activitypub
---
 app/lib/activitypub/activity/delete.rb  | 2 +-
 app/services/delete_account_service.rb  | 5 ++++-
 app/services/resolve_account_service.rb | 2 +-
 app/workers/account_deletion_worker.rb  | 3 ++-
 4 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/app/lib/activitypub/activity/delete.rb b/app/lib/activitypub/activity/delete.rb
index 09b9e5e0e83..2e5293b832a 100644
--- a/app/lib/activitypub/activity/delete.rb
+++ b/app/lib/activitypub/activity/delete.rb
@@ -13,7 +13,7 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
 
   def delete_person
     lock_or_return("delete_in_progress:#{@account.id}") do
-      DeleteAccountService.new.call(@account, reserve_username: false)
+      DeleteAccountService.new.call(@account, reserve_username: false, skip_activitypub: true)
     end
   end
 
diff --git a/app/services/delete_account_service.rb b/app/services/delete_account_service.rb
index de6488c7836..778d064de00 100644
--- a/app/services/delete_account_service.rb
+++ b/app/services/delete_account_service.rb
@@ -41,6 +41,7 @@ class DeleteAccountService < BaseService
   # @option [Boolean] :reserve_email Keep user record. Only applicable for local accounts
   # @option [Boolean] :reserve_username Keep account record
   # @option [Boolean] :skip_side_effects Side effects are ActivityPub and streaming API payloads
+  # @option [Boolean] :skip_activitypub Skip sending ActivityPub payloads. Implied by :skip_side_effects
   # @option [Time]    :suspended_at Only applicable when :reserve_username is true
   def call(account, **options)
     @account = account
@@ -52,6 +53,8 @@ class DeleteAccountService < BaseService
       @options[:skip_side_effects] = true
     end
 
+    @options[:skip_activitypub] = true if @options[:skip_side_effects]
+
     reject_follows!
     purge_user!
     purge_profile!
@@ -62,7 +65,7 @@ class DeleteAccountService < BaseService
   private
 
   def reject_follows!
-    return if @account.local? || !@account.activitypub?
+    return if @account.local? || !@account.activitypub? || @options[:skip_activitypub]
 
     # When deleting a remote account, the account obviously doesn't
     # actually become deleted on its origin server, i.e. unlike a
diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb
index 4783e6d33b2..c5291fa67c8 100644
--- a/app/services/resolve_account_service.rb
+++ b/app/services/resolve_account_service.rb
@@ -145,7 +145,7 @@ class ResolveAccountService < BaseService
   end
 
   def queue_deletion!
-    AccountDeletionWorker.perform_async(@account.id, reserve_username: false)
+    AccountDeletionWorker.perform_async(@account.id, reserve_username: false, skip_activitypub: true)
   end
 
   def lock_options
diff --git a/app/workers/account_deletion_worker.rb b/app/workers/account_deletion_worker.rb
index 81c3b91ad8d..98b67419d01 100644
--- a/app/workers/account_deletion_worker.rb
+++ b/app/workers/account_deletion_worker.rb
@@ -7,7 +7,8 @@ class AccountDeletionWorker
 
   def perform(account_id, options = {})
     reserve_username = options.with_indifferent_access.fetch(:reserve_username, true)
-    DeleteAccountService.new.call(Account.find(account_id), reserve_username: reserve_username, reserve_email: false)
+    skip_activitypub = options.with_indifferent_access.fetch(:skip_activitypub, false)
+    DeleteAccountService.new.call(Account.find(account_id), reserve_username: reserve_username, skip_activitypub: skip_activitypub, reserve_email: false)
   rescue ActiveRecord::RecordNotFound
     true
   end

From 8e2530ea16332f389edaeba7e87d7447c077bb00 Mon Sep 17 00:00:00 2001
From: Darius Kazemi <darius.kazemi@gmail.com>
Date: Thu, 19 Nov 2020 08:40:18 -0800
Subject: [PATCH 21/31] Mark correctly as read reverse-order announcements
 (#15070)

This fixes a bug in #15065 where the "read" indicator was not getting correctly set. The ID of a dismissed announcement is now correct.
---
 .../features/getting_started/components/announcements.js        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/javascript/mastodon/features/getting_started/components/announcements.js b/app/javascript/mastodon/features/getting_started/components/announcements.js
index d53bd8055c3..5bc3abac665 100644
--- a/app/javascript/mastodon/features/getting_started/components/announcements.js
+++ b/app/javascript/mastodon/features/getting_started/components/announcements.js
@@ -396,7 +396,7 @@ class Announcements extends ImmutablePureComponent {
   _markAnnouncementAsRead () {
     const { dismissAnnouncement, announcements } = this.props;
     const { index } = this.state;
-    const announcement = announcements.get(index);
+    const announcement = announcements.get(announcements.size - 1 - index);
     if (!announcement.get('read')) dismissAnnouncement(announcement.get('id'));
   }
 

From dbe5a961213c76775a46cb06bc7513053b0b4015 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Nov 2020 17:42:47 +0100
Subject: [PATCH 22/31] Bump @testing-library/react from 11.1.1 to 11.2.0
 (#15186)

Bumps [@testing-library/react](https://github.com/testing-library/react-testing-library) from 11.1.1 to 11.2.0.
- [Release notes](https://github.com/testing-library/react-testing-library/releases)
- [Changelog](https://github.com/testing-library/react-testing-library/blob/master/CHANGELOG.md)
- [Commits](https://github.com/testing-library/react-testing-library/compare/v11.1.1...v11.2.0)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 package.json |  2 +-
 yarn.lock    | 48 +++++++++++++++++++-----------------------------
 2 files changed, 20 insertions(+), 30 deletions(-)

diff --git a/package.json b/package.json
index 3129edeb2bb..d2ea05d0a34 100644
--- a/package.json
+++ b/package.json
@@ -171,7 +171,7 @@
   },
   "devDependencies": {
     "@testing-library/jest-dom": "^5.11.6",
-    "@testing-library/react": "^11.1.1",
+    "@testing-library/react": "^11.2.0",
     "babel-eslint": "^10.1.0",
     "babel-jest": "^26.6.3",
     "eslint": "^7.13.0",
diff --git a/yarn.lock b/yarn.lock
index a36877bb14b..aceae8d2f22 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -940,7 +940,7 @@
   dependencies:
     regenerator-runtime "^0.12.0"
 
-"@babel/runtime@^7.1.2", "@babel/runtime@^7.10.2", "@babel/runtime@^7.10.3", "@babel/runtime@^7.11.2", "@babel/runtime@^7.12.1", "@babel/runtime@^7.12.5", "@babel/runtime@^7.2.0", "@babel/runtime@^7.4.4", "@babel/runtime@^7.5.5", "@babel/runtime@^7.6.3", "@babel/runtime@^7.7.2", "@babel/runtime@^7.8.4", "@babel/runtime@^7.9.2":
+"@babel/runtime@^7.1.2", "@babel/runtime@^7.10.2", "@babel/runtime@^7.11.2", "@babel/runtime@^7.12.1", "@babel/runtime@^7.12.5", "@babel/runtime@^7.2.0", "@babel/runtime@^7.4.4", "@babel/runtime@^7.5.5", "@babel/runtime@^7.6.3", "@babel/runtime@^7.7.2", "@babel/runtime@^7.8.4", "@babel/runtime@^7.9.2":
   version "7.12.5"
   resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.12.5.tgz#410e7e487441e1b360c29be715d870d9b985882e"
   integrity sha512-plcc+hbExy3McchJCEQG3knOsuh3HH+Prx1P6cLIkET/0dLuQDEnrT+s27Axgc9bqfsmNUNHfscgMUdBpC9xfg==
@@ -1304,7 +1304,7 @@
     "@types/yargs" "^15.0.0"
     chalk "^3.0.0"
 
-"@jest/types@^26.3.0", "@jest/types@^26.6.2":
+"@jest/types@^26.6.2":
   version "26.6.2"
   resolved "https://registry.yarnpkg.com/@jest/types/-/types-26.6.2.tgz#bef5a532030e1d88a2f5a6d933f84e97226ed48e"
   integrity sha512-fC6QCp7Sc5sX6g8Tvbmj4XUTbyrik0akgRy03yjXbQaBWWNWGE7SGtJk98m0N8nzegD/7SggrUlivxo5ax4KWQ==
@@ -1341,19 +1341,19 @@
   dependencies:
     "@sinonjs/commons" "^1.7.0"
 
-"@testing-library/dom@^7.26.4":
-  version "7.26.5"
-  resolved "https://registry.yarnpkg.com/@testing-library/dom/-/dom-7.26.5.tgz#804a74fc893bf6da1a7970dbca7b94c2bbfe983d"
-  integrity sha512-2v/fv0s4keQjJIcD4bjfJMFtvxz5icartxUWdIZVNJR539WD9oxVrvIAPw+3Ydg4RLgxt0rvQx3L9cAjCci0Kg==
+"@testing-library/dom@^7.27.1":
+  version "7.27.1"
+  resolved "https://registry.yarnpkg.com/@testing-library/dom/-/dom-7.27.1.tgz#b760182513357e4448a8461f9565d733a88d71d0"
+  integrity sha512-AF56RoeUU8bO4DOvLyMI44H3O1LVKZQi2D/m5fNDr+iR4drfOFikTr26hT6IY7YG+l8g69FXsHERa+uThaYYQg==
   dependencies:
     "@babel/code-frame" "^7.10.4"
-    "@babel/runtime" "^7.10.3"
+    "@babel/runtime" "^7.12.5"
     "@types/aria-query" "^4.2.0"
     aria-query "^4.2.2"
     chalk "^4.1.0"
-    dom-accessibility-api "^0.5.1"
+    dom-accessibility-api "^0.5.4"
     lz-string "^1.4.4"
-    pretty-format "^26.4.2"
+    pretty-format "^26.6.2"
 
 "@testing-library/jest-dom@^5.11.6":
   version "5.11.6"
@@ -1369,13 +1369,13 @@
     lodash "^4.17.15"
     redent "^3.0.0"
 
-"@testing-library/react@^11.1.1":
-  version "11.1.1"
-  resolved "https://registry.yarnpkg.com/@testing-library/react/-/react-11.1.1.tgz#226d8dc7491b702fcaac2d7d88d42892e655893a"
-  integrity sha512-DT/P2opE9o4NWCd/oIL73b6VF/Xk9AY8iYSstKfz9cXw0XYPQ5IhA/cuYfoN9nU+mAynW8DpAVfEWdM6e7zF6g==
+"@testing-library/react@^11.2.0":
+  version "11.2.0"
+  resolved "https://registry.yarnpkg.com/@testing-library/react/-/react-11.2.0.tgz#ce977a76b6342ea95c71ccd6de3012b1635fb559"
+  integrity sha512-90xKYJzskZ7q/AoSuWraQL4EGZlr75uZvDt3nrO4M+rugN02zjO45tmOBq/JBOgDiMIL1tkhHioKXjJsVaSINA==
   dependencies:
-    "@babel/runtime" "^7.12.1"
-    "@testing-library/dom" "^7.26.4"
+    "@babel/runtime" "^7.12.5"
+    "@testing-library/dom" "^7.27.1"
 
 "@types/aria-query@^4.2.0":
   version "4.2.0"
@@ -3715,10 +3715,10 @@ doctrine@^3.0.0:
   dependencies:
     esutils "^2.0.2"
 
-dom-accessibility-api@^0.5.1:
-  version "0.5.2"
-  resolved "https://registry.yarnpkg.com/dom-accessibility-api/-/dom-accessibility-api-0.5.2.tgz#ef3cdb5d3f0d599d8f9c8b18df2fb63c9793739d"
-  integrity sha512-k7hRNKAiPJXD2aBqfahSo4/01cTsKWXf+LqJgglnkN2Nz8TsxXKQBXHhKe0Ye9fEfHEZY49uSA5Sr3AqP/sWKA==
+dom-accessibility-api@^0.5.4:
+  version "0.5.4"
+  resolved "https://registry.yarnpkg.com/dom-accessibility-api/-/dom-accessibility-api-0.5.4.tgz#b06d059cdd4a4ad9a79275f9d414a5c126241166"
+  integrity sha512-TvrjBckDy2c6v6RLxPv5QXOnU+SmF9nBII5621Ve5fu6Z/BDrENurBEvlC1f44lKEUVqOpK4w9E5Idc5/EgkLQ==
 
 dom-helpers@^3.2.1, dom-helpers@^3.4.0:
   version "3.4.0"
@@ -8393,16 +8393,6 @@ pretty-format@^25.2.1, pretty-format@^25.5.0:
     ansi-styles "^4.0.0"
     react-is "^16.12.0"
 
-pretty-format@^26.4.2:
-  version "26.4.2"
-  resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-26.4.2.tgz#d081d032b398e801e2012af2df1214ef75a81237"
-  integrity sha512-zK6Gd8zDsEiVydOCGLkoBoZuqv8VTiHyAbKznXe/gaph/DAeZOmit9yMfgIz5adIgAMMs5XfoYSwAX3jcCO1tA==
-  dependencies:
-    "@jest/types" "^26.3.0"
-    ansi-regex "^5.0.0"
-    ansi-styles "^4.0.0"
-    react-is "^16.12.0"
-
 pretty-format@^26.6.2:
   version "26.6.2"
   resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-26.6.2.tgz#e35c2705f14cb7fe2fe94fa078345b444120fc93"

From 1a9099ca7ca42f5ca855cacc95233856668cbb08 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Nov 2020 17:43:35 +0100
Subject: [PATCH 23/31] Bump rubocop from 0.93.1 to 1.3.0 (#15170)

Bumps [rubocop](https://github.com/rubocop-hq/rubocop) from 0.93.1 to 1.3.0.
- [Release notes](https://github.com/rubocop-hq/rubocop/releases)
- [Changelog](https://github.com/rubocop-hq/rubocop/blob/master/CHANGELOG.md)
- [Commits](https://github.com/rubocop-hq/rubocop/compare/v0.93.1...v1.3.0)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 Gemfile      | 2 +-
 Gemfile.lock | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Gemfile b/Gemfile
index 26bc6fd524c..631f1265f79 100644
--- a/Gemfile
+++ b/Gemfile
@@ -139,7 +139,7 @@ group :development do
   gem 'letter_opener', '~> 1.7'
   gem 'letter_opener_web', '~> 1.4'
   gem 'memory_profiler'
-  gem 'rubocop', '~> 0.93', require: false
+  gem 'rubocop', '~> 1.3', require: false
   gem 'rubocop-rails', '~> 2.8', require: false
   gem 'brakeman', '~> 4.10', require: false
   gem 'bundler-audit', '~> 0.7', require: false
diff --git a/Gemfile.lock b/Gemfile.lock
index eaf6ebf1d57..558bfc2b869 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -532,16 +532,16 @@ GEM
     rspec-support (3.9.3)
     rspec_junit_formatter (0.4.1)
       rspec-core (>= 2, < 4, != 2.12.0)
-    rubocop (0.93.1)
+    rubocop (1.3.0)
       parallel (~> 1.10)
       parser (>= 2.7.1.5)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8)
       rexml
-      rubocop-ast (>= 0.6.0)
+      rubocop-ast (>= 1.1.1)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-ast (0.8.0)
+    rubocop-ast (1.1.1)
       parser (>= 2.7.1.5)
     rubocop-rails (2.8.1)
       activesupport (>= 4.2.0)
@@ -775,7 +775,7 @@ DEPENDENCIES
   rspec-rails (~> 4.0)
   rspec-sidekiq (~> 3.1)
   rspec_junit_formatter (~> 0.4)
-  rubocop (~> 0.93)
+  rubocop (~> 1.3)
   rubocop-rails (~> 2.8)
   ruby-progressbar (~> 1.10)
   sanitize (~> 5.2)

From 67ace1d8907f7534ab827ed262a22df4c3de1ba6 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Nov 2020 17:43:58 +0100
Subject: [PATCH 24/31] Bump omniauth-cas from 1.1.1 to 2.0.0 (#15165)

Bumps [omniauth-cas](https://github.com/dlindahl/omniauth-cas) from 1.1.1 to 2.0.0.
- [Release notes](https://github.com/dlindahl/omniauth-cas/releases)
- [Changelog](https://github.com/dlindahl/omniauth-cas/blob/master/CHANGELOG.md)
- [Commits](https://github.com/dlindahl/omniauth-cas/compare/v1.1.1...v2.0.0)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 Gemfile      | 2 +-
 Gemfile.lock | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Gemfile b/Gemfile
index 631f1265f79..330222922ab 100644
--- a/Gemfile
+++ b/Gemfile
@@ -44,7 +44,7 @@ group :pam_authentication, optional: true do
 end
 
 gem 'net-ldap', '~> 0.16'
-gem 'omniauth-cas', '~> 1.1'
+gem 'omniauth-cas', '~> 2.0'
 gem 'omniauth-saml', '~> 1.10'
 gem 'omniauth', '~> 1.9'
 
diff --git a/Gemfile.lock b/Gemfile.lock
index 558bfc2b869..dffd7d791e1 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -371,7 +371,7 @@ GEM
     omniauth (1.9.1)
       hashie (>= 3.4.6)
       rack (>= 1.6.2, < 3)
-    omniauth-cas (1.1.1)
+    omniauth-cas (2.0.0)
       addressable (~> 2.3)
       nokogiri (~> 1.5)
       omniauth (~> 1.2)
@@ -741,7 +741,7 @@ DEPENDENCIES
   nsa (~> 0.2)
   oj (~> 3.10)
   omniauth (~> 1.9)
-  omniauth-cas (~> 1.1)
+  omniauth-cas (~> 2.0)
   omniauth-saml (~> 1.10)
   ox (~> 2.13)
   paperclip (~> 6.0)

From 541b9f8c1c2c987544b00f64792aa37ee7bd25a5 Mon Sep 17 00:00:00 2001
From: Daigo 3 Dango <zunda@users.noreply.github.com>
Date: Thu, 19 Nov 2020 16:46:46 +0000
Subject: [PATCH 25/31] Use Ruby 2.7.2 (#15150)

thwait and e2mmap are no longer needed in Gemfile.
Gems properly require those.
---
 .ruby-version | 2 +-
 Dockerfile    | 2 +-
 Gemfile       | 3 ---
 Gemfile.lock  | 2 --
 4 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/.ruby-version b/.ruby-version
index 338a5b5d8fe..37c2961c243 100644
--- a/.ruby-version
+++ b/.ruby-version
@@ -1 +1 @@
-2.6.6
+2.7.2
diff --git a/Dockerfile b/Dockerfile
index c52f89fdca1..57d061fd0ed 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -40,7 +40,7 @@ RUN apt update && \
 	cd .. && rm -rf jemalloc-$JE_VER $JE_VER.tar.gz
 
 # Install Ruby
-ENV RUBY_VER="2.6.6"
+ENV RUBY_VER="2.7.2"
 ENV CPPFLAGS="-I/opt/jemalloc/include"
 ENV LDFLAGS="-L/opt/jemalloc/lib/"
 RUN apt update && \
diff --git a/Gemfile b/Gemfile
index 330222922ab..30b5435b53f 100644
--- a/Gemfile
+++ b/Gemfile
@@ -11,9 +11,6 @@ gem 'sprockets', '~> 3.7.2'
 gem 'thor', '~> 1.0'
 gem 'rack', '~> 2.2.3'
 
-gem 'thwait', '~> 0.2.0'
-gem 'e2mmap', '~> 0.1.0'
-
 gem 'hamlit-rails', '~> 0.2'
 gem 'pg', '~> 1.2'
 gem 'makara', '~> 0.4'
diff --git a/Gemfile.lock b/Gemfile.lock
index dffd7d791e1..b333c4e995f 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -704,7 +704,6 @@ DEPENDENCIES
   discard (~> 1.2)
   doorkeeper (~> 5.4)
   dotenv-rails (~> 2.7)
-  e2mmap (~> 0.1.0)
   ed25519 (~> 1.2)
   fabrication (~> 2.21)
   faker (~> 2.14)
@@ -793,7 +792,6 @@ DEPENDENCIES
   streamio-ffmpeg (~> 3.0)
   strong_migrations (~> 0.7)
   thor (~> 1.0)
-  thwait (~> 0.2.0)
   tty-prompt (~> 0.22)
   twitter-text (~> 1.14)
   tzinfo-data (~> 1.2020)

From 022d2353a77edaddd6a681405521f27f5fac11b2 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Nov 2020 17:47:20 +0100
Subject: [PATCH 26/31] Bump webpack-cli from 3.3.12 to 4.2.0 (#15123)

Bumps [webpack-cli](https://github.com/webpack/webpack-cli) from 3.3.12 to 4.2.0.
- [Release notes](https://github.com/webpack/webpack-cli/releases)
- [Changelog](https://github.com/webpack/webpack-cli/blob/master/CHANGELOG.md)
- [Commits](https://github.com/webpack/webpack-cli/compare/v3.3.12...webpack-cli@4.2.0)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 package.json |   2 +-
 yarn.lock    | 245 +++++++++++++++++++++++++++------------------------
 2 files changed, 130 insertions(+), 117 deletions(-)

diff --git a/package.json b/package.json
index d2ea05d0a34..28c63061080 100644
--- a/package.json
+++ b/package.json
@@ -165,7 +165,7 @@
     "webpack": "^4.44.2",
     "webpack-assets-manifest": "^3.1.1",
     "webpack-bundle-analyzer": "^4.1.0",
-    "webpack-cli": "^3.3.12",
+    "webpack-cli": "^4.2.0",
     "webpack-merge": "^5.4.0",
     "wicg-inert": "^3.1.0"
   },
diff --git a/yarn.lock b/yarn.lock
index aceae8d2f22..65cd2869611 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1697,6 +1697,18 @@
     "@webassemblyjs/wast-parser" "1.9.0"
     "@xtuc/long" "4.2.2"
 
+"@webpack-cli/info@^1.1.0":
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/@webpack-cli/info/-/info-1.1.0.tgz#c596d5bc48418b39df00c5ed7341bf0f102dbff1"
+  integrity sha512-uNWSdaYHc+f3LdIZNwhdhkjjLDDl3jP2+XBqAq9H8DjrJUvlOKdP8TNruy1yEaDfgpAIgbSAN7pye4FEHg9tYQ==
+  dependencies:
+    envinfo "^7.7.3"
+
+"@webpack-cli/serve@^1.1.0":
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/@webpack-cli/serve/-/serve-1.1.0.tgz#13ad38f89b6e53d1133bac0006a128217a6ebf92"
+  integrity sha512-7RfnMXCpJ/NThrhq4gYQYILB18xWyoQcBey81oIyVbmgbc6m5ZHHyFK+DyH7pLHJf0p14MxL4mTsoPAgBSTpIg==
+
 "@xtuc/ieee754@^1.2.0":
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/@xtuc/ieee754/-/ieee754-1.2.0.tgz#eef014a3145ae477a1cbc00cd1e552336dceb790"
@@ -1947,6 +1959,11 @@ arr-union@^3.1.0:
   resolved "https://registry.yarnpkg.com/arr-union/-/arr-union-3.1.0.tgz#e39b09aea9def866a8f206e288af63919bae39c4"
   integrity sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ=
 
+array-back@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/array-back/-/array-back-4.0.1.tgz#9b80312935a52062e1a233a9c7abeb5481b30e90"
+  integrity sha512-Z/JnaVEXv+A9xabHzN43FiiiWEE7gPCRXMrVmRm00tWbjZRul1iHm7ECzlyNq1p4a4ATXz+G9FJ3GqGOkOV3fg==
+
 array-flatten@1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2"
@@ -2978,6 +2995,16 @@ combined-stream@^1.0.6, combined-stream@~1.0.6:
   dependencies:
     delayed-stream "~1.0.0"
 
+command-line-usage@^6.1.0:
+  version "6.1.1"
+  resolved "https://registry.yarnpkg.com/command-line-usage/-/command-line-usage-6.1.1.tgz#c908e28686108917758a49f45efb4f02f76bc03f"
+  integrity sha512-F59pEuAR9o1SF/bD0dQBDluhpT4jJQNWUHEuVBqpDmCUo6gPjCi+m9fCWnWZVR/oG6cMTUms4h+3NPl74wGXvA==
+  dependencies:
+    array-back "^4.0.1"
+    chalk "^2.4.2"
+    table-layout "^1.0.1"
+    typical "^5.2.0"
+
 commander@^2.20.0, commander@^2.8.1:
   version "2.20.3"
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33"
@@ -3197,7 +3224,7 @@ cross-env@^7.0.2:
   dependencies:
     cross-spawn "^7.0.1"
 
-cross-spawn@^6.0.0, cross-spawn@^6.0.5:
+cross-spawn@^6.0.0:
   version "6.0.5"
   resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4"
   integrity sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==
@@ -3541,6 +3568,11 @@ deep-extend@^0.5.1:
   resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.5.1.tgz#b894a9dd90d3023fbf1c55a394fb858eb2066f1f"
   integrity sha512-N8vBdOa+DF7zkRrDCsaOXoCs/E2fJfx9B9MrKnnSiHNh4ws7eSys6YQE4KvT1cecKmOASYQBhbKjeuDD9lT81w==
 
+deep-extend@~0.6.0:
+  version "0.6.0"
+  resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.6.0.tgz#c4fa7c95404a17a9c3e8ca7e1537312b736330ac"
+  integrity sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==
+
 deep-is@^0.1.3, deep-is@~0.1.3:
   version "0.1.3"
   resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34"
@@ -3634,11 +3666,6 @@ destroy@~1.0.4:
   resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80"
   integrity sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=
 
-detect-file@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/detect-file/-/detect-file-1.0.0.tgz#f0d66d03672a825cb1b73bdb3fe62310c8e552b7"
-  integrity sha1-8NZtA2cqglyxtzvbP+YjEMjlUrc=
-
 detect-newline@^3.0.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/detect-newline/-/detect-newline-3.1.0.tgz#576f5dfc63ae1a192ff192d8ad3af6308991b651"
@@ -3889,7 +3916,7 @@ end-of-stream@^1.0.0, end-of-stream@^1.1.0:
   dependencies:
     once "^1.4.0"
 
-enhanced-resolve@^4.1.1, enhanced-resolve@^4.3.0:
+enhanced-resolve@^4.3.0:
   version "4.3.0"
   resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-4.3.0.tgz#3b806f3bfafc1ec7de69551ef93cca46c1704126"
   integrity sha512-3e87LvavsdxyoCfGusJnrZ5G8SLPOFeHSNpZI/ATL9a5leXo2k0w6MKnbqhdBad9qTobSfB20Ld7UmgoNbAZkQ==
@@ -3898,7 +3925,7 @@ enhanced-resolve@^4.1.1, enhanced-resolve@^4.3.0:
     memory-fs "^0.5.0"
     tapable "^1.0.0"
 
-enquirer@^2.3.5:
+enquirer@^2.3.5, enquirer@^2.3.6:
   version "2.3.6"
   resolved "https://registry.yarnpkg.com/enquirer/-/enquirer-2.3.6.tgz#2a7fe5dd634a1e4125a975ec994ff5456dc3734d"
   integrity sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg==
@@ -3910,6 +3937,11 @@ entities@^2.0.0:
   resolved "https://registry.yarnpkg.com/entities/-/entities-2.0.3.tgz#5c487e5742ab93c15abb5da22759b8590ec03b7f"
   integrity sha512-MyoZ0jgnLvB2X3Lg5HqpFmn1kybDiIfEQmKzTb5apr51Rb+T3KdmMiqa70T+bhGnyv7bQ6WMj2QMHpGMmlrUYQ==
 
+envinfo@^7.7.3:
+  version "7.7.3"
+  resolved "https://registry.yarnpkg.com/envinfo/-/envinfo-7.7.3.tgz#4b2d8622e3e7366afb8091b23ed95569ea0208cc"
+  integrity sha512-46+j5QxbPWza0PB1i15nZx0xQ4I/EfQxg9J8Had3b408SV63nEtor2e+oiY63amTo9KTuh2a3XLObNwduxYwwA==
+
 errno@^0.1.3, errno@~0.1.7:
   version "0.1.7"
   resolved "https://registry.yarnpkg.com/errno/-/errno-0.1.7.tgz#4684d71779ad39af177e3f007996f7c67c852618"
@@ -4380,10 +4412,10 @@ execa@^1.0.0:
     signal-exit "^3.0.0"
     strip-eof "^1.0.0"
 
-execa@^4.0.0:
-  version "4.0.3"
-  resolved "https://registry.yarnpkg.com/execa/-/execa-4.0.3.tgz#0a34dabbad6d66100bd6f2c576c8669403f317f2"
-  integrity sha512-WFDXGHckXPWZX19t1kCsXzOpqX9LWYNqn4C+HqZlk/V0imTkzJZqf87ZBhvpHaftERYknpk0fjSylnXVlVgI0A==
+execa@^4.0.0, execa@^4.1.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/execa/-/execa-4.1.0.tgz#4e5491ad1572f2f17a77d388c6c857135b22847a"
+  integrity sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA==
   dependencies:
     cross-spawn "^7.0.0"
     get-stream "^5.0.0"
@@ -4423,13 +4455,6 @@ expand-brackets@^2.1.4:
     snapdragon "^0.8.1"
     to-regex "^3.0.1"
 
-expand-tilde@^2.0.0, expand-tilde@^2.0.2:
-  version "2.0.2"
-  resolved "https://registry.yarnpkg.com/expand-tilde/-/expand-tilde-2.0.2.tgz#97e801aa052df02454de46b02bf621642cdc8502"
-  integrity sha1-l+gBqgUt8CRU3kawK/YhZCzchQI=
-  dependencies:
-    homedir-polyfill "^1.0.1"
-
 expect@^26.6.2:
   version "26.6.2"
   resolved "https://registry.yarnpkg.com/expect/-/expect-26.6.2.tgz#c6b996bf26bf3fe18b67b2d0f51fc981ba934417"
@@ -4698,16 +4723,6 @@ find-up@^4.0.0, find-up@^4.1.0:
     locate-path "^5.0.0"
     path-exists "^4.0.0"
 
-findup-sync@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/findup-sync/-/findup-sync-3.0.0.tgz#17b108f9ee512dfb7a5c7f3c8b27ea9e1a9c08d1"
-  integrity sha512-YbffarhcicEhOrm4CtrwdKBdCuz576RLdhJDsIfvNtxUuhdRet1qZcsMjqbePtAseKdAnDyM/IyXbu7PRPRLYg==
-  dependencies:
-    detect-file "^1.0.0"
-    is-glob "^4.0.0"
-    micromatch "^3.0.4"
-    resolve-dir "^1.0.1"
-
 flat-cache@^1.2.1:
   version "1.3.4"
   resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-1.3.4.tgz#2c2ef77525cc2929007dfffa1dd314aa9c9dee6f"
@@ -4965,42 +4980,6 @@ glob@^7.0.0, glob@^7.0.3, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, gl
     once "^1.3.0"
     path-is-absolute "^1.0.0"
 
-global-modules@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-1.0.0.tgz#6d770f0eb523ac78164d72b5e71a8877265cc3ea"
-  integrity sha512-sKzpEkf11GpOFuw0Zzjzmt4B4UZwjOcG757PPvrfhxcLFbq0wpsgpOqxpxtxFiCG4DtG93M6XRVbF2oGdev7bg==
-  dependencies:
-    global-prefix "^1.0.1"
-    is-windows "^1.0.1"
-    resolve-dir "^1.0.0"
-
-global-modules@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-2.0.0.tgz#997605ad2345f27f51539bea26574421215c7780"
-  integrity sha512-NGbfmJBp9x8IxyJSd1P+otYK8vonoJactOogrVfFRIAEY1ukil8RSKDz2Yo7wh1oihl51l/r6W4epkeKJHqL8A==
-  dependencies:
-    global-prefix "^3.0.0"
-
-global-prefix@^1.0.1:
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/global-prefix/-/global-prefix-1.0.2.tgz#dbf743c6c14992593c655568cb66ed32c0122ebe"
-  integrity sha1-2/dDxsFJklk8ZVVoy2btMsASLr4=
-  dependencies:
-    expand-tilde "^2.0.2"
-    homedir-polyfill "^1.0.1"
-    ini "^1.3.4"
-    is-windows "^1.0.1"
-    which "^1.2.14"
-
-global-prefix@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/global-prefix/-/global-prefix-3.0.0.tgz#fc85f73064df69f50421f47f883fe5b913ba9b97"
-  integrity sha512-awConJSVCHVGND6x3tmMaKcQvwXLhjdkmomy2W+Goaui8YPgYgXJZewhg3fWC+DlfqqQuWg8AwqjGTD2nAPVWg==
-  dependencies:
-    ini "^1.3.5"
-    kind-of "^6.0.2"
-    which "^1.3.1"
-
 globals@^11.1.0:
   version "11.12.0"
   resolved "https://registry.yarnpkg.com/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e"
@@ -5217,13 +5196,6 @@ hoist-non-react-statics@^3.3.0, hoist-non-react-statics@^3.3.2:
   dependencies:
     react-is "^16.7.0"
 
-homedir-polyfill@^1.0.1:
-  version "1.0.3"
-  resolved "https://registry.yarnpkg.com/homedir-polyfill/-/homedir-polyfill-1.0.3.tgz#743298cef4e5af3e194161fbadcc2151d3a058e8"
-  integrity sha512-eSmmWE5bZTK2Nou4g0AI3zZ9rswp7GRKoKXS1BLUkvPviOqs4YTN1djQIqrXy9k5gEtdLPy86JjRwsNM9tnDcA==
-  dependencies:
-    parse-passwd "^1.0.0"
-
 hosted-git-info@^2.1.4:
   version "2.8.8"
   resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.8.tgz#7539bd4bc1e0e0a895815a2e0262420b12858488"
@@ -5497,11 +5469,6 @@ inherits@2.0.3:
   resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de"
   integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=
 
-ini@^1.3.4, ini@^1.3.5:
-  version "1.3.5"
-  resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.5.tgz#eee25f56db1c9ec6085e0c22778083f596abf927"
-  integrity sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==
-
 inquirer@^0.12.0:
   version "0.12.0"
   resolved "https://registry.yarnpkg.com/inquirer/-/inquirer-0.12.0.tgz#1ef2bfd63504df0bc75785fff8c2c41df12f077e"
@@ -5538,10 +5505,10 @@ internal-slot@^1.0.2:
     has "^1.0.3"
     side-channel "^1.0.2"
 
-interpret@^1.4.0:
-  version "1.4.0"
-  resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.4.0.tgz#665ab8bc4da27a774a40584e812e3e0fa45b1a1e"
-  integrity sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==
+interpret@^2.2.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/interpret/-/interpret-2.2.0.tgz#1a78a0b5965c40a5416d007ad6f50ad27c417df9"
+  integrity sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw==
 
 intersection-observer@^0.11.0:
   version "0.11.0"
@@ -5695,6 +5662,13 @@ is-core-module@^2.0.0:
   dependencies:
     has "^1.0.3"
 
+is-core-module@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.1.0.tgz#a4cc031d9b1aca63eecbd18a650e13cb4eeab946"
+  integrity sha512-YcV7BgVMRFRua2FqQzKtTDMz8iCuLEyGKjr70q8Zm1yy2qKcurbFEd79PAdHV77oL3NrAaOVQIbMmiHQCHB7ZA==
+  dependencies:
+    has "^1.0.3"
+
 is-data-descriptor@^0.1.4:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz#0b5ee648388e2c860282e793f1856fec3f301b56"
@@ -5939,7 +5913,7 @@ is-url@^1.2.4:
   resolved "https://registry.yarnpkg.com/is-url/-/is-url-1.2.4.tgz#04a4df46d28c4cff3d73d01ff06abeb318a1aa52"
   integrity sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==
 
-is-windows@^1.0.1, is-windows@^1.0.2:
+is-windows@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/is-windows/-/is-windows-1.0.2.tgz#d1850eb9791ecd18e6182ce12a30f396634bb19d"
   integrity sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==
@@ -6951,7 +6925,7 @@ methods@~1.1.2:
   resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee"
   integrity sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=
 
-micromatch@^3.0.4, micromatch@^3.1.10, micromatch@^3.1.4:
+micromatch@^3.1.10, micromatch@^3.1.4:
   version "3.1.10"
   resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-3.1.10.tgz#70859bc95c9840952f359a068a3fc49f9ecfac23"
   integrity sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==
@@ -7743,11 +7717,6 @@ parse-json@^5.0.0:
     json-parse-better-errors "^1.0.1"
     lines-and-columns "^1.1.6"
 
-parse-passwd@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/parse-passwd/-/parse-passwd-1.0.0.tgz#6d5b934a456993b23d37f40a382d6f1666a8e5c6"
-  integrity sha1-bVuTSkVpk7I9N/QKOC1vFmao5cY=
-
 parse5@5.1.1:
   version "5.1.1"
   resolved "https://registry.yarnpkg.com/parse5/-/parse5-5.1.1.tgz#f68e4e5ba1852ac2cadc00f4555fff6c2abb6178"
@@ -8975,6 +8944,13 @@ readline2@^1.0.1:
     is-fullwidth-code-point "^1.0.0"
     mute-stream "0.0.5"
 
+rechoir@^0.7.0:
+  version "0.7.0"
+  resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.7.0.tgz#32650fd52c21ab252aa5d65b19310441c7e03aca"
+  integrity sha512-ADsDEH2bvbjltXEP+hTIAmeFekTFK0V2BTxMkok6qILyAJEXV0AFfoWcAq4yfll5VdIMd/RVXq0lR+wQi5ZU3Q==
+  dependencies:
+    resolve "^1.9.0"
+
 redent@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/redent/-/redent-3.0.0.tgz#e557b7998316bb53c9f1f56fa626352c6963059f"
@@ -9010,6 +8986,11 @@ redis@^3.0.2:
     redis-errors "^1.2.0"
     redis-parser "^3.0.0"
 
+reduce-flatten@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/reduce-flatten/-/reduce-flatten-2.0.0.tgz#734fd84e65f375d7ca4465c69798c25c9d10ae27"
+  integrity sha512-EJ4UNY/U1t2P/2k6oqotuX2Cc3T6nxJwsM0N0asT7dhrtH1ltUxDn4NalSYmPE2rCkVpcf/X6R0wDwcFpzhd4w==
+
 redux-immutable@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/redux-immutable/-/redux-immutable-4.0.0.tgz#3a1a32df66366462b63691f0e1dc35e472bbc9f3"
@@ -9226,14 +9207,6 @@ resolve-cwd@^3.0.0:
   dependencies:
     resolve-from "^5.0.0"
 
-resolve-dir@^1.0.0, resolve-dir@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/resolve-dir/-/resolve-dir-1.0.1.tgz#79a40644c362be82f26effe739c9bb5382046f43"
-  integrity sha1-eaQGRMNivoLybv/nOcm7U4IEb0M=
-  dependencies:
-    expand-tilde "^2.0.0"
-    global-modules "^1.0.0"
-
 resolve-from@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-1.0.1.tgz#26cbfe935d1aeeeabb29bc3fe5aeb01e93d44226"
@@ -9277,6 +9250,14 @@ resolve@^1.10.0, resolve@^1.12.0, resolve@^1.13.1, resolve@^1.17.0, resolve@^1.1
     is-core-module "^2.0.0"
     path-parse "^1.0.6"
 
+resolve@^1.9.0:
+  version "1.19.0"
+  resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.19.0.tgz#1af5bf630409734a067cae29318aac7fa29a267c"
+  integrity sha512-rArEXAgsBG4UgRGcynxWIWKFvh/XZCcS8UJdHhwy91zwAvCZIbcs+vAbflgBnNjYMs/i/i+/Ux6IZhML1yPvxg==
+  dependencies:
+    is-core-module "^2.1.0"
+    path-parse "^1.0.6"
+
 restore-cursor@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-1.0.1.tgz#34661f46886327fed2991479152252df92daa541"
@@ -10254,6 +10235,16 @@ symbol-tree@^3.2.4:
   resolved "https://registry.yarnpkg.com/symbol-tree/-/symbol-tree-3.2.4.tgz#430637d248ba77e078883951fb9aa0eed7c63fa2"
   integrity sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==
 
+table-layout@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/table-layout/-/table-layout-1.0.1.tgz#8411181ee951278ad0638aea2f779a9ce42894f9"
+  integrity sha512-dEquqYNJiGwY7iPfZ3wbXDI944iqanTSchrACLL2nOB+1r+h1Nzu2eH+DuPPvWvm5Ry7iAPeFlgEtP5bIp5U7Q==
+  dependencies:
+    array-back "^4.0.1"
+    deep-extend "~0.6.0"
+    typical "^5.2.0"
+    wordwrapjs "^4.0.0"
+
 table@^3.7.8:
   version "3.8.3"
   resolved "https://registry.yarnpkg.com/table/-/table-3.8.3.tgz#2bbc542f0fda9861a755d3947fefd8b3f513855f"
@@ -10624,6 +10615,11 @@ typedarray@^0.0.6:
   resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777"
   integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=
 
+typical@^5.0.0, typical@^5.2.0:
+  version "5.2.0"
+  resolved "https://registry.yarnpkg.com/typical/-/typical-5.2.0.tgz#4daaac4f2b5315460804f0acf6cb69c52bb93066"
+  integrity sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==
+
 unicode-canonical-property-names-ecmascript@^1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz#2619800c4c825800efdd8343af7dd9933cbe2818"
@@ -10819,10 +10815,10 @@ uuid@^8.3.0, uuid@^8.3.1:
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.1.tgz#2ba2e6ca000da60fce5a196954ab241131e05a31"
   integrity sha512-FOmRr+FmWEIG8uhZv6C2bTgEVXsHk08kE7mPlrBbEe+c3r9pjceVPgupIfNIhc4yx55H69OXANrUaSuu9eInKg==
 
-v8-compile-cache@^2.0.3, v8-compile-cache@^2.1.1:
-  version "2.1.1"
-  resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.1.1.tgz#54bc3cdd43317bca91e35dcaf305b1a7237de745"
-  integrity sha512-8OQ9CL+VWyt3JStj7HX7/ciTL2V3Rl1Wf5OL+SNTm0yK1KvtReVulksyeRnCANHHuUxHlQig+JJDlUhBt1NQDQ==
+v8-compile-cache@^2.0.3, v8-compile-cache@^2.2.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.2.0.tgz#9471efa3ef9128d2f7c6a7ca39c4dd6b5055b132"
+  integrity sha512-gTpR5XQNKFwOd4clxfnhaqvfqMpqEwr4tOtCyz4MtYZX2JYhfr1JvBFKdS+7K/9rfpZR3VLX+YWBbKoxCgS43Q==
 
 v8-to-istanbul@^7.0.0:
   version "7.0.0"
@@ -10976,22 +10972,24 @@ webpack-bundle-analyzer@^4.1.0:
     opener "^1.5.2"
     ws "^7.3.1"
 
-webpack-cli@^3.3.12:
-  version "3.3.12"
-  resolved "https://registry.yarnpkg.com/webpack-cli/-/webpack-cli-3.3.12.tgz#94e9ada081453cd0aa609c99e500012fd3ad2d4a"
-  integrity sha512-NVWBaz9k839ZH/sinurM+HcDvJOTXwSjYp1ku+5XKeOC03z8v5QitnK/x+lAxGXFyhdayoIf/GOpv85z3/xPag==
+webpack-cli@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/webpack-cli/-/webpack-cli-4.2.0.tgz#10a09030ad2bd4d8b0f78322fba6ea43ec56aaaa"
+  integrity sha512-EIl3k88vaF4fSxWSgtAQR+VwicfLMTZ9amQtqS4o+TDPW9HGaEpbFBbAZ4A3ZOT5SOnMxNOzROsSTPiE8tBJPA==
   dependencies:
-    chalk "^2.4.2"
-    cross-spawn "^6.0.5"
-    enhanced-resolve "^4.1.1"
-    findup-sync "^3.0.0"
-    global-modules "^2.0.0"
-    import-local "^2.0.0"
-    interpret "^1.4.0"
-    loader-utils "^1.4.0"
-    supports-color "^6.1.0"
-    v8-compile-cache "^2.1.1"
-    yargs "^13.3.2"
+    "@webpack-cli/info" "^1.1.0"
+    "@webpack-cli/serve" "^1.1.0"
+    colorette "^1.2.1"
+    command-line-usage "^6.1.0"
+    commander "^6.2.0"
+    enquirer "^2.3.6"
+    execa "^4.1.0"
+    import-local "^3.0.2"
+    interpret "^2.2.0"
+    leven "^3.1.0"
+    rechoir "^0.7.0"
+    v8-compile-cache "^2.2.0"
+    webpack-merge "^4.2.2"
 
 webpack-dev-middleware@^3.7.2:
   version "3.7.2"
@@ -11051,6 +11049,13 @@ webpack-log@^2.0.0:
     ansi-colors "^3.0.0"
     uuid "^3.3.2"
 
+webpack-merge@^4.2.2:
+  version "4.2.2"
+  resolved "https://registry.yarnpkg.com/webpack-merge/-/webpack-merge-4.2.2.tgz#a27c52ea783d1398afd2087f547d7b9d2f43634d"
+  integrity sha512-TUE1UGoTX2Cd42j3krGYqObZbOD+xF7u28WB7tfUordytSjbWTIjK/8V0amkBfTYN4/pB/GIDlJZZ657BGG19g==
+  dependencies:
+    lodash "^4.17.15"
+
 webpack-merge@^5.4.0:
   version "5.4.0"
   resolved "https://registry.yarnpkg.com/webpack-merge/-/webpack-merge-5.4.0.tgz#81bef0a7d23fc1e6c24b06ad8bf22ddeb533a3a3"
@@ -11143,7 +11148,7 @@ which-module@^2.0.0:
   resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a"
   integrity sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=
 
-which@^1.2.14, which@^1.2.9, which@^1.3.1:
+which@^1.2.9:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a"
   integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==
@@ -11179,6 +11184,14 @@ word-wrap@^1.2.3, word-wrap@~1.2.3:
   resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c"
   integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==
 
+wordwrapjs@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/wordwrapjs/-/wordwrapjs-4.0.0.tgz#9aa9394155993476e831ba8e59fb5795ebde6800"
+  integrity sha512-Svqw723a3R34KvsMgpjFBYCgNOSdcW3mQFK4wIfhGQhtaFVOJmdYoXgi63ne3dTlWgatVcUc7t4HtQ/+bUVIzQ==
+  dependencies:
+    reduce-flatten "^2.0.0"
+    typical "^5.0.0"
+
 worker-farm@^1.7.0:
   version "1.7.0"
   resolved "https://registry.yarnpkg.com/worker-farm/-/worker-farm-1.7.0.tgz#26a94c5391bbca926152002f69b84a4bf772e5a8"

From 96c1e7132971877fba308c51cd42306f0b1bf166 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Thu, 19 Nov 2020 17:48:13 +0100
Subject: [PATCH 27/31] Add import/export feature for bookmarks (#14956)

* Add ability to export bookmarks

* Add support for importing bookmarks

* Add bookmark import tests

* Add bookmarks export test
---
 .../settings/exports/bookmarks_controller.rb  | 19 ++++++++
 app/models/export.rb                          | 12 +++++
 app/models/import.rb                          |  2 +-
 app/services/import_service.rb                | 45 +++++++++++++++++++
 app/views/settings/exports/show.html.haml     |  4 ++
 config/locales/en.yml                         |  2 +
 config/routes.rb                              |  1 +
 .../exports/bookmarks_controller_specs.rb     | 17 +++++++
 spec/fixtures/files/bookmark-imports.txt      |  4 ++
 spec/services/import_service_spec.rb          | 42 +++++++++++++++++
 10 files changed, 147 insertions(+), 1 deletion(-)
 create mode 100644 app/controllers/settings/exports/bookmarks_controller.rb
 create mode 100644 spec/controllers/settings/exports/bookmarks_controller_specs.rb
 create mode 100644 spec/fixtures/files/bookmark-imports.txt

diff --git a/app/controllers/settings/exports/bookmarks_controller.rb b/app/controllers/settings/exports/bookmarks_controller.rb
new file mode 100644
index 00000000000..c12e2f147ac
--- /dev/null
+++ b/app/controllers/settings/exports/bookmarks_controller.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Settings
+  module Exports
+    class BookmarksController < BaseController
+      include ExportControllerConcern
+
+      def index
+        send_export_file
+      end
+
+      private
+
+      def export_data
+        @export.to_bookmarks_csv
+      end
+    end
+  end
+end
diff --git a/app/models/export.rb b/app/models/export.rb
index cab01f11ad0..5216eed5ea7 100644
--- a/app/models/export.rb
+++ b/app/models/export.rb
@@ -9,6 +9,14 @@ class Export
     @account = account
   end
 
+  def to_bookmarks_csv
+    CSV.generate do |csv|
+      account.bookmarks.includes(:status).reorder(id: :desc).each do |bookmark|
+        csv << [ActivityPub::TagManager.instance.uri_for(bookmark.status)]
+      end
+    end
+  end
+
   def to_blocked_accounts_csv
     to_csv account.blocking.select(:username, :domain)
   end
@@ -55,6 +63,10 @@ class Export
     account.statuses_count
   end
 
+  def total_bookmarks
+    account.bookmarks.count
+  end
+
   def total_follows
     account.following_count
   end
diff --git a/app/models/import.rb b/app/models/import.rb
index c78a04d0734..70245328959 100644
--- a/app/models/import.rb
+++ b/app/models/import.rb
@@ -24,7 +24,7 @@ class Import < ApplicationRecord
 
   belongs_to :account
 
-  enum type: [:following, :blocking, :muting, :domain_blocking]
+  enum type: [:following, :blocking, :muting, :domain_blocking, :bookmarks]
 
   validates :type, presence: true
 
diff --git a/app/services/import_service.rb b/app/services/import_service.rb
index 7e55452de73..288e47f1ea1 100644
--- a/app/services/import_service.rb
+++ b/app/services/import_service.rb
@@ -18,6 +18,8 @@ class ImportService < BaseService
       import_mutes!
     when 'domain_blocking'
       import_domain_blocks!
+    when 'bookmarks'
+      import_bookmarks!
     end
   end
 
@@ -88,6 +90,39 @@ class ImportService < BaseService
     end
   end
 
+  def import_bookmarks!
+    parse_import_data!(['#uri'])
+    items = @data.take(ROWS_PROCESSING_LIMIT).map { |row| row['#uri'].strip }
+
+    if @import.overwrite?
+      presence_hash = items.each_with_object({}) { |id, mapping| mapping[id] = true }
+
+      @account.bookmarks.find_each do |bookmark|
+        if presence_hash[bookmark.status.uri]
+          items.delete(bookmark.status.uri)
+        else
+          bookmark.destroy!
+        end
+      end
+    end
+
+    statuses = items.map do |uri|
+      status = ActivityPub::TagManager.instance.uri_to_resource(uri, Status)
+      next if status.nil? && ActivityPub::TagManager.instance.local_uri?(uri)
+
+      status || ActivityPub::FetchRemoteStatusService.new.call(uri)
+    end.compact
+
+    account_ids         = statuses.map(&:account_id)
+    preloaded_relations = relations_map_for_account(@account, account_ids)
+
+    statuses.keep_if { |status| StatusPolicy.new(@account, status, preloaded_relations).show? }
+
+    statuses.each do |status|
+      @account.bookmarks.find_or_create_by!(account: @account, status: status)
+    end
+  end
+
   def parse_import_data!(default_headers)
     data = CSV.parse(import_data, headers: true)
     data = CSV.parse(import_data, headers: default_headers) unless data.headers&.first&.strip&.include?(' ')
@@ -101,4 +136,14 @@ class ImportService < BaseService
   def follow_limit
     FollowLimitValidator.limit_for_account(@account)
   end
+
+  def relations_map_for_account(account, account_ids)
+    {
+      blocking: {},
+      blocked_by: Account.blocked_by_map(account_ids, account.id),
+      muting: {},
+      following: Account.following_map(account_ids, account.id),
+      domain_blocking_by_domain: {},
+    }
+  end
 end
diff --git a/app/views/settings/exports/show.html.haml b/app/views/settings/exports/show.html.haml
index 0bb80e93726..18b52c0c2c6 100644
--- a/app/views/settings/exports/show.html.haml
+++ b/app/views/settings/exports/show.html.haml
@@ -36,6 +36,10 @@
         %th= t('exports.domain_blocks')
         %td= number_with_delimiter @export.total_domain_blocks
         %td= table_link_to 'download', t('exports.csv'), settings_exports_domain_blocks_path(format: :csv)
+      %tr
+        %th= t('exports.bookmarks')
+        %td= number_with_delimiter @export.total_bookmarks
+        %td= table_link_to 'download', t('bookmarks.csv'), settings_exports_bookmarks_path(format: :csv)
 
 %hr.spacer/
 
diff --git a/config/locales/en.yml b/config/locales/en.yml
index bec09908278..263ffcdc772 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -842,6 +842,7 @@ en:
       request: Request your archive
       size: Size
     blocks: You block
+    bookmarks: Bookmarks
     csv: CSV
     domain_blocks: Domain blocks
     lists: Lists
@@ -918,6 +919,7 @@ en:
     success: Your data was successfully uploaded and will now be processed in due time
     types:
       blocking: Blocking list
+      bookmarks: Bookmarks
       domain_blocking: Domain blocking list
       following: Following list
       muting: Muting list
diff --git a/config/routes.rb b/config/routes.rb
index 54c76799ca9..a534b433e07 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -125,6 +125,7 @@ Rails.application.routes.draw do
       resources :mutes, only: :index, controller: :muted_accounts
       resources :lists, only: :index, controller: :lists
       resources :domain_blocks, only: :index, controller: :blocked_domains
+      resources :bookmarks, only: :index, controller: :bookmarks
     end
 
     resources :two_factor_authentication_methods, only: [:index] do
diff --git a/spec/controllers/settings/exports/bookmarks_controller_specs.rb b/spec/controllers/settings/exports/bookmarks_controller_specs.rb
new file mode 100644
index 00000000000..85761577bd9
--- /dev/null
+++ b/spec/controllers/settings/exports/bookmarks_controller_specs.rb
@@ -0,0 +1,17 @@
+require 'rails_helper'
+
+describe Settings::Exports::BookmarksController do
+  render_views
+
+  describe 'GET #index' do
+    it 'returns a csv of the bookmarked toots' do
+      user = Fabricate(:user)
+      user.account.bookmarks.create!(status: Fabricate(:status, uri: 'https://foo.bar/statuses/1312'))
+
+      sign_in user, scope: :user
+      get :index, format: :csv
+
+      expect(response.body).to eq "https://foo.bar/statuses/1312\n"
+    end
+  end
+end
diff --git a/spec/fixtures/files/bookmark-imports.txt b/spec/fixtures/files/bookmark-imports.txt
new file mode 100644
index 00000000000..7cc8901a0ac
--- /dev/null
+++ b/spec/fixtures/files/bookmark-imports.txt
@@ -0,0 +1,4 @@
+https://example.com/statuses/1312
+https://local.com/users/foo/statuses/42
+https://unknown-remote.com/users/bar/statuses/1
+https://example.com/statuses/direct
diff --git a/spec/services/import_service_spec.rb b/spec/services/import_service_spec.rb
index b1909d4fd14..764225aa729 100644
--- a/spec/services/import_service_spec.rb
+++ b/spec/services/import_service_spec.rb
@@ -1,6 +1,8 @@
 require 'rails_helper'
 
 RSpec.describe ImportService, type: :service do
+  include RoutingHelper
+
   let!(:account) { Fabricate(:account, locked: false) }
   let!(:bob)     { Fabricate(:account, username: 'bob', locked: false) }
   let!(:eve)     { Fabricate(:account, username: 'eve', domain: 'example.com', locked: false, protocol: :activitypub, inbox_url: 'https://example.com/inbox') }
@@ -169,4 +171,44 @@ RSpec.describe ImportService, type: :service do
       end
     end
   end
+
+  context 'import bookmarks' do
+    subject { ImportService.new }
+
+    let(:csv) { attachment_fixture('bookmark-imports.txt') }
+
+    around(:each) do |example|
+      local_before = Rails.configuration.x.local_domain
+      web_before = Rails.configuration.x.web_domain
+      Rails.configuration.x.local_domain = 'local.com'
+      Rails.configuration.x.web_domain = 'local.com'
+      example.run
+      Rails.configuration.x.web_domain = web_before
+      Rails.configuration.x.local_domain = local_before
+    end
+
+    let(:local_account)  { Fabricate(:account, username: 'foo', domain: '') }
+    let!(:remote_status) { Fabricate(:status, uri: 'https://example.com/statuses/1312') }
+    let!(:direct_status) { Fabricate(:status, uri: 'https://example.com/statuses/direct', visibility: :direct) }
+
+    before do
+      service = double
+      allow(ActivityPub::FetchRemoteStatusService).to receive(:new).and_return(service)
+      allow(service).to receive(:call).with('https://unknown-remote.com/users/bar/statuses/1') do
+        Fabricate(:status, uri: 'https://unknown-remote.com/users/bar/statuses/1')
+      end
+    end
+
+    describe 'when no bookmarks are set' do
+      let(:import) { Import.create(account: account, type: 'bookmarks', data: csv) }
+      it 'adds the toots the user has access to to bookmarks' do
+        local_status = Fabricate(:status, account: local_account, uri: 'https://local.com/users/foo/statuses/42', id: 42, local: true)
+        subject.call(import)
+        expect(account.bookmarks.map(&:status).map(&:id)).to include(local_status.id)
+        expect(account.bookmarks.map(&:status).map(&:id)).to include(remote_status.id)
+        expect(account.bookmarks.map(&:status).map(&:id)).not_to include(direct_status.id)
+        expect(account.bookmarks.count).to eq 3
+      end
+    end
+  end
 end

From 8b8004a9626442ae31e4dffd79e874e9cde050c6 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Thu, 19 Nov 2020 19:52:06 +0100
Subject: [PATCH 28/31] Fix webfinger redirect handling in
 ResolveAccountService (#15187)

* Fix webfinger redirect handling in ResolveAccountService

ResolveAccountService#process_webfinger! handled a one-step webfinger
redirection, but only accepting the result if it matched the exact URI passed
as input, defeating the point of a redirection check.

Instead, use the same logic as in `ActivityPub::FetchRemoteAccountService`,
updating the resulting `acct:` URI with the result of the first webfinger
query.

* Add tests
---
 app/services/resolve_account_service.rb       | 23 +++++---
 spec/services/resolve_account_service_spec.rb | 52 ++++++++++++++++---
 2 files changed, 60 insertions(+), 15 deletions(-)

diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb
index c5291fa67c8..74b0b82d04c 100644
--- a/app/services/resolve_account_service.rb
+++ b/app/services/resolve_account_service.rb
@@ -29,6 +29,7 @@ class ResolveAccountService < BaseService
     # At this point we are in need of a Webfinger query, which may
     # yield us a different username/domain through a redirect
     process_webfinger!(@uri)
+    @domain = nil if TagManager.instance.local_domain?(@domain)
 
     # Because the username/domain pair may be different than what
     # we already checked, we need to check if we've already got
@@ -78,25 +79,31 @@ class ResolveAccountService < BaseService
     @uri = [@username, @domain].compact.join('@')
   end
 
-  def process_webfinger!(uri, redirected = false)
+  def process_webfinger!(uri)
     @webfinger                           = webfinger!("acct:#{uri}")
-    confirmed_username, confirmed_domain = @webfinger.subject.gsub(/\Aacct:/, '').split('@')
+    confirmed_username, confirmed_domain = split_acct(@webfinger.subject)
 
     if confirmed_username.casecmp(@username).zero? && confirmed_domain.casecmp(@domain).zero?
       @username = confirmed_username
       @domain   = confirmed_domain
-      @uri      = uri
-    elsif !redirected
-      return process_webfinger!("#{confirmed_username}@#{confirmed_domain}", true)
-    else
-      raise Webfinger::RedirectError, "The URI #{uri} tries to hijack #{@username}@#{@domain}"
+      return
     end
 
-    @domain = nil if TagManager.instance.local_domain?(@domain)
+    # Account doesn't match, so it may have been redirected
+    @webfinger         = webfinger!("acct:#{confirmed_username}@#{confirmed_domain}")
+    @username, @domain = split_acct(@webfinger.subject)
+
+    unless confirmed_username.casecmp(@username).zero? && confirmed_domain.casecmp(@domain).zero?
+      raise Webfinger::RedirectError, "The URI #{uri} tries to hijack #{@username}@#{@domain}"
+    end
   rescue Webfinger::GoneError
     @gone = true
   end
 
+  def split_acct(acct)
+    acct.gsub(/\Aacct:/, '').split('@')
+  end
+
   def process_account!
     return unless activitypub_ready?
 
diff --git a/spec/services/resolve_account_service_spec.rb b/spec/services/resolve_account_service_spec.rb
index 76cb9ed8dc3..5bd0ec264d1 100644
--- a/spec/services/resolve_account_service_spec.rb
+++ b/spec/services/resolve_account_service_spec.rb
@@ -4,11 +4,8 @@ RSpec.describe ResolveAccountService, type: :service do
   subject { described_class.new }
 
   before do
-    stub_request(:get, "https://quitter.no/.well-known/host-meta").to_return(request_fixture('.host-meta.txt'))
-    stub_request(:get, "https://example.com/.well-known/webfinger?resource=acct:catsrgr8@example.com").to_return(status: 404)
     stub_request(:get, "https://example.com/.well-known/host-meta").to_return(status: 404)
     stub_request(:get, "https://quitter.no/avatar/7477-300-20160211190340.png").to_return(request_fixture('avatar.txt'))
-    stub_request(:get, "https://quitter.no/.well-known/webfinger?resource=acct:catsrgr8@quitter.no").to_return(status: 404)
     stub_request(:get, "https://ap.example.com/.well-known/webfinger?resource=acct:foo@ap.example.com").to_return(request_fixture('activitypub-webfinger.txt'))
     stub_request(:get, "https://ap.example.com/users/foo").to_return(request_fixture('activitypub-actor.txt'))
     stub_request(:get, "https://ap.example.com/users/foo.atom").to_return(request_fixture('activitypub-feed.txt'))
@@ -16,12 +13,25 @@ RSpec.describe ResolveAccountService, type: :service do
     stub_request(:get, 'https://example.com/.well-known/webfinger?resource=acct:hoge@example.com').to_return(status: 410)
   end
 
-  it 'returns nil if no such user can be resolved via webfinger' do
-    expect(subject.call('catsrgr8@quitter.no')).to be_nil
+  context 'when there is an LRDD endpoint but no resolvable account' do
+    before do
+      stub_request(:get, "https://quitter.no/.well-known/host-meta").to_return(request_fixture('.host-meta.txt'))
+      stub_request(:get, "https://quitter.no/.well-known/webfinger?resource=acct:catsrgr8@quitter.no").to_return(status: 404)
+    end
+
+    it 'returns nil' do
+      expect(subject.call('catsrgr8@quitter.no')).to be_nil
+    end
   end
 
-  it 'returns nil if the domain does not have webfinger' do
-    expect(subject.call('catsrgr8@example.com')).to be_nil
+  context 'when there is no LRDD endpoint nor resolvable account' do
+    before do
+      stub_request(:get, "https://example.com/.well-known/webfinger?resource=acct:catsrgr8@example.com").to_return(status: 404)
+    end
+
+    it 'returns nil' do
+      expect(subject.call('catsrgr8@example.com')).to be_nil
+    end
   end
 
   context 'when webfinger returns http gone' do
@@ -48,6 +58,34 @@ RSpec.describe ResolveAccountService, type: :service do
     end
   end
 
+  context 'with a legitimate webfinger redirection' do
+    before do
+      webfinger = { subject: 'acct:foo@ap.example.com', links: [{ rel: 'self', href: 'https://ap.example.com/users/foo' }] }
+      stub_request(:get, 'https://redirected.example.com/.well-known/webfinger?resource=acct:Foo@redirected.example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' })
+    end
+
+    it 'returns new remote account' do
+      account = subject.call('Foo@redirected.example.com')
+
+      expect(account.activitypub?).to eq true
+      expect(account.acct).to eq 'foo@ap.example.com'
+      expect(account.inbox_url).to eq 'https://ap.example.com/users/foo/inbox'
+    end
+  end
+
+  context 'with too many webfinger redirections' do
+    before do
+      webfinger = { subject: 'acct:foo@evil.example.com', links: [{ rel: 'self', href: 'https://ap.example.com/users/foo' }] }
+      stub_request(:get, 'https://redirected.example.com/.well-known/webfinger?resource=acct:Foo@redirected.example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' })
+      webfinger2 = { subject: 'acct:foo@ap.example.com', links: [{ rel: 'self', href: 'https://ap.example.com/users/foo' }] }
+      stub_request(:get, 'https://evil.example.com/.well-known/webfinger?resource=acct:foo@evil.example.com').to_return(body: Oj.dump(webfinger2), headers: { 'Content-Type': 'application/jrd+json' })
+    end
+
+    it 'returns new remote account' do
+      expect { subject.call('Foo@redirected.example.com') }.to raise_error Webfinger::RedirectError
+    end
+  end
+
   context 'with an ActivityPub account' do
     it 'returns new remote account' do
       account = subject.call('foo@ap.example.com')

From b82aa33deab18e499fb32d37c1915d986c3ce88e Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 15 Nov 2020 14:24:54 +0100
Subject: [PATCH 29/31] [Glitch] Add hotkeys for audio/video control

Port 04a079e7230bc4e73b897e3c4462011cddf36474 to glitch-soc

Signed-off-by: Thibaut Girka <thib@sitedethib.com>
---
 .../flavours/glitch/features/audio/index.js   | 51 +++++++++++-
 .../flavours/glitch/features/video/index.js   | 78 +++++++++++++++++++
 2 files changed, 128 insertions(+), 1 deletion(-)

diff --git a/app/javascript/flavours/glitch/features/audio/index.js b/app/javascript/flavours/glitch/features/audio/index.js
index c050a63a9c9..ac0468f7021 100644
--- a/app/javascript/flavours/glitch/features/audio/index.js
+++ b/app/javascript/flavours/glitch/features/audio/index.js
@@ -392,13 +392,59 @@ class Audio extends React.PureComponent {
     return this.props.foregroundColor || '#ffffff';
   }
 
+  seekBy (time) {
+    const currentTime = this.audio.currentTime + time;
+
+    if (!isNaN(currentTime)) {
+      this.setState({ currentTime }, () => {
+        this.audio.currentTime = currentTime;
+      });
+    }
+  }
+
+  handleAudioKeyDown = e => {
+    // On the audio element or the seek bar, we can safely use the space bar
+    // for playback control because there are no buttons to press
+
+    if (e.key === ' ') {
+      e.preventDefault();
+      e.stopPropagation();
+      this.togglePlay();
+    }
+  }
+
+  handleKeyDown = e => {
+    switch(e.key) {
+    case 'k':
+      e.preventDefault();
+      e.stopPropagation();
+      this.togglePlay();
+      break;
+    case 'm':
+      e.preventDefault();
+      e.stopPropagation();
+      this.toggleMute();
+      break;
+    case 'j':
+      e.preventDefault();
+      e.stopPropagation();
+      this.seekBy(-10);
+      break;
+    case 'l':
+      e.preventDefault();
+      e.stopPropagation();
+      this.seekBy(10);
+      break;
+    }
+  }
+
   render () {
     const { src, intl, alt, editable, autoPlay } = this.props;
     const { paused, muted, volume, currentTime, duration, buffer, dragging } = this.state;
     const progress = Math.min((currentTime / duration) * 100, 100);
 
     return (
-      <div className={classNames('audio-player', { editable })} ref={this.setPlayerRef} style={{ backgroundColor: this._getBackgroundColor(), color: this._getForegroundColor(), width: '100%', height: this.props.fullscreen ? '100%' : (this.state.height || this.props.height) }} onMouseEnter={this.handleMouseEnter} onMouseLeave={this.handleMouseLeave}>
+      <div className={classNames('audio-player', { editable })} ref={this.setPlayerRef} style={{ backgroundColor: this._getBackgroundColor(), color: this._getForegroundColor(), width: '100%', height: this.props.fullscreen ? '100%' : (this.state.height || this.props.height) }} onMouseEnter={this.handleMouseEnter} onMouseLeave={this.handleMouseLeave} tabIndex='0' onKeyDown={this.handleKeyDown}>
         <audio
           src={src}
           ref={this.setAudioRef}
@@ -412,12 +458,14 @@ class Audio extends React.PureComponent {
 
         <canvas
           role='button'
+          tabIndex='0'
           className='audio-player__canvas'
           width={this.state.width}
           height={this.state.height}
           style={{ width: '100%', position: 'absolute', top: 0, left: 0 }}
           ref={this.setCanvasRef}
           onClick={this.togglePlay}
+          onKeyDown={this.handleAudioKeyDown}
           title={alt}
           aria-label={alt}
         />
@@ -438,6 +486,7 @@ class Audio extends React.PureComponent {
             className={classNames('video-player__seek__handle', { active: dragging })}
             tabIndex='0'
             style={{ left: `${progress}%`, backgroundColor: this._getAccentColor() }}
+            onKeyDown={this.handleAudioKeyDown}
           />
         </div>
 
diff --git a/app/javascript/flavours/glitch/features/video/index.js b/app/javascript/flavours/glitch/features/video/index.js
index 87081285691..21a2594e7b4 100644
--- a/app/javascript/flavours/glitch/features/video/index.js
+++ b/app/javascript/flavours/glitch/features/video/index.js
@@ -279,6 +279,81 @@ class Video extends React.PureComponent {
     }
   }, 15);
 
+  seekBy (time) {
+    const currentTime = this.video.currentTime + time;
+
+    if (!isNaN(currentTime)) {
+      this.setState({ currentTime }, () => {
+        this.video.currentTime = currentTime;
+      });
+    }
+  }
+
+  handleVideoKeyDown = e => {
+    // On the video element or the seek bar, we can safely use the space bar
+    // for playback control because there are no buttons to press
+
+    if (e.key === ' ') {
+      e.preventDefault();
+      e.stopPropagation();
+      this.togglePlay();
+    }
+  }
+
+  handleKeyDown = e => {
+    const frameTime = 1 / 25;
+
+    switch(e.key) {
+    case 'k':
+      e.preventDefault();
+      e.stopPropagation();
+      this.togglePlay();
+      break;
+    case 'm':
+      e.preventDefault();
+      e.stopPropagation();
+      this.toggleMute();
+      break;
+    case 'f':
+      e.preventDefault();
+      e.stopPropagation();
+      this.toggleFullscreen();
+      break;
+    case 'j':
+      e.preventDefault();
+      e.stopPropagation();
+      this.seekBy(-10);
+      break;
+    case 'l':
+      e.preventDefault();
+      e.stopPropagation();
+      this.seekBy(10);
+      break;
+    case ',':
+      e.preventDefault();
+      e.stopPropagation();
+      this.seekBy(-frameTime);
+      break;
+    case '.':
+      e.preventDefault();
+      e.stopPropagation();
+      this.seekBy(frameTime);
+      break;
+    }
+
+    // If we are in fullscreen mode, we don't want any hotkeys
+    // interacting with the UI that's not visible
+
+    if (this.state.fullscreen) {
+      e.preventDefault();
+      e.stopPropagation();
+
+      if (e.key === 'Escape') {
+        exitFullscreen();
+      }
+    }
+  }
+
   togglePlay = () => {
     if (this.state.paused) {
       this.setState({ paused: false }, () => this.video.play());
@@ -504,6 +579,7 @@ class Video extends React.PureComponent {
         onMouseEnter={this.handleMouseEnter}
         onMouseLeave={this.handleMouseLeave}
         onMouseDown={this.handleMouseDownRoot}
+        onKeyDown={this.handleKeyDown}
         tabIndex={0}
       >
         <Blurhash
@@ -528,6 +604,7 @@ class Video extends React.PureComponent {
           height={height}
           volume={volume}
           onClick={this.togglePlay}
+          onKeyDown={this.handleVideoKeyDown}
           onPlay={this.handlePlay}
           onPause={this.handlePause}
           onLoadedData={this.handleLoadedData}
@@ -550,6 +627,7 @@ class Video extends React.PureComponent {
               className={classNames('video-player__seek__handle', { active: dragging })}
               tabIndex='0'
               style={{ left: `${progress}%` }}
+              onKeyDown={this.handleVideoKeyDown}
             />
           </div>
 

From 15be0bde8a7d45ed032dd8b5a634216ccb9a056b Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Wed, 18 Nov 2020 18:01:58 +0100
Subject: [PATCH 30/31] [Glitch] Fix image uploads being random data when
 canvas read access is blocked

Port d8d43a427a549cf063de3f6b3c22a08a06f53ffa to glitch-soc

Signed-off-by: Thibaut Girka <thib@sitedethib.com>
---
 .../flavours/glitch/util/resize_image.js      | 50 +++++++++++++++----
 1 file changed, 41 insertions(+), 9 deletions(-)

diff --git a/app/javascript/flavours/glitch/util/resize_image.js b/app/javascript/flavours/glitch/util/resize_image.js
index 8c89b2841a1..57fc6a6ec5a 100644
--- a/app/javascript/flavours/glitch/util/resize_image.js
+++ b/app/javascript/flavours/glitch/util/resize_image.js
@@ -41,6 +41,45 @@ const dropOrientationIfNeeded = (orientation) => new Promise(resolve => {
   }
 });
 
+// Some browsers don't allow reading from a canvas and instead return all-white
+// or randomized data. Use a pre-defined image to check if reading the canvas
+// works.
+const checkCanvasReliability = () => new Promise((resolve, reject) => {
+  switch(_browser_quirks['canvas-read-unreliable']) {
+  case true:
+    reject('Canvas reading unreliable');
+    break;
+  case false:
+    resolve();
+    break;
+  default:
+    // 2×2 GIF with white, red, green and blue pixels
+    const testImageURL =
+      'data:image/gif;base64,R0lGODdhAgACAKEDAAAA//8AAAD/AP///ywAAAAAAgACAAACA1wEBQA7';
+    const refData =
+      [255, 255, 255, 255,  255, 0, 0, 255,  0, 255, 0, 255,  0, 0, 255, 255];
+    const img = new Image();
+    img.onload = () => {
+      const canvas  = document.createElement('canvas');
+      const context = canvas.getContext('2d');
+      context.drawImage(img, 0, 0, 2, 2);
+      const imageData = context.getImageData(0, 0, 2, 2);
+      if (imageData.data.every((x, i) => refData[i] === x)) {
+        _browser_quirks['canvas-read-unreliable'] = false;
+        resolve();
+      } else {
+        _browser_quirks['canvas-read-unreliable'] = true;
+        reject('Canvas reading unreliable');
+      }
+    };
+    img.onerror = () => {
+      _browser_quirks['canvas-read-unreliable'] = true;
+      reject('Failed to load test image');
+    };
+    img.src = testImageURL;
+  }
+});
+
 const getImageUrl = inputFile => new Promise((resolve, reject) => {
   if (window.URL && URL.createObjectURL) {
     try {
@@ -110,14 +149,6 @@ const processImage = (img, { width, height, orientation, type = 'image/png' }) =
 
   context.drawImage(img, 0, 0, width, height);
 
-  // The Tor Browser and maybe other browsers may prevent reading from canvas
-  // and return an all-white image instead. Assume reading failed if the resized
-  // image is perfectly white.
-  const imageData = context.getImageData(0, 0, width, height);
-  if (imageData.data.every(value => value === 255)) {
-    throw 'Failed to read from canvas';
-  }
-
   canvas.toBlob(resolve, type);
 });
 
@@ -127,7 +158,8 @@ const resizeImage = (img, type = 'image/png') => new Promise((resolve, reject) =
   const newWidth  = Math.round(Math.sqrt(MAX_IMAGE_PIXELS * (width / height)));
   const newHeight = Math.round(Math.sqrt(MAX_IMAGE_PIXELS * (height / width)));
 
-  getOrientation(img, type)
+  checkCanvasReliability()
+    .then(getOrientation(img, type))
     .then(orientation => processImage(img, {
       width: newWidth,
       height: newHeight,

From 3281c9b4c0935a9296e0ea5f359243d732605832 Mon Sep 17 00:00:00 2001
From: Thibaut Girka <thib@sitedethib.com>
Date: Fri, 20 Nov 2020 14:20:17 +0100
Subject: [PATCH 31/31] Fix video not taking focus on interaction

Revert part of #737 to restore hotkey functionality
---
 app/javascript/flavours/glitch/features/video/index.js | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/app/javascript/flavours/glitch/features/video/index.js b/app/javascript/flavours/glitch/features/video/index.js
index 21a2594e7b4..ea40b6073df 100644
--- a/app/javascript/flavours/glitch/features/video/index.js
+++ b/app/javascript/flavours/glitch/features/video/index.js
@@ -182,10 +182,7 @@ class Video extends React.PureComponent {
     this.volume = c;
   }
 
-  handleMouseDownRoot = e => {
-    e.preventDefault();
-    e.stopPropagation();
-  }
+  handleClickRoot = e => e.stopPropagation();
 
   handlePlay = () => {
     this.setState({ paused: false });
@@ -578,7 +575,7 @@ class Video extends React.PureComponent {
         ref={this.setPlayerRef}
         onMouseEnter={this.handleMouseEnter}
         onMouseLeave={this.handleMouseLeave}
-        onMouseDown={this.handleMouseDownRoot}
+        onClick={this.handleClickRoot}
         onKeyDown={this.handleKeyDown}
         tabIndex={0}
       >