Merge branch 'master' into glitch-soc/merge-upstream

app/controllers/admin/dashboard_controller.rb

@@ -27,7 +27,7 @@ module Admin
       @saml_enabled          = ENV['SAML_ENABLED'] == 'true'
       @pam_enabled           = ENV['PAM_ENABLED'] == 'true'
       @hidden_service        = ENV['ALLOW_ACCESS_TO_HIDDEN_SERVICE'] == 'true'
-      @trending_hashtags     = TrendingTags.get(7)
+      @trending_hashtags     = TrendingTags.get(10, filtered: false)
       @profile_directory     = Setting.profile_directory
       @timeline_preview      = Setting.timeline_preview
       @keybase_integration   = Setting.enable_keybase

app/controllers/admin/tags_controller.rb

@@ -4,41 +4,49 @@ module Admin
   class TagsController < BaseController
     before_action :set_tags, only: :index
     before_action :set_tag, except: :index
-    before_action :set_filter_params
 
     def index
       authorize :tag, :index?
     end
 
-    def hide
+    def show
-      authorize @tag, :hide?
+      authorize @tag, :show?
-      @tag.account_tag_stat.update!(hidden: true)
-      redirect_to admin_tags_path(@filter_params)
     end
 
-    def unhide
+    def update
-      authorize @tag, :unhide?
+      authorize @tag, :update?
-      @tag.account_tag_stat.update!(hidden: false)
+
-      redirect_to admin_tags_path(@filter_params)
+      if @tag.update(tag_params.merge(reviewed_at: Time.now.utc))
+        redirect_to admin_tag_path(@tag.id)
+      else
+        render :show
+      end
     end
 
     private
 
     def set_tags
-      @tags = Tag.discoverable
+      @tags = filtered_tags.page(params[:page])
-      @tags.merge!(Tag.hidden) if filter_params[:hidden]
     end
 
     def set_tag
       @tag = Tag.find(params[:id])
     end
 
-    def set_filter_params
+    def filtered_tags
-      @filter_params = filter_params.to_hash.symbolize_keys
+      scope = Tag
+      scope = scope.discoverable if filter_params[:context] == 'directory'
+      scope = scope.reviewed if filter_params[:review] == 'reviewed'
+      scope = scope.pending_review if filter_params[:review] == 'pending_review'
+      scope.reorder(score: :desc)
     end
 
     def filter_params
-      params.permit(:hidden)
+      params.slice(:context, :review).permit(:context, :review)
+    end
+
+    def tag_params
+      params.require(:tag).permit(:name, :trendable, :usable, :listable)
     end
   end
 end

app/controllers/api/v1/trends_controller.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class Api::V1::TrendsController < Api::BaseController
+  before_action :set_tags
+
+  respond_to :json
+
+  def index
+    render json: @tags, each_serializer: REST::TagSerializer
+  end
+
+  private
+
+  def set_tags
+    @tags = TrendingTags.get(limit_param(10))
+  end
+end

app/controllers/settings/preferences_controller.rb

@@ -58,7 +58,7 @@ class Settings::PreferencesController < Settings::BaseController
       :setting_default_content_type,
       :setting_use_blurhash,
       :setting_use_pending_items,
-      notification_emails: %i(follow follow_request reblog favourite mention digest report pending_account),
+      notification_emails: %i(follow follow_request reblog favourite mention digest report pending_account trending_tag),
       interactions: %i(must_be_follower must_be_following must_be_following_dm)
     )
   end

app/helpers/admin/filter_helper.rb

@@ -5,7 +5,7 @@ module Admin::FilterHelper
   REPORT_FILTERS       = %i(resolved account_id target_account_id).freeze
   INVITE_FILTER        = %i(available expired).freeze
   CUSTOM_EMOJI_FILTERS = %i(local remote by_domain shortcode).freeze
-  TAGS_FILTERS         = %i(hidden).freeze
+  TAGS_FILTERS         = %i(context review).freeze
   INSTANCES_FILTERS    = %i(limited by_domain).freeze
   FOLLOWERS_FILTERS    = %i(relationship status by_domain activity order).freeze
 
@@ -14,6 +14,7 @@ module Admin::FilterHelper
   def filter_link_to(text, link_to_params, link_class_params = link_to_params)
     new_url   = filtered_url_for(link_to_params)
     new_class = filtered_url_for(link_class_params)
+
     link_to text, new_url, class: filter_link_class(new_class)
   end
 

app/javascript/mastodon/actions/modal.js

@@ -9,8 +9,9 @@ export function openModal(type, props) {
   };
 };
 
-export function closeModal() {
+export function closeModal(type) {
   return {
     type: MODAL_CLOSE,
+    modalType: type,
   };
 };

app/javascript/mastodon/components/dropdown_menu.js

@@ -45,7 +45,10 @@ class DropdownMenu extends React.PureComponent {
     document.addEventListener('click', this.handleDocumentClick, false);
     document.addEventListener('keydown', this.handleKeyDown, false);
     document.addEventListener('touchend', this.handleDocumentClick, listenerOptions);
-    if (this.focusedItem && this.props.openedViaKeyboard) this.focusedItem.focus();
+    this.activeElement = document.activeElement;
+    if (this.focusedItem && this.props.openedViaKeyboard) {
+      this.focusedItem.focus();
+    }
     this.setState({ mounted: true });
   }
 
@@ -53,6 +56,9 @@ class DropdownMenu extends React.PureComponent {
     document.removeEventListener('click', this.handleDocumentClick, false);
     document.removeEventListener('keydown', this.handleKeyDown, false);
     document.removeEventListener('touchend', this.handleDocumentClick, listenerOptions);
+    if (this.activeElement) {
+      this.activeElement.focus();
+    }
   }
 
   setRef = c => {
@@ -81,6 +87,18 @@ class DropdownMenu extends React.PureComponent {
         element.focus();
       }
       break;
+    case 'Tab':
+      if (e.shiftKey) {
+        element = items[index-1] || items[items.length-1];
+      } else {
+        element = items[index+1] || items[0];
+      }
+      if (element) {
+        element.focus();
+        e.preventDefault();
+        e.stopPropagation();
+      }
+      break;
     case 'Home':
       element = items[0];
       if (element) {
@@ -93,11 +111,14 @@ class DropdownMenu extends React.PureComponent {
         element.focus();
       }
       break;
+    case 'Escape':
+      this.props.onClose();
+      break;
     }
   }
 
-  handleItemKeyDown = e => {
+  handleItemKeyUp = e => {
-    if (e.key === 'Enter') {
+    if (e.key === 'Enter' || e.key === ' ') {
       this.handleClick(e);
     }
   }
@@ -126,7 +147,7 @@ class DropdownMenu extends React.PureComponent {
 
     return (
       <li className='dropdown-menu__item' key={`${text}-${i}`}>
-        <a href={href} target={target} data-method={method} rel='noopener' role='button' tabIndex='0' ref={i === 0 ? this.setFocusRef : null} onClick={this.handleClick} onKeyDown={this.handleItemKeyDown} data-index={i}>
+        <a href={href} target={target} data-method={method} rel='noopener' role='button' tabIndex='0' ref={i === 0 ? this.setFocusRef : null} onClick={this.handleClick} onKeyUp={this.handleItemKeyUp} data-index={i}>
           {text}
         </a>
       </li>
@@ -202,19 +223,6 @@ export default class Dropdown extends React.PureComponent {
     this.props.onClose(this.state.id);
   }
 
-  handleKeyDown = e => {
-    switch(e.key) {
-    case ' ':
-    case 'Enter':
-      this.handleClick(e);
-      e.preventDefault();
-      break;
-    case 'Escape':
-      this.handleClose();
-      break;
-    }
-  }
-
   handleItemClick = e => {
     const i = Number(e.currentTarget.getAttribute('data-index'));
     const { action, to } = this.props.items[i];
@@ -249,7 +257,7 @@ export default class Dropdown extends React.PureComponent {
     const open = this.state.id === openDropdownId;
 
     return (
-      <div onKeyDown={this.handleKeyDown}>
+      <div>
         <IconButton
           icon={icon}
           title={title}

app/javascript/mastodon/components/icon_button.js

@@ -12,6 +12,8 @@ export default class IconButton extends React.PureComponent {
     title: PropTypes.string.isRequired,
     icon: PropTypes.string.isRequired,
     onClick: PropTypes.func,
+    onMouseDown: PropTypes.func,
+    onKeyDown: PropTypes.func,
     size: PropTypes.number,
     active: PropTypes.bool,
     pressed: PropTypes.bool,
@@ -42,6 +44,18 @@ export default class IconButton extends React.PureComponent {
     }
   }
 
+  handleMouseDown = (e) => {
+    if (!this.props.disabled && this.props.onMouseDown) {
+      this.props.onMouseDown(e);
+    }
+  }
+
+  handleKeyDown = (e) => {
+    if (!this.props.disabled && this.props.onKeyDown) {
+      this.props.onKeyDown(e);
+    }
+  }
+
   render () {
     const style = {
       fontSize: `${this.props.size}px`,
@@ -84,6 +98,8 @@ export default class IconButton extends React.PureComponent {
           title={title}
           className={classes}
           onClick={this.handleClick}
+          onMouseDown={this.handleMouseDown}
+          onKeyDown={this.handleKeyDown}
           style={style}
           tabIndex={tabIndex}
           disabled={disabled}
@@ -103,6 +119,8 @@ export default class IconButton extends React.PureComponent {
             title={title}
             className={classes}
             onClick={this.handleClick}
+            onMouseDown={this.handleMouseDown}
+            onKeyDown={this.handleKeyDown}
             style={style}
             tabIndex={tabIndex}
             disabled={disabled}

app/javascript/mastodon/components/modal_root.js

@@ -21,8 +21,30 @@ export default class ModalRoot extends React.PureComponent {
     }
   }
 
+  handleKeyDown = (e) => {
+    if (e.key === 'Tab') {
+      const focusable = Array.from(this.node.querySelectorAll('button:not([disabled]), [href], input:not([disabled]), select:not([disabled]), textarea:not([disabled]), [tabindex]:not([tabindex="-1"])')).filter((x) => window.getComputedStyle(x).display !== 'none');
+      const index = focusable.indexOf(e.target);
+
+      let element;
+
+      if (e.shiftKey) {
+        element = focusable[index - 1] || focusable[focusable.length - 1];
+      } else {
+        element = focusable[index + 1] || focusable[0];
+      }
+
+      if (element) {
+        element.focus();
+        e.stopPropagation();
+        e.preventDefault();
+      }
+    }
+  }
+
   componentDidMount () {
     window.addEventListener('keyup', this.handleKeyUp, false);
+    window.addEventListener('keydown', this.handleKeyDown, false);
   }
 
   componentWillReceiveProps (nextProps) {
@@ -52,6 +74,7 @@ export default class ModalRoot extends React.PureComponent {
 
   componentWillUnmount () {
     window.removeEventListener('keyup', this.handleKeyUp);
+    window.removeEventListener('keydown', this.handleKeyDown);
   }
 
   getSiblings = () => {

app/javascript/mastodon/components/status_content.js

@@ -8,71 +8,9 @@ import classnames from 'classnames';
 import PollContainer from 'mastodon/containers/poll_container';
 import Icon from 'mastodon/components/icon';
 import { autoPlayGif } from 'mastodon/initial_state';
-import { decode as decodeIDNA } from 'mastodon/utils/idna';
 
 const MAX_HEIGHT = 642; // 20px * 32 (+ 2px padding at the top)
 
-// Regex matching what "looks like a link", that is, something that starts with
-// an optional "http://" or "https://" scheme and then what could look like a
-// domain main, that is, at least two sequences of characters not including spaces
-// and separated by "." or an homoglyph. The idea is not to match valid URLs or
-// domain names, but what could be confused for a valid URL or domain name,
-// especially to the untrained eye.
-
-const h_confusables = 'h\u13c2\u1d58d\u1d4f1\u1d691\u0068\uff48\u1d525\u210e\u1d489\u1d629\u0570\u1d4bd\u1d65d\u1d421\u1d5c1\u1d5f5\u04bb\u1d559';
-const t_confusables = 't\u1d42d\u1d5cd\u1d531\u1d565\u1d4c9\u1d669\u1d4fd\u1d69d\u0074\u1d461\u1d601\u1d495\u1d635\u1d599';
-const p_confusables = 'p\u0440\u03c1\u1d52d\u1d631\u1d665\u1d429\uff50\u1d6e0\u1d45d\u1d561\u1d595\u1d71a\u1d699\u1d78e\u2ca3\u1d754\u1d6d2\u1d491\u1d7c8\u1d746\u1d4c5\u1d70c\u1d5c9\u0070\u1d780\u03f1\u1d5fd\u2374\u1d7ba\u1d4f9';
-const s_confusables = 's\u1d530\u118c1\u1d494\u1d634\u1d4c8\u1d668\uabaa\u1d42c\u1d5cc\u1d460\u1d600\ua731\u0073\uff53\u1d564\u0455\u1d598\u1d4fc\u1d69c\u10448\u01bd';
-const column_confusables = ':\u0903\u0a83\u0703\u1803\u05c3\u0704\u0589\u1809\ua789\u16ec\ufe30\u02d0\u2236\u02f8\u003a\uff1a\u205a\ua4fd';
-const slash_confusables = '/\u2041\u2f03\u2044\u2cc6\u27cb\u30ce\u002f\u2571\u31d3\u3033\u1735\u2215\u29f8\u1d23a\u4e3f';
-const dot_confusables = '.\u002e\u0660\u06f0\u0701\u0702\u2024\ua4f8\ua60e\u10a50\u1d16d';
-
-const linkRegex = new RegExp(`^\\s*(([${h_confusables}][${t_confusables}][${t_confusables}][${p_confusables}][${s_confusables}]?[${column_confusables}][${slash_confusables}][${slash_confusables}]))?[^:/\\n ]+([${dot_confusables}][^:/\\n ]+)+`);
-
-const isLinkMisleading = (link) => {
-  let linkTextParts = [];
-
-  // Reconstruct visible text, as we do not have much control over how links
-  // from remote software look, and we can't rely on `innerText` because the
-  // `invisible` class does not set `display` to `none`.
-
-  const walk = (node) => {
-    switch (node.nodeType) {
-    case Node.TEXT_NODE:
-      linkTextParts.push(node.textContent);
-      break;
-    case Node.ELEMENT_NODE:
-      if (node.classList.contains('invisible')) return;
-      const children = node.childNodes;
-      for (let i = 0; i < children.length; i++) {
-        walk(children[i]);
-      }
-      break;
-    }
-  };
-
-  walk(link);
-
-  const linkText = linkTextParts.join('');
-  const targetURL = new URL(link.href);
-
-  // The following may not work with international domain names
-  if (linkText === targetURL.origin || linkText === targetURL.host || 'www.' + linkText === targetURL.host || linkText.startsWith(targetURL.origin + '/') || linkText.startsWith(targetURL.host + '/')) {
-    return false;
-  }
-
-  // The link hasn't been recognized, maybe it features an international domain name
-  const hostname = decodeIDNA(targetURL.hostname);
-  const host = targetURL.host.replace(targetURL.hostname, hostname);
-  const origin = targetURL.origin.replace(targetURL.host, host);
-  if (linkText === origin || linkText === host || linkText.startsWith(origin + '/') || linkText.startsWith(host + '/')) {
-    return false;
-  }
-
-  // If the link text looks like an URL or auto-generated link, it is misleading
-  return linkRegex.test(linkText);
-};
-
 export default class StatusContent extends React.PureComponent {
 
   static contextTypes = {
@@ -118,34 +56,6 @@ export default class StatusContent extends React.PureComponent {
       } else {
         link.setAttribute('title', link.href);
         link.classList.add('unhandled-link');
-
-        if (isLinkMisleading(link)) {
-          while (link.firstChild) {
-            link.removeChild(link.firstChild);
-          }
-
-          const prefix = (link.href.match(/https?:\/\/(www\.)?/) || [''])[0];
-          const text   = link.href.substr(prefix.length, 30);
-          const suffix = link.href.substr(prefix.length + 30);
-          const cutoff = !!suffix;
-
-          const prefixTag = document.createElement('span');
-          prefixTag.classList.add('invisible');
-          prefixTag.textContent = prefix;
-          link.appendChild(prefixTag);
-
-          const textTag = document.createElement('span');
-          if (cutoff) {
-            textTag.classList.add('ellipsis');
-          }
-          textTag.textContent = text;
-          link.appendChild(textTag);
-
-          const suffixTag = document.createElement('span');
-          suffixTag.classList.add('invisible');
-          suffixTag.textContent = suffix;
-          link.appendChild(suffixTag);
-        }
       }
 
       link.setAttribute('target', '_blank');

app/javascript/mastodon/containers/dropdown_menu_container.js

@@ -20,7 +20,7 @@ const mapDispatchToProps = (dispatch, { status, items }) => ({
     }) : openDropdownMenu(id, dropdownPlacement, keyboard));
   },
   onClose(id) {
-    dispatch(closeModal());
+    dispatch(closeModal('ACTIONS'));
     dispatch(closeDropdownMenu(id));
   },
 });

app/javascript/mastodon/features/compose/components/privacy_dropdown.js

@@ -73,6 +73,19 @@ class PrivacyDropdownMenu extends React.PureComponent {
         this.props.onChange(element.getAttribute('data-index'));
       }
       break;
+    case 'Tab':
+      if (e.shiftKey) {
+        element = this.node.childNodes[index - 1] || this.node.lastChild;
+      } else {
+        element = this.node.childNodes[index + 1] || this.node.firstChild;
+      }
+      if (element) {
+        element.focus();
+        this.props.onChange(element.getAttribute('data-index'));
+        e.preventDefault();
+        e.stopPropagation();
+      }
+      break;
     case 'Home':
       element = this.node.firstChild;
       if (element) {
@@ -180,6 +193,9 @@ class PrivacyDropdown extends React.PureComponent {
       }
     } else {
       const { top } = target.getBoundingClientRect();
+      if (this.state.open && this.activeElement) {
+        this.activeElement.focus();
+      }
       this.setState({ placement: top * 2 < innerHeight ? 'bottom' : 'top' });
       this.setState({ open: !this.state.open });
     }
@@ -202,7 +218,25 @@ class PrivacyDropdown extends React.PureComponent {
     }
   }
 
+  handleMouseDown = () => {
+    if (!this.state.open) {
+      this.activeElement = document.activeElement;
+    }
+  }
+
+  handleButtonKeyDown = (e) => {
+    switch(e.key) {
+    case ' ':
+    case 'Enter':
+      this.handleMouseDown();
+      break;
+    }
+  }
+
   handleClose = () => {
+    if (this.state.open && this.activeElement) {
+      this.activeElement.focus();
+    }
     this.setState({ open: false });
   }
 
@@ -229,7 +263,7 @@ class PrivacyDropdown extends React.PureComponent {
 
     return (
       <div className={classNames('privacy-dropdown', placement, { active: open })} onKeyDown={this.handleKeyDown}>
-        <div className={classNames('privacy-dropdown__value', { active: this.options.indexOf(valueOption) === 0 })}>
+        <div className={classNames('privacy-dropdown__value', { active: this.options.indexOf(valueOption) === (placement === 'bottom' ? 0 : (this.options.length - 1)) })}>
           <IconButton
             className='privacy-dropdown__value-icon'
             icon={valueOption.icon}
@@ -239,6 +273,8 @@ class PrivacyDropdown extends React.PureComponent {
             active={open}
             inverted
             onClick={this.handleToggle}
+            onMouseDown={this.handleMouseDown}
+            onKeyDown={this.handleButtonKeyDown}
             style={{ height: null, lineHeight: '27px' }}
           />
         </div>

app/javascript/mastodon/features/status/components/card.js

@@ -2,9 +2,18 @@ import React from 'react';
 import PropTypes from 'prop-types';
 import Immutable from 'immutable';
 import ImmutablePropTypes from 'react-immutable-proptypes';
+import punycode from 'punycode';
 import classnames from 'classnames';
 import Icon from 'mastodon/components/icon';
-import { decode as decodeIDNA } from 'mastodon/utils/idna';
+
+const IDNA_PREFIX = 'xn--';
+
+const decodeIDNA = domain => {
+  return domain
+    .split('.')
+    .map(part => part.indexOf(IDNA_PREFIX) === 0 ? punycode.decode(part.slice(IDNA_PREFIX.length)) : part)
+    .join('.');
+};
 
 const getHostname = url => {
   const parser = document.createElement('a');

app/javascript/mastodon/reducers/modal.js

@@ -10,7 +10,7 @@ export default function modal(state = initialState, action) {
   case MODAL_OPEN:
     return { modalType: action.modalType, modalProps: action.modalProps };
   case MODAL_CLOSE:
-    return initialState;
+    return (action.modalType === undefined || action.modalType === state.modalType) ? initialState : state;
   default:
     return state;
   }

app/javascript/mastodon/utils/idna.js

@@ -1,10 +0,0 @@
-import punycode from 'punycode';
-
-const IDNA_PREFIX = 'xn--';
-
-export const decode = domain => {
-  return domain
-    .split('.')
-    .map(part => part.indexOf(IDNA_PREFIX) === 0 ? punycode.decode(part.slice(IDNA_PREFIX.length)) : part)
-    .join('.');
-};

app/javascript/mastodon/utils/resize_image.js

@@ -67,6 +67,14 @@ const processImage = (img, { width, height, orientation, type = 'image/png' }) =
 
   context.drawImage(img, 0, 0, width, height);
 
+  // The Tor Browser and maybe other browsers may prevent reading from canvas
+  // and return an all-white image instead. Assume reading failed if the resized
+  // image is perfectly white.
+  const imageData = context.getImageData(0, 0, width, height);
+  if (imageData.every(value => value === 255)) {
+    throw 'Failed to read from canvas';
+  }
+
   canvas.toBlob(resolve, type);
 });
 

app/mailers/admin_mailer.rb

@@ -24,4 +24,14 @@ class AdminMailer < ApplicationMailer
       mail to: @me.user_email, subject: I18n.t('admin_mailer.new_pending_account.subject', instance: @instance, username: @account.username)
     end
   end
+
+  def new_trending_tag(recipient, tag)
+    @tag      = tag
+    @me       = recipient
+    @instance = Rails.configuration.x.local_domain
+
+    locale_for_account(@me) do
+      mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_tag.subject', instance: @instance, name: @tag.name)
+    end
+  end
 end

app/models/application_record.rb

@@ -2,5 +2,16 @@
 
 class ApplicationRecord < ActiveRecord::Base
   self.abstract_class = true
+
   include Remotable
+
+  def boolean_with_default(key, default_value)
+    value = attributes[key]
+
+    if value.nil?
+      default_value
+    else
+      value
+    end
+  end
 end

app/models/tag.rb

@@ -8,6 +8,11 @@
 #  created_at          :datetime         not null
 #  updated_at          :datetime         not null
 #  score               :integer
+#  usable              :boolean
+#  trendable           :boolean
+#  listable            :boolean
+#  reviewed_at         :datetime
+#  requested_review_at :datetime
 #
 
 class Tag < ApplicationRecord
@@ -22,16 +27,17 @@ class Tag < ApplicationRecord
   HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i
 
   validates :name, presence: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
+  validate :validate_name_change, if: -> { !new_record? && name_changed? }
 
-  scope :discoverable, -> { joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).where(account_tag_stats: { hidden: false }).order(Arel.sql('account_tag_stats.accounts_count desc')) }
+  scope :reviewed, -> { where.not(reviewed_at: nil) }
-  scope :hidden, -> { where(account_tag_stats: { hidden: true }) }
+  scope :pending_review, -> { where(reviewed_at: nil).where.not(requested_review_at: nil) }
+  scope :discoverable, -> { where.not(listable: false).joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).order(Arel.sql('account_tag_stats.accounts_count desc')) }
   scope :most_used, ->(account) { joins(:statuses).where(statuses: { account: account }).group(:id).order(Arel.sql('count(*) desc')) }
 
   delegate :accounts_count,
            :accounts_count=,
            :increment_count!,
            :decrement_count!,
-           :hidden?,
            to: :account_tag_stat
 
   after_save :save_account_tag_stat
@@ -48,6 +54,40 @@ class Tag < ApplicationRecord
     name
   end
 
+  def usable
+    boolean_with_default('usable', true)
+  end
+
+  alias usable? usable
+
+  def listable
+    boolean_with_default('listable', true)
+  end
+
+  alias listable? listable
+
+  def trendable
+    boolean_with_default('trendable', false)
+  end
+
+  alias trendable? trendable
+
+  def requires_review?
+    reviewed_at.nil?
+  end
+
+  def reviewed?
+    reviewed_at.present?
+  end
+
+  def requested_review?
+    requested_review_at.present?
+  end
+
+  def trending?
+    TrendingTags.trending?(self)
+  end
+
   def history
     days = []
 
@@ -117,4 +157,8 @@ class Tag < ApplicationRecord
     return unless account_tag_stat&.changed?
     account_tag_stat.save
   end
+
+  def validate_name_change
+    errors.add(:name, I18n.t('tags.does_not_match_previous_name')) unless name_was.mb_chars.casecmp(name.mb_chars).zero?
+  end
 end

app/models/trending_tags.rb

@@ -10,20 +10,28 @@ class TrendingTags
     include Redisable
 
     def record_use!(tag, account, at_time = Time.now.utc)
-      return if disallowed_hashtags.include?(tag.name) || account.silenced? || account.bot?
+      return if account.silenced? || account.bot? || !tag.usable? || !(tag.trendable? || tag.requires_review?)
 
       increment_historical_use!(tag.id, at_time)
       increment_unique_use!(tag.id, account.id, at_time)
-      increment_vote!(tag.id, at_time)
+      increment_vote!(tag, at_time)
     end
 
-    def get(limit)
+    def get(limit, filtered: true)
-      key     = "#{KEY}:#{Time.now.utc.beginning_of_day.to_i}"
+      tag_ids = redis.zrevrange("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", 0, limit - 1).map(&:to_i)
-      tag_ids = redis.zrevrange(key, 0, limit - 1).map(&:to_i)
+
-      tags    = Tag.where(id: tag_ids).to_a.each_with_object({}) { |tag, h| h[tag.id] = tag }
+      tags = Tag.where(id: tag_ids)
+      tags = tags.where(trendable: true) if filtered
+      tags = tags.each_with_object({}) { |tag, h| h[tag.id] = tag }
+
       tag_ids.map { |tag_id| tags[tag_id] }.compact
     end
 
+    def trending?(tag)
+      rank = redis.zrevrank("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", tag.id)
+      rank.present? && rank <= 10
+    end
+
     private
 
     def increment_historical_use!(tag_id, at_time)
@@ -38,33 +46,27 @@ class TrendingTags
       redis.expire(key, EXPIRE_HISTORY_AFTER)
     end
 
-    def increment_vote!(tag_id, at_time)
+    def increment_vote!(tag, at_time)
       key      = "#{KEY}:#{at_time.beginning_of_day.to_i}"
-      expected = redis.pfcount("activity:tags:#{tag_id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
+      expected = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
       expected = 1.0 if expected.zero?
-      observed = redis.pfcount("activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
+      observed = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
 
       if expected > observed || observed < THRESHOLD
-        redis.zrem(key, tag_id.to_s)
+        redis.zrem(key, tag.id)
       else
         score    = ((observed - expected)**2) / expected
-        added = redis.zadd(key, score, tag_id.to_s)
+        old_rank = redis.zrevrank(key, tag.id)
-        bump_tag_score!(tag_id) if added
+
+        redis.zadd(key, score, tag.id)
+        request_review!(tag) if (old_rank.nil? || old_rank > 10) && redis.zrevrank(key, tag.id) <= 10 && !tag.trendable? && tag.requires_review? && !tag.requested_review?
       end
 
       redis.expire(key, EXPIRE_TRENDS_AFTER)
     end
 
-    def bump_tag_score!(tag_id)
+    def request_review!(tag)
-      Tag.where(id: tag_id).update_all('score = COALESCE(score, 0) + 1')
+      User.staff.includes(:account).find_each { |u| AdminMailer.new_trending_tag(u.account, tag).deliver_later! if u.allows_trending_tag_emails? }
-    end
-
-    def disallowed_hashtags
-      return @disallowed_hashtags if defined?(@disallowed_hashtags)
-
-      @disallowed_hashtags = Setting.disallowed_hashtags.nil? ? [] : Setting.disallowed_hashtags
-      @disallowed_hashtags = @disallowed_hashtags.split(' ') if @disallowed_hashtags.is_a? String
-      @disallowed_hashtags = @disallowed_hashtags.map(&:downcase)
     end
   end
 end

app/models/user.rb

@@ -207,6 +207,10 @@ class User < ApplicationRecord
     settings.notification_emails['pending_account']
   end
 
+  def allows_trending_tag_emails?
+    settings.notification_emails['trending_tag']
+  end
+
   def hides_network?
     @hides_network ||= settings.hide_network
   end

app/policies/tag_policy.rb

@@ -5,11 +5,11 @@ class TagPolicy < ApplicationPolicy
     staff?
   end
 
-  def hide?
+  def show?
     staff?
   end
 
-  def unhide?
+  def update?
     staff?
   end
 end

app/validators/disallowed_hashtags_validator.rb

@@ -4,24 +4,7 @@ class DisallowedHashtagsValidator < ActiveModel::Validator
   def validate(status)
     return unless status.local? && !status.reblog?
 
-    @status = status
+    disallowed_hashtags = Tag.matching_name(Extractor.extract_hashtags(status.text)).reject(&:usable?)
-    tags    = select_tags
+    status.errors.add(:text, I18n.t('statuses.disallowed_hashtags', tags: disallowed_hashtags.map(&:name).join(', '), count: disallowed_hashtags.size)) unless disallowed_hashtags.empty?
-
-    status.errors.add(:text, I18n.t('statuses.disallowed_hashtags', tags: tags.join(', '), count: tags.size)) unless tags.empty?
-  end
-
-  private
-
-  def select_tags
-    tags = Extractor.extract_hashtags(@status.text)
-    tags.keep_if { |tag| disallowed_hashtags.include? tag.downcase }
-  end
-
-  def disallowed_hashtags
-    return @disallowed_hashtags if @disallowed_hashtags
-
-    @disallowed_hashtags = Setting.disallowed_hashtags.nil? ? [] : Setting.disallowed_hashtags
-    @disallowed_hashtags = @disallowed_hashtags.split(' ') if @disallowed_hashtags.is_a? String
-    @disallowed_hashtags = @disallowed_hashtags.map(&:downcase)
   end
 end

app/views/admin/dashboard/index.html.haml

@@ -109,5 +109,5 @@
       %ul
         - @trending_hashtags.each do |tag|
           %li
-            = link_to "##{tag.name}", web_url("timelines/tag/#{tag.name}")
+            = link_to content_tag(:span, "##{tag.name}", class: !tag.trendable? && !tag.reviewed? ? 'warning-hint' : (!tag.trendable? ? 'negative-hint' : nil)), admin_tag_path(tag.id)
             %span.pull-right= number_with_delimiter(tag.history[0][:accounts].to_i)

app/views/admin/tags/_tag.html.haml

@@ -1,12 +1,16 @@
-%tr
+.directory__tag
-  %td
+  = link_to admin_tag_path(tag.id) do
-    = link_to explore_hashtag_path(tag) do
+    %h4
       = fa_icon 'hashtag'
       = tag.name
-  %td
+
-    = t('directories.people', count: tag.accounts_count)
+      %small
-  %td
+        = t('admin.tags.in_directory', count: tag.accounts_count)
-    - if tag.hidden?
+        •
-      = table_link_to 'eye', t('admin.tags.unhide'), unhide_admin_tag_path(tag.id, **@filter_params), method: :post
+        = t('admin.tags.unique_uses_today', count: tag.history.first[:accounts])
-    - else
+
-      = table_link_to 'eye-slash', t('admin.tags.hide'), hide_admin_tag_path(tag.id, **@filter_params), method: :post
+        - if tag.trending?
+          = fa_icon 'fire fw'
+          = t('admin.tags.trending_right_now')
+
+    .trends__item__current= number_to_human tag.history.first[:uses], strip_insignificant_zeros: true

app/views/admin/tags/index.html.haml

@@ -3,17 +3,19 @@
 
 .filters
   .filter-subset
-    %strong= t('admin.reports.status')
+    %strong= t('admin.tags.context')
     %ul
-      %li= filter_link_to t('admin.tags.visible'), hidden: nil
+      %li= filter_link_to t('generic.all'), context: nil
-      %li= filter_link_to t('admin.tags.hidden'), hidden: '1'
+      %li= filter_link_to t('admin.tags.directory'), context: 'directory'
 
-.table-wrapper
+  .filter-subset
-  %table.table
+    %strong= t('admin.tags.review')
-    %thead
+    %ul
-      %tr
+      %li= filter_link_to t('generic.all'), review: nil
-        %th= t('admin.tags.name')
+      %li= filter_link_to t('admin.tags.reviewed'), review: 'reviewed'
-        %th= t('admin.tags.accounts')
+      %li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), review: 'pending_review'
-        %th
+
-    %tbody
+%hr.spacer/
-      = render @tags
+
+= render @tags
+= paginate @tags

app/views/admin/tags/show.html.haml

@@ -0,0 +1,16 @@
+- content_for :page_title do
+  = "##{@tag.name}"
+
+= simple_form_for @tag, url: admin_tag_path(@tag.id) do |f|
+  = render 'shared/error_messages', object: @tag
+
+  .fields-group
+    = f.input :name, wrapper: :with_block_label
+
+  .fields-group
+    = f.input :usable, as: :boolean, wrapper: :with_label
+    = f.input :trendable, as: :boolean, wrapper: :with_label
+    = f.input :listable, as: :boolean, wrapper: :with_label
+
+  .actions
+    = f.button :button, t('generic.save_changes'), type: :submit

app/views/admin_mailer/new_trending_tag.text.erb

@@ -0,0 +1,5 @@
+<%= raw t('application_mailer.salutation', name: display_name(@me)) %>
+
+<%= raw t('admin_mailer.new_trending_tag.body', name: @tag.name) %>
+
+<%= raw t('application_mailer.view')%> <%= admin_tags_url(review: 'pending_review') %>

app/views/settings/preferences/notifications/show.html.haml

@@ -15,6 +15,7 @@
       - if current_user.staff?
         = ff.input :report, as: :boolean, wrapper: :with_label
         = ff.input :pending_account, as: :boolean, wrapper: :with_label
+        = ff.input :trending_tag, as: :boolean, wrapper: :with_label
 
   .fields-group
     = f.simple_fields_for :notification_emails, hash_to_object(current_user.settings.notification_emails) do |ff|

config/locales/en.yml

@@ -496,13 +496,14 @@ en:
       title: Account statuses
       with_media: With media
     tags:
-      accounts: Accounts
+      context: Context
-      hidden: Hidden
+      directory: In directory
-      hide: Hide from directory
+      in_directory: "%{count} in directory"
-      name: Hashtag
+      review: Review status
+      reviewed: Reviewed
       title: Hashtags
-      unhide: Show in directory
+      trending_right_now: Trending right now
-      visible: Visible
+      unique_uses_today: "%{count} posting today"
     title: Administration
     warning_presets:
       add_new: Add new
@@ -518,6 +519,9 @@ en:
       body: "%{reporter} has reported %{target}"
       body_remote: Someone from %{domain} has reported %{target}
       subject: New report for %{instance} (#%{id})
+    new_trending_tag:
+      body: 'The hashtag #%{name} is trending today, but has not been previously reviewed. It will not be displayed publicly unless you allow it to, or just save the form as it is to never hear about it again.'
+      subject: New hashtag up for review on %{instance} (#%{name})
   appearance:
     advanced_web_interface: Advanced web interface
     advanced_web_interface_hint: 'If you want to make use of your entire screen width, the advanced web interface allows you to configure many different columns to see as much information at the same time as you want: Home, notifications, federated timeline, any number of lists and hashtags.'
@@ -954,6 +958,8 @@ en:
     pinned: Pinned toot
     reblogged: boosted
     sensitive_content: Sensitive content
+  tags:
+    does_not_match_previous_name: does not match the previous name
   terms:
     body_html: |
       <h2>Privacy Policy</h2>

config/locales/simple_form.en.yml

@@ -53,6 +53,8 @@ en:
         text: This will help us review your application
       sessions:
         otp: 'Enter the two-factor code generated by your phone app or use one of your recovery codes:'
+      tag:
+        name: You can only change the casing of the letters, for example, to make it more readable
       user:
         chosen_languages: When checked, only toots in selected languages will be displayed in public timelines
     labels:
@@ -148,6 +150,11 @@ en:
         pending_account: Send e-mail when a new account needs review
         reblog: Send e-mail when someone boosts your status
         report: Send e-mail when a new report is submitted
+        trending_tag: Send e-mail when an unreviewed hashtag is trending
+      tag:
+        listable: Allow this hashtag to appear on the profile directory
+        trendable: Allow this hashtag to appear under trends
+        usable: Allow toots to use this hashtag
     'no': 'No'
     recommended: Recommended
     required:

config/navigation.rb

@@ -44,7 +44,7 @@ SimpleNavigation::Configuration.run do |navigation|
       s.item :reports, safe_join([fa_icon('flag fw'), t('admin.reports.title')]), admin_reports_url, highlights_on: %r{/admin/reports}
       s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_url, highlights_on: %r{/admin/accounts|/admin/pending_accounts}
       s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path
-      s.item :tags, safe_join([fa_icon('tag fw'), t('admin.tags.title')]), admin_tags_path
+      s.item :tags, safe_join([fa_icon('hashtag fw'), t('admin.tags.title')]), admin_tags_path, highlights_on: %r{/admin/tags}
       s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: whitelist_mode? ? nil : '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks|/admin/domain_allows}, if: -> { current_user.admin? }
       s.item :email_domain_blocks, safe_join([fa_icon('envelope fw'), t('admin.email_domain_blocks.title')]), admin_email_domain_blocks_url, highlights_on: %r{/admin/email_domain_blocks}, if: -> { current_user.admin? }
     end

config/routes.rb

@@ -245,13 +245,7 @@ Rails.application.routes.draw do
     end
 
     resources :account_moderation_notes, only: [:create, :destroy]
-
+    resources :tags, only: [:index, :show, :update]
-    resources :tags, only: [:index] do
-      member do
-        post :hide
-        post :unhide
-      end
-    end
   end
 
   get '/admin', to: redirect('/admin/dashboard', status: 302)
@@ -322,6 +316,7 @@ Rails.application.routes.draw do
       resources :favourites,   only: [:index]
       resources :bookmarks,    only: [:index]
       resources :reports,      only: [:create]
+      resources :trends,       only: [:index]
       resources :filters,      only: [:index, :create, :show, :update, :destroy]
       resources :endorsements, only: [:index]
 

config/settings.yml

@@ -47,6 +47,7 @@ defaults: &defaults
     digest: true
     report: true
     pending_account: true
+    trending_tag: true
   interactions:
     must_be_follower: false
     must_be_following: false

db/migrate/20190805123746_add_capabilities_to_tags.rb

@@ -0,0 +1,9 @@
+class AddCapabilitiesToTags < ActiveRecord::Migration[5.2]
+  def change
+    add_column :tags, :usable, :boolean
+    add_column :tags, :trendable, :boolean
+    add_column :tags, :listable, :boolean
+    add_column :tags, :reviewed_at, :datetime
+    add_column :tags, :requested_review_at, :datetime
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_29_185330) do
+ActiveRecord::Schema.define(version: 2019_08_05_123746) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -673,6 +673,11 @@ ActiveRecord::Schema.define(version: 2019_07_29_185330) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.integer "score"
+    t.boolean "usable"
+    t.boolean "trendable"
+    t.boolean "listable"
+    t.datetime "reviewed_at"
+    t.datetime "requested_review_at"
     t.index "lower((name)::text)", name: "index_tags_on_name_lower", unique: true
   end
 

spec/controllers/admin/tags_controller_spec.rb

@@ -10,62 +10,14 @@ RSpec.describe Admin::TagsController, type: :controller do
   end
 
   describe 'GET #index' do
-    before do
+    let!(:tag) { Fabricate(:tag) }
-      account_tag_stat = Fabricate(:tag).account_tag_stat
-      account_tag_stat.update(hidden: hidden, accounts_count: 1)
-      get :index, params: { hidden: hidden }
-    end
 
-    context 'with hidden tags' do
+    before do
-      let(:hidden) { true }
+      get :index
+    end
 
     it 'returns status 200' do
       expect(response).to have_http_status(200)
     end
   end
-
-    context 'without hidden tags' do
-      let(:hidden) { false }
-
-      it 'returns status 200' do
-        expect(response).to have_http_status(200)
-      end
-    end
-  end
-
-  describe 'POST #hide' do
-    let(:tag) { Fabricate(:tag) }
-
-    before do
-      tag.account_tag_stat.update(hidden: false)
-      post :hide, params: { id: tag.id }
-    end
-
-    it 'hides tag' do
-      tag.reload
-      expect(tag).to be_hidden
-    end
-
-    it 'redirects to admin_tags_path' do
-      expect(response).to redirect_to(admin_tags_path(controller.instance_variable_get(:@filter_params)))
-    end
-  end
-
-  describe 'POST #unhide' do
-    let(:tag) { Fabricate(:tag) }
-
-    before do
-      tag.account_tag_stat.update(hidden: true)
-      post :unhide, params: { id: tag.id }
-    end
-
-    it 'unhides tag' do
-      tag.reload
-      expect(tag).not_to be_hidden
-    end
-
-    it 'redirects to admin_tags_path' do
-      expect(response).to redirect_to(admin_tags_path(controller.instance_variable_get(:@filter_params)))
-    end
-  end
 end

spec/policies/tag_policy_spec.rb

@@ -8,7 +8,7 @@ RSpec.describe TagPolicy do
   let(:admin)   { Fabricate(:user, admin: true).account }
   let(:john)    { Fabricate(:user).account }
 
-  permissions :index?, :hide?, :unhide? do
+  permissions :index?, :show?, :update? do
     context 'staff?' do
       it 'permits' do
         expect(subject).to permit(admin, Tag)

spec/validators/disallowed_hashtags_validator_spec.rb

@@ -3,42 +3,44 @@
 require 'rails_helper'
 
 RSpec.describe DisallowedHashtagsValidator, type: :validator do
+  let(:disallowed_tags) { [] }
+
   describe '#validate' do
     before do
-      allow_any_instance_of(described_class).to receive(:select_tags) { tags }
+      disallowed_tags.each { |name| Fabricate(:tag, name: name, usable: false) }
       described_class.new.validate(status)
     end
 
-    let(:status) { double(errors: errors, local?: local, reblog?: reblog, text: '') }
+    let(:status) { double(errors: errors, local?: local, reblog?: reblog, text: disallowed_tags.map { |x| '#' + x }.join(' ')) }
     let(:errors) { double(add: nil) }
 
-    context 'unless status.local? && !status.reblog?' do
+    context 'for a remote reblog' do
       let(:local)  { false }
       let(:reblog) { true }
 
-      it 'not calls errors.add' do
+      it 'does not add errors' do
         expect(errors).not_to have_received(:add).with(:text, any_args)
       end
     end
 
-    context 'status.local? && !status.reblog?' do
+    context 'for a local original status' do
       let(:local)  { true }
       let(:reblog) { false }
 
-      context 'tags.empty?' do
+      context 'when does not contain any disallowed hashtags' do
-        let(:tags) { [] }
+        let(:disallowed_tags) { [] }
 
-        it 'not calls errors.add' do
+        it 'does not add errors' do
           expect(errors).not_to have_received(:add).with(:text, any_args)
         end
       end
 
-      context '!tags.empty?' do
+      context 'when contains disallowed hashtags' do
-        let(:tags) { %w(a b c) }
+        let(:disallowed_tags) { %w(a b c) }
 
-        it 'calls errors.add' do
+        it 'adds an error' do
           expect(errors).to have_received(:add)
-            .with(:text, I18n.t('statuses.disallowed_hashtags', tags: tags.join(', '), count: tags.size))
+            .with(:text, I18n.t('statuses.disallowed_hashtags', tags: disallowed_tags.join(', '), count: disallowed_tags.size))
         end
       end
     end