Merge remote-tracking branch 'origin/master' into gs-master

app/controllers/application_controller.rb

@@ -104,7 +104,7 @@ class ApplicationController < ActionController::Base
     unless uncached_ids.empty?
       uncached = klass.where(id: uncached_ids).with_includes.map { |item| [item.id, item] }.to_h
 
-      uncached.values.each do |item|
+      uncached.each_value do |item|
         Rails.cache.write(item.cache_key, item)
       end
     end

app/controllers/auth/sessions_controller.rb

@@ -62,7 +62,7 @@ class Auth::SessionsController < Devise::SessionsController
 
     if user_params[:otp_attempt].present? && session[:otp_user_id]
       authenticate_with_two_factor_via_otp(user)
-    elsif user && user.valid_password?(user_params[:password])
+    elsif user&.valid_password?(user_params[:password])
       prompt_for_two_factor(user)
     end
   end

app/helpers/admin/filter_helper.rb

@@ -18,7 +18,7 @@ module Admin::FilterHelper
 
   def selected?(more_params)
     new_url = filtered_url_for(more_params)
-    filter_link_class(new_url) == 'selected' ? true : false
+    filter_link_class(new_url) == 'selected'
   end
 
   private

app/javascript/styles/mastodon/accounts.scss

@@ -571,7 +571,19 @@
   font-size: 12px;
   line-height: 12px;
   font-weight: 500;
-  color: $success-green;
+  color: $ui-secondary-color;
-  background-color: rgba($success-green, 0.1);
+  background-color: rgba($ui-secondary-color, 0.1);
-  border: 1px solid rgba($success-green, 0.5);
+  border: 1px solid rgba($ui-secondary-color, 0.5);
+
+  &.moderator {
+    color: $success-green;
+    background-color: rgba($success-green, 0.1);
+    border-color: rgba($success-green, 0.5);
+  }
+
+  &.admin {
+    color: $error-red;
+    background-color: rgba($error-red, 0.1);
+    border-color: rgba($error-red, 0.5);
+  }
 }

app/lib/activitypub/activity/create.rb

@@ -173,7 +173,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   end
 
   def language_from_content
-    return nil unless language_map?
+    return LanguageDetector.instance.detect(text_from_content, @account) unless language_map?
     @object['contentMap'].keys.first
   end
 

app/lib/extractor.rb

@@ -5,7 +5,8 @@ module Extractor
 
   module_function
 
-  def extract_mentions_or_lists_with_indices(text) # :yields: username, list_slug, start, end
+  # :yields: username, list_slug, start, end
+  def extract_mentions_or_lists_with_indices(text)
     return [] unless text =~ Twitter::Regex[:at_signs]
 
     possible_entries = []

app/lib/language_detector.rb

@@ -38,12 +38,31 @@ class LanguageDetector
   end
 
   def simplify_text(text)
-    text.dup.tap do |new_text|
+    new_text = remove_html(text)
-      new_text.gsub!(FetchLinkCardService::URL_PATTERN, '')
+    new_text.gsub!(FetchLinkCardService::URL_PATTERN, '')
-      new_text.gsub!(Account::MENTION_RE, '')
+    new_text.gsub!(Account::MENTION_RE, '')
-      new_text.gsub!(Tag::HASHTAG_RE, '')
+    new_text.gsub!(Tag::HASHTAG_RE, '')
-      new_text.gsub!(/\s+/, ' ')
+    new_text.gsub!(/:#{CustomEmoji::SHORTCODE_RE_FRAGMENT}:/, '')
-    end
+    new_text.gsub!(/\s+/, ' ')
+    new_text
+  end
+
+  def new_scrubber
+    scrubber = Rails::Html::PermitScrubber.new
+    scrubber.tags = %w(br p)
+    scrubber
+  end
+
+  def scrubber
+    @scrubber ||= new_scrubber
+  end
+
+  def remove_html(text)
+    text = Loofah.fragment(text).scrub!(scrubber).to_s
+    text.gsub!('<br>', "\n")
+    text.gsub!('</p><p>', "\n\n")
+    text.gsub!(/(^<p>|<\/p>$)/, '')
+    text
   end
 
   def default_locale(account)

app/models/account.rb

@@ -117,6 +117,8 @@ class Account < ApplicationRecord
            :current_sign_in_at,
            :confirmed?,
            :admin?,
+           :moderator?,
+           :staff?,
            :locale,
            to: :user,
            prefix: true,

app/models/concerns/account_finder_concern.rb

@@ -44,7 +44,7 @@ module AccountFinderConcern
     end
 
     def with_usernames
-      Account.where.not(username: [nil, ''])
+      Account.where.not(username: '')
     end
 
     def matching_username

app/models/web/push_subscription.rb

@@ -24,12 +24,12 @@ class Web::PushSubscription < ApplicationRecord
   end
 
   def pushable?(notification)
-    data && data.key?('alerts') && data['alerts'][notification.type.to_s]
+    data&.key?('alerts') && data['alerts'][notification.type.to_s]
   end
 
   def as_payload
     payload = { id: id, endpoint: endpoint }
-    payload[:alerts] = data['alerts'] if data && data.key?('alerts')
+    payload[:alerts] = data['alerts'] if data&.key?('alerts')
     payload
   end
 

app/services/batched_remove_status_service.rb

@@ -26,7 +26,7 @@ class BatchedRemoveStatusService < BaseService
     statuses.each(&:destroy)
 
     # Batch by source account
-    statuses.group_by(&:account_id).each do |_, account_statuses|
+    statuses.group_by(&:account_id).each_value do |account_statuses|
       account = account_statuses.first.account
 
       unpush_from_home_timelines(account, account_statuses)

app/services/resolve_remote_account_service.rb

@@ -124,11 +124,11 @@ class ResolveRemoteAccountService < BaseService
   end
 
   def auto_suspend?
-    domain_block && domain_block.suspend?
+    domain_block&.suspend?
   end
 
   def auto_silence?
-    domain_block && domain_block.silence?
+    domain_block&.silence?
   end
 
   def domain_block

app/views/accounts/_header.html.haml

@@ -30,8 +30,12 @@
 
     - if account.user_admin?
       .roles
-        .account-role
+        .account-role.admin
           = t 'accounts.roles.admin'
+    - elsif account.user_moderator?
+      .roles
+        .account-role.moderator
+          = t 'accounts.roles.moderator'
     .bio
       .account__header__content.p-note.emojify!=processed_bio[:text]
       - if processed_bio[:metadata].length > 0

app/views/settings/applications/new.html.haml

@@ -3,6 +3,6 @@
 
 = simple_form_for @application, url: settings_applications_path do |f|
   = render 'fields', f: f
-  
+
   .actions
     = f.button :button, t('doorkeeper.applications.buttons.submit'), type: :submit

app/views/settings/applications/show.html.haml

@@ -25,7 +25,7 @@
 
 = simple_form_for @application, url: settings_application_path(@application), method: :put do |f|
   = render 'fields', f: f
-    
+
   .actions
     = f.button :button, t('generic.save_changes'), type: :submit
 

config/locales/en.yml

@@ -48,6 +48,7 @@ en:
     reserved_username: The username is reserved
     roles:
       admin: Admin
+      moderator: Mod
     unfollow: Unfollow
   admin:
     account_moderation_notes:

config/routes.rb

@@ -168,7 +168,13 @@ Rails.application.routes.draw do
     resources :account_moderation_notes, only: [:create, :destroy]
   end
 
-  get '/admin', to: redirect('/admin/settings/edit', status: 302)
+  authenticate :user, lambda { |u| u.admin? } do
+    get '/admin', to: redirect('/admin/settings/edit', status: 302)
+  end
+
+  authenticate :user, lambda { |u| u.moderator? } do
+    get '/admin', to: redirect('/admin/reports', status: 302)
+  end
 
   namespace :api do
     # PubSubHubbub outgoing subscriptions

lib/mastodon/migration_helpers.rb

@@ -84,7 +84,7 @@ module Mastodon
 
     BACKGROUND_MIGRATION_BATCH_SIZE = 1000 # Number of rows to process per job
     BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1000 # Number of jobs to bulk queue at a time
-    
+
     # Gets an estimated number of rows for a table
     def estimate_rows_in_table(table_name)
       exec_query('SELECT reltuples FROM pg_class WHERE relname = ' +
@@ -313,14 +313,14 @@ module Mastodon
       end
 
       table = Arel::Table.new(table_name)
-      
+
       total = estimate_rows_in_table(table_name).to_i
       if total == 0
         count_arel = table.project(Arel.star.count.as('count'))
         count_arel = yield table, count_arel if block_given?
-        
+
         total = exec_query(count_arel.to_sql).to_hash.first['count'].to_i
-        
+
         return if total == 0
       end
 
@@ -339,7 +339,7 @@ module Mastodon
       # In case there are no rows but we didn't catch it in the estimated size:
       return unless first_row
       start_id = first_row['id'].to_i
-      
+
       say "Migrating #{table_name}.#{column} (~#{total.to_i} rows)"
 
       started_time = Time.now
@@ -347,7 +347,7 @@ module Mastodon
       migrated = 0
       loop do
         stop_row = nil
-        
+
         suppress_messages do
           stop_arel = table.project(table[:id])
             .where(table[:id].gteq(start_id))
@@ -373,29 +373,29 @@ module Mastodon
 
           execute(update_arel.to_sql)
         end
-        
+
         migrated += batch_size
         if Time.now - last_time > 1
           status = "Migrated #{migrated} rows"
-          
+
           percentage = 100.0 * migrated / total
           status += " (~#{sprintf('%.2f', percentage)}%, "
-          
+
           remaining_time = (100.0 - percentage) * (Time.now - started_time) / percentage
-          
+
           status += "#{(remaining_time / 60).to_i}:"
           status += sprintf('%02d', remaining_time.to_i % 60)
           status += ' remaining, '
-          
+
           # Tell users not to interrupt if we're almost done.
           if remaining_time > 10
             status += 'safe to interrupt'
           else
             status += 'DO NOT interrupt'
           end
-          
+
           status += ')'
-          
+
           say status, true
           last_time = Time.now
         end
@@ -483,7 +483,7 @@ module Mastodon
 
       check_trigger_permissions!(table)
       trigger_name = rename_trigger_name(table, old, new)
-      
+
       # If we were in the middle of update_column_in_batches, we should remove
       # the old column and start over, as we have no idea where we were.
       if column_for(table, new)
@@ -492,7 +492,7 @@ module Mastodon
         else
           remove_rename_triggers_for_mysql(trigger_name)
         end
-        
+
         remove_column(table, new)
       end
 
@@ -546,12 +546,12 @@ module Mastodon
       temp_column = rename_column_name(column)
 
       rename_column_concurrently(table, column, temp_column, type: new_type)
-      
+
       # Primary keys don't necessarily have an associated index.
       if ActiveRecord::Base.get_primary_key(table) == column.to_s
         old_pk_index_name = "index_#{table}_on_#{column}"
         new_pk_index_name = "index_#{table}_on_#{column}_cm"
-        
+
         unless indexes_for(table, column).find{|i| i.name == old_pk_index_name}
           add_concurrent_index(table, [temp_column], {
             unique: true,
@@ -572,14 +572,14 @@ module Mastodon
       # Wait for the indices to be built
       indexes_for(table, column).each do |index|
         expected_name = index.name + '_cm'
-        
+
         puts "Waiting for index #{expected_name}"
         sleep 1 until indexes_for(table, temp_column).find {|i| i.name == expected_name }
       end
-      
+
       was_primary = (ActiveRecord::Base.get_primary_key(table) == column.to_s)
       old_default_fn = column_for(table, column).default_function
-      
+
       old_fks = []
       if was_primary
         # Get any foreign keys pointing at this column we need to recreate, and
@@ -613,7 +613,7 @@ module Mastodon
             target_col: temp_column,
             on_delete: extract_foreign_key_action(old_fk['on_delete'])
           )
-          
+
           remove_foreign_key(old_fk['src_table'], name: old_fk['name'])
         end
       end
@@ -629,15 +629,15 @@ module Mastodon
       transaction do
         # This has to be performed in a transaction as otherwise we might have
         # inconsistent data.
-        
+
         cleanup_concurrent_column_rename(table, column, temp_column)
         rename_column(table, temp_column, column)
-        
+
         # If there was an old default function, we didn't copy it. Do that now
         # in the transaction, so we don't miss anything.
         change_column_default(table, column, -> { old_default_fn }) if old_default_fn
       end
-      
+
       # Rename any indices back to what they should be.
       indexes_for(table, column).each do |index|
         next unless index.name.end_with?('_cm')
@@ -645,7 +645,7 @@ module Mastodon
         real_index_name = index.name.sub(/_cm$/, '')
         rename_index(table, index.name, real_index_name)
       end
-      
+
       # Rename any foreign keys back to names based on the real column.
       foreign_keys_for(table, column).each do |fk|
         old_fk_name = concurrent_foreign_key_name(fk.from_table, temp_column, 'id')
@@ -653,7 +653,7 @@ module Mastodon
         execute("ALTER TABLE #{fk.from_table} RENAME CONSTRAINT " +
           "#{old_fk_name} TO #{new_fk_name}")
       end
-      
+
       # Rename any foreign keys from other tables to names based on the real
       # column.
       old_fks.each do |old_fk|
@@ -664,7 +664,7 @@ module Mastodon
         execute("ALTER TABLE #{old_fk['src_table']} RENAME CONSTRAINT " +
           "#{old_fk_name} TO #{new_fk_name}")
       end
-      
+
       # If the old column was a primary key, mark the new one as a primary key.
       if was_primary
         execute("ALTER TABLE #{table} ADD PRIMARY KEY USING INDEX " +
@@ -791,7 +791,7 @@ module Mastodon
         # This is necessary as we can't properly rename indexes such as
         # "ci_taggings_idx".
         name = index.name + '_cm'
-        
+
         # If the order contained the old column, map it to the new one.
         order = index.orders
         if order.key?(old)

spec/controllers/settings/applications_controller_spec.rb

@@ -2,10 +2,10 @@ require 'rails_helper'
 
 describe Settings::ApplicationsController do
   render_views
-  
+
   let!(:user) { Fabricate(:user) }
   let!(:app) { Fabricate(:application, owner: user) }
-  
+
   before do
     sign_in user, scope: :user
   end
@@ -21,7 +21,7 @@ describe Settings::ApplicationsController do
     end
   end
 
-  
+
   describe 'GET #show' do
     it 'returns http success' do
       get :show, params: { id: app.id }
@@ -110,7 +110,7 @@ describe Settings::ApplicationsController do
       end
     end
   end
-  
+
   describe 'PATCH #update' do
     context 'success' do
       let(:opts) {
@@ -131,7 +131,7 @@ describe Settings::ApplicationsController do
         call_update
         expect(app.reload.website).to eql(opts[:website])
       end
-      
+
       it 'redirects back to applications page' do
         expect(call_update).to redirect_to(settings_applications_path)
       end

spec/fixtures/requests/idn.txt

@@ -6,7 +6,7 @@ Content-Length: 38111
 Last-Modified: Wed, 20 Jul 2016 02:50:52 GMT
 Connection: keep-alive
 Accept-Ranges: bytes
-
+
 <!DOCTYPE html>
 <html>
 	<head>
@@ -21,16 +21,16 @@ Accept-Ranges: bytes
             var s = document.getElementsByTagName("script")[0]; 
             s.parentNode.insertBefore(hm, s);
           })();
-	
+
     	</script>
-
+
-
+
 		<link rel="stylesheet" type="text/css" href="css/common.css"/>
 		<script src="js/jquery-1.11.1.min.js" type="text/javascript" charset="utf-8"></script>
 		<script src="js/common.js" type="text/javascript" charset="utf-8"></script>
 		<script src="js/carousel.js" type="text/javascript" charset="utf-8"></script>
 		<title>中国域名网站</title>
-
+
 	</head>
 	<body>
 		<div class="head-tips" id="headTip">
@@ -453,7 +453,7 @@ Accept-Ranges: bytes
 					<li><a href="http://新疆农业大学.中国" target="_blank">新疆农业大学.中国</a></li>
 					<li><a href="http://浙江万里学院.中国" target="_blank">浙江万里学院.中国</a></li>
 					<li><a href="http://重庆大学.中国" target="_blank">重庆大学.中国</a></li>
-					
+
 				</ul>
 			</div>
 		</div>
@@ -472,7 +472,7 @@ Accept-Ranges: bytes
 	<script>
 	$("#headTip").hide()
 	var hostname = window.location.hostname || "";
-
+
 	var tips =  "您所访问的域名 <font size='' color='#ff0000'>" + hostname +"</font> 无法到达，您可以尝试重新访问，或使用搜索相关信息"
 	if (hostname != "导航.中国") {
 		$("#headTip").html(tips);

spec/helpers/stream_entries_helper_spec.rb

@@ -77,7 +77,7 @@ RSpec.describe StreamEntriesHelper, type: :helper do
     params[:controller] = StreamEntriesHelper::EMBEDDED_CONTROLLER
     params[:action] = StreamEntriesHelper::EMBEDDED_ACTION
   end
-  
+
   describe '#style_classes' do
     it do
       status = double(reblog?: false)
@@ -202,7 +202,7 @@ RSpec.describe StreamEntriesHelper, type: :helper do
       expect(css_class).to eq 'h-cite'
     end
   end
-  
+
   describe '#rtl?' do
     it 'is false if text is empty' do
       expect(helper).not_to be_rtl ''

spec/lib/user_settings_decorator_spec.rb

@@ -62,7 +62,7 @@ describe UserSettingsDecorator do
       settings.update(values)
       expect(user.settings['auto_play_gif']).to eq false
     end
-    
+
     it 'updates the user settings value for system font in UI' do
       values = { 'setting_system_font_ui' => '0' }
 

spec/models/remote_follow_spec.rb

@@ -35,7 +35,7 @@ RSpec.describe RemoteFollow do
 
     context 'attrs with acct' do
       let(:attrs) { { acct: 'gargron@quitter.no' }}
-      
+
       it do
         is_expected.to be true
       end

spec/models/status_spec.rb

@@ -47,8 +47,27 @@ RSpec.describe Status, type: :model do
   end
 
   describe '#verb' do
-    it 'is always post' do
+    context 'if destroyed?' do
-      expect(subject.verb).to be :post
+      it 'returns :delete' do
+        subject.destroy!
+        expect(subject.verb).to be :delete
+      end
+    end
+
+    context 'unless destroyed?' do
+      context 'if reblog?' do
+        it 'returns :share' do
+          subject.reblog = other
+          expect(subject.verb).to be :share
+        end
+      end
+
+      context 'unless reblog?' do
+        it 'returns :post' do
+          subject.reblog = nil
+          expect(subject.verb).to be :post
+        end
+      end
     end
   end
 

spec/services/activitypub/fetch_remote_status_service_spec.rb

@@ -27,7 +27,7 @@ RSpec.describe ActivityPub::FetchRemoteStatusService do
 
       it 'creates status' do
         status = sender.statuses.first
-        
+
         expect(status).to_not be_nil
         expect(status.text).to eq 'Lorem ipsum'
       end

spec/support/matchers/model/model_have_error_on_field.rb

@@ -9,7 +9,7 @@ RSpec::Matchers.define :model_have_error_on_field do |expected|
 
   failure_message do |record|
     keys = record.errors.keys
-    
+
     "expect record.errors(#{keys}) to include #{expected}"
   end
 end