diff --git a/.circleci/config.yml b/.circleci/config.yml index b9228f996cf..2a60ae68413 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -133,6 +133,12 @@ jobs: - run: command: ./bin/rails tests:migrations:populate_v2_4 name: Populate database with test data + - run: + command: ./bin/rails db:migrate VERSION=20180707154237 + name: Run migrations up to v2.4.3 + - run: + command: ./bin/rails tests:migrations:populate_v2_4_3 + name: Populate database with test data - run: command: ./bin/rails db:migrate name: Run all remaining migrations @@ -167,14 +173,22 @@ jobs: - run: command: ./bin/rails tests:migrations:populate_v2_4 name: Populate database with test data + - run: + command: ./bin/rails db:migrate VERSION=20180707154237 + name: Run migrations up to v2.4.3 + environment: + SKIP_POST_DEPLOYMENT_MIGRATIONS: true + - run: + command: ./bin/rails tests:migrations:populate_v2_4_3 + name: Populate database with test data - run: command: ./bin/rails db:migrate - name: Run all pre-deployment migrations + name: Run all remaining pre-deployment migrations environment: SKIP_POST_DEPLOYMENT_MIGRATIONS: true - run: command: ./bin/rails db:migrate - name: Run all post-deployment remaining migrations + name: Run all post-deployment migrations - run: command: ./bin/rails tests:migrations:check_database name: Check migration result diff --git a/Gemfile b/Gemfile index d732f6eed65..9a7635b0606 100644 --- a/Gemfile +++ b/Gemfile @@ -153,3 +153,5 @@ gem 'concurrent-ruby', require: false gem 'connection_pool', require: false gem 'xorcist', '~> 1.1' + +gem 'cocoon', '~> 1.2' diff --git a/Gemfile.lock b/Gemfile.lock index 36a0984e196..7e022b19861 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -163,6 +163,7 @@ GEM elasticsearch-dsl chunky_png (1.4.0) climate_control (0.2.0) + cocoon (1.2.15) coderay (1.1.3) color_diff (0.1) concurrent-ruby (1.1.10) @@ -746,6 +747,7 @@ DEPENDENCIES charlock_holmes (~> 0.7.7) chewy (~> 7.2) climate_control (~> 0.2) + cocoon (~> 1.2) color_diff (~> 0.1) concurrent-ruby connection_pool diff --git a/app/controllers/api/v1/filters/keywords_controller.rb b/app/controllers/api/v1/filters/keywords_controller.rb new file mode 100644 index 00000000000..d3718a1371d --- /dev/null +++ b/app/controllers/api/v1/filters/keywords_controller.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +class Api::V1::Filters::KeywordsController < Api::BaseController + before_action -> { doorkeeper_authorize! :read, :'read:filters' }, only: [:index, :show] + before_action -> { doorkeeper_authorize! :write, :'write:filters' }, except: [:index, :show] + before_action :require_user! + + before_action :set_keywords, only: :index + before_action :set_keyword, only: [:show, :update, :destroy] + + def index + render json: @keywords, each_serializer: REST::FilterKeywordSerializer + end + + def create + @keyword = current_account.custom_filters.find(params[:filter_id]).keywords.create!(resource_params) + + render json: @keyword, serializer: REST::FilterKeywordSerializer + end + + def show + render json: @keyword, serializer: REST::FilterKeywordSerializer + end + + def update + @keyword.update!(resource_params) + + render json: @keyword, serializer: REST::FilterKeywordSerializer + end + + def destroy + @keyword.destroy! + render_empty + end + + private + + def set_keywords + filter = current_account.custom_filters.includes(:keywords).find(params[:filter_id]) + @keywords = filter.keywords + end + + def set_keyword + @keyword = CustomFilterKeyword.includes(:custom_filter).where(custom_filter: { account: current_account }).find(params[:id]) + end + + def resource_params + params.permit(:keyword, :whole_word) + end +end diff --git a/app/controllers/api/v1/filters_controller.rb b/app/controllers/api/v1/filters_controller.rb index b0ace3af04c..07cd1414780 100644 --- a/app/controllers/api/v1/filters_controller.rb +++ b/app/controllers/api/v1/filters_controller.rb @@ -8,21 +8,32 @@ class Api::V1::FiltersController < Api::BaseController before_action :set_filter, only: [:show, :update, :destroy] def index - render json: @filters, each_serializer: REST::FilterSerializer + render json: @filters, each_serializer: REST::V1::FilterSerializer end def create - @filter = current_account.custom_filters.create!(resource_params) - render json: @filter, serializer: REST::FilterSerializer + ApplicationRecord.transaction do + filter_category = current_account.custom_filters.create!(resource_params) + @filter = filter_category.keywords.create!(keyword_params) + end + + render json: @filter, serializer: REST::V1::FilterSerializer end def show - render json: @filter, serializer: REST::FilterSerializer + render json: @filter, serializer: REST::V1::FilterSerializer end def update - @filter.update!(resource_params) - render json: @filter, serializer: REST::FilterSerializer + ApplicationRecord.transaction do + @filter.update!(keyword_params) + @filter.custom_filter.assign_attributes(filter_params) + raise Mastodon::ValidationError, I18n.t('filters.errors.deprecated_api_multiple_keywords') if @filter.custom_filter.changed? && @filter.custom_filter.keywords.count > 1 + + @filter.custom_filter.save! + end + + render json: @filter, serializer: REST::V1::FilterSerializer end def destroy @@ -33,14 +44,22 @@ class Api::V1::FiltersController < Api::BaseController private def set_filters - @filters = current_account.custom_filters + @filters = CustomFilterKeyword.includes(:custom_filter).where(custom_filter: { account: current_account }) end def set_filter - @filter = current_account.custom_filters.find(params[:id]) + @filter = CustomFilterKeyword.includes(:custom_filter).where(custom_filter: { account: current_account }).find(params[:id]) end def resource_params params.permit(:phrase, :expires_in, :irreversible, :whole_word, context: []) end + + def filter_params + resource_params.slice(:expires_in, :irreversible, :context) + end + + def keyword_params + resource_params.slice(:phrase, :whole_word) + end end diff --git a/app/controllers/api/v2/filters_controller.rb b/app/controllers/api/v2/filters_controller.rb new file mode 100644 index 00000000000..8ff3076cfba --- /dev/null +++ b/app/controllers/api/v2/filters_controller.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +class Api::V2::FiltersController < Api::BaseController + before_action -> { doorkeeper_authorize! :read, :'read:filters' }, only: [:index, :show] + before_action -> { doorkeeper_authorize! :write, :'write:filters' }, except: [:index, :show] + before_action :require_user! + before_action :set_filters, only: :index + before_action :set_filter, only: [:show, :update, :destroy] + + def index + render json: @filters, each_serializer: REST::FilterSerializer, rules_requested: true + end + + def create + @filter = current_account.custom_filters.create!(resource_params) + + render json: @filter, serializer: REST::FilterSerializer, rules_requested: true + end + + def show + render json: @filter, serializer: REST::FilterSerializer, rules_requested: true + end + + def update + @filter.update!(resource_params) + + render json: @filter, serializer: REST::FilterSerializer, rules_requested: true + end + + def destroy + @filter.destroy! + render_empty + end + + private + + def set_filters + @filters = current_account.custom_filters.includes(:keywords) + end + + def set_filter + @filter = current_account.custom_filters.find(params[:id]) + end + + def resource_params + params.permit(:title, :expires_in, :filter_action, context: [], keywords_attributes: [:id, :keyword, :whole_word, :_destroy]) + end +end diff --git a/app/controllers/filters_controller.rb b/app/controllers/filters_controller.rb index 79a1ab02b1f..5ed53bce1d2 100644 --- a/app/controllers/filters_controller.rb +++ b/app/controllers/filters_controller.rb @@ -4,16 +4,16 @@ class FiltersController < ApplicationController layout 'admin' before_action :authenticate_user! - before_action :set_filters, only: :index before_action :set_filter, only: [:edit, :update, :destroy] before_action :set_body_classes def index - @filters = current_account.custom_filters.order(:phrase) + @filters = current_account.custom_filters.includes(:keywords).order(:phrase) end def new - @filter = current_account.custom_filters.build + @filter = current_account.custom_filters.build(action: :warn) + @filter.keywords.build end def create @@ -43,16 +43,12 @@ class FiltersController < ApplicationController private - def set_filters - @filters = current_account.custom_filters - end - def set_filter @filter = current_account.custom_filters.find(params[:id]) end def resource_params - params.require(:custom_filter).permit(:phrase, :expires_in, :irreversible, :whole_word, context: []) + params.require(:custom_filter).permit(:title, :expires_in, :filter_action, context: [], keywords_attributes: [:id, :keyword, :whole_word, :_destroy]) end def set_body_classes diff --git a/app/javascript/mastodon/actions/filters.js b/app/javascript/mastodon/actions/filters.js deleted file mode 100644 index 7fa1c9a70d1..00000000000 --- a/app/javascript/mastodon/actions/filters.js +++ /dev/null @@ -1,26 +0,0 @@ -import api from '../api'; - -export const FILTERS_FETCH_REQUEST = 'FILTERS_FETCH_REQUEST'; -export const FILTERS_FETCH_SUCCESS = 'FILTERS_FETCH_SUCCESS'; -export const FILTERS_FETCH_FAIL = 'FILTERS_FETCH_FAIL'; - -export const fetchFilters = () => (dispatch, getState) => { - dispatch({ - type: FILTERS_FETCH_REQUEST, - skipLoading: true, - }); - - api(getState) - .get('/api/v1/filters') - .then(({ data }) => dispatch({ - type: FILTERS_FETCH_SUCCESS, - filters: data, - skipLoading: true, - })) - .catch(err => dispatch({ - type: FILTERS_FETCH_FAIL, - err, - skipLoading: true, - skipAlert: true, - })); -}; diff --git a/app/javascript/mastodon/actions/importer/index.js b/app/javascript/mastodon/actions/importer/index.js index f4372fb31d0..9c69be601ed 100644 --- a/app/javascript/mastodon/actions/importer/index.js +++ b/app/javascript/mastodon/actions/importer/index.js @@ -5,6 +5,7 @@ export const ACCOUNTS_IMPORT = 'ACCOUNTS_IMPORT'; export const STATUS_IMPORT = 'STATUS_IMPORT'; export const STATUSES_IMPORT = 'STATUSES_IMPORT'; export const POLLS_IMPORT = 'POLLS_IMPORT'; +export const FILTERS_IMPORT = 'FILTERS_IMPORT'; function pushUnique(array, object) { if (array.every(element => element.id !== object.id)) { @@ -28,6 +29,10 @@ export function importStatuses(statuses) { return { type: STATUSES_IMPORT, statuses }; } +export function importFilters(filters) { + return { type: FILTERS_IMPORT, filters }; +} + export function importPolls(polls) { return { type: POLLS_IMPORT, polls }; } @@ -61,11 +66,16 @@ export function importFetchedStatuses(statuses) { const accounts = []; const normalStatuses = []; const polls = []; + const filters = []; function processStatus(status) { pushUnique(normalStatuses, normalizeStatus(status, getState().getIn(['statuses', status.id]))); pushUnique(accounts, status.account); + if (status.filtered) { + status.filtered.forEach(result => pushUnique(filters, result.filter)); + } + if (status.reblog && status.reblog.id) { processStatus(status.reblog); } @@ -80,6 +90,7 @@ export function importFetchedStatuses(statuses) { dispatch(importPolls(polls)); dispatch(importFetchedAccounts(accounts)); dispatch(importStatuses(normalStatuses)); + dispatch(importFilters(filters)); }; } diff --git a/app/javascript/mastodon/actions/importer/normalizer.js b/app/javascript/mastodon/actions/importer/normalizer.js index ca76e3494d1..8a22f83fa43 100644 --- a/app/javascript/mastodon/actions/importer/normalizer.js +++ b/app/javascript/mastodon/actions/importer/normalizer.js @@ -42,6 +42,14 @@ export function normalizeAccount(account) { return account; } +export function normalizeFilterResult(result) { + const normalResult = { ...result }; + + normalResult.filter = normalResult.filter.id; + + return normalResult; +} + export function normalizeStatus(status, normalOldStatus) { const normalStatus = { ...status }; normalStatus.account = status.account.id; @@ -54,6 +62,10 @@ export function normalizeStatus(status, normalOldStatus) { normalStatus.poll = status.poll.id; } + if (status.filtered) { + normalStatus.filtered = status.filtered.map(normalizeFilterResult); + } + // Only calculate these values when status first encountered and // when the underlying values change. Otherwise keep the ones // already in the reducer diff --git a/app/javascript/mastodon/actions/notifications.js b/app/javascript/mastodon/actions/notifications.js index 84dfbeef3d1..3c42f71da32 100644 --- a/app/javascript/mastodon/actions/notifications.js +++ b/app/javascript/mastodon/actions/notifications.js @@ -12,10 +12,8 @@ import { saveSettings } from './settings'; import { defineMessages } from 'react-intl'; import { List as ImmutableList } from 'immutable'; import { unescapeHTML } from '../utils/html'; -import { getFiltersRegex } from '../selectors'; import { usePendingItems as preferPendingItems } from 'mastodon/initial_state'; import compareId from 'mastodon/compare_id'; -import { searchTextFromRawStatus } from 'mastodon/actions/importer/normalizer'; import { requestNotificationPermission } from '../utils/notifications'; export const NOTIFICATIONS_UPDATE = 'NOTIFICATIONS_UPDATE'; @@ -62,20 +60,17 @@ export function updateNotifications(notification, intlMessages, intlLocale) { const showInColumn = activeFilter === 'all' ? getState().getIn(['settings', 'notifications', 'shows', notification.type], true) : activeFilter === notification.type; const showAlert = getState().getIn(['settings', 'notifications', 'alerts', notification.type], true); const playSound = getState().getIn(['settings', 'notifications', 'sounds', notification.type], true); - const filters = getFiltersRegex(getState(), { contextType: 'notifications' }); let filtered = false; - if (['mention', 'status'].includes(notification.type)) { - const dropRegex = filters[0]; - const regex = filters[1]; - const searchIndex = searchTextFromRawStatus(notification.status); + if (['mention', 'status'].includes(notification.type) && notification.status.filtered) { + const filters = notification.status.filtered.filter(result => result.filter.context.includes('notifications')); - if (dropRegex && dropRegex.test(searchIndex)) { + if (filters.some(result => result.filter.filter_action === 'hide')) { return; } - filtered = regex && regex.test(searchIndex); + filtered = filters.length > 0; } if (['follow_request'].includes(notification.type)) { diff --git a/app/javascript/mastodon/actions/streaming.js b/app/javascript/mastodon/actions/streaming.js index d76f045c878..84709083fa2 100644 --- a/app/javascript/mastodon/actions/streaming.js +++ b/app/javascript/mastodon/actions/streaming.js @@ -21,7 +21,6 @@ import { updateReaction as updateAnnouncementsReaction, deleteAnnouncement, } from './announcements'; -import { fetchFilters } from './filters'; import { getLocale } from '../locales'; const { messages } = getLocale(); @@ -97,9 +96,6 @@ export const connectTimelineStream = (timelineId, channelName, params = {}, opti case 'conversation': dispatch(updateConversations(JSON.parse(data.payload))); break; - case 'filters_changed': - dispatch(fetchFilters()); - break; case 'announcement': dispatch(updateAnnouncements(JSON.parse(data.payload))); break; diff --git a/app/javascript/mastodon/components/status.js b/app/javascript/mastodon/components/status.js index 7c44669d2b5..4ca39282426 100644 --- a/app/javascript/mastodon/components/status.js +++ b/app/javascript/mastodon/components/status.js @@ -116,6 +116,7 @@ class Status extends ImmutablePureComponent { state = { showMedia: defaultMediaVisibility(this.props.status), statusId: undefined, + forceFilter: undefined, }; static getDerivedStateFromProps(nextProps, prevState) { @@ -277,6 +278,15 @@ class Status extends ImmutablePureComponent { this.handleToggleMediaVisibility(); } + handleUnfilterClick = e => { + this.setState({ forceFilter: false }); + e.preventDefault(); + } + + handleFilterClick = () => { + this.setState({ forceFilter: true }); + } + _properStatus () { const { status } = this.props; @@ -328,7 +338,8 @@ class Status extends ImmutablePureComponent { ); } - if (status.get('filtered') || status.getIn(['reblog', 'filtered'])) { + const matchedFilters = status.get('filtered') || status.getIn(['reblog', 'filtered']); + if (this.state.forceFilter === undefined ? matchedFilters : this.state.forceFilter) { const minHandlers = this.props.muted ? {} : { moveUp: this.handleHotkeyMoveUp, moveDown: this.handleHotkeyMoveDown, @@ -337,7 +348,11 @@ class Status extends ImmutablePureComponent { return (
- + : {matchedFilters.join(', ')}. + {' '} +
); @@ -496,7 +511,7 @@ class Status extends ImmutablePureComponent { {media} - + diff --git a/app/javascript/mastodon/components/status_action_bar.js b/app/javascript/mastodon/components/status_action_bar.js index 1d8fe23dae8..ab8755be04c 100644 --- a/app/javascript/mastodon/components/status_action_bar.js +++ b/app/javascript/mastodon/components/status_action_bar.js @@ -38,6 +38,7 @@ const messages = defineMessages({ admin_account: { id: 'status.admin_account', defaultMessage: 'Open moderation interface for @{name}' }, admin_status: { id: 'status.admin_status', defaultMessage: 'Open this status in the moderation interface' }, copy: { id: 'status.copy', defaultMessage: 'Copy link to status' }, + hide: { id: 'status.hide', defaultMessage: 'Hide toot' }, blockDomain: { id: 'account.block_domain', defaultMessage: 'Block domain {domain}' }, unblockDomain: { id: 'account.unblock_domain', defaultMessage: 'Unblock domain {domain}' }, unmute: { id: 'account.unmute', defaultMessage: 'Unmute @{name}' }, @@ -76,6 +77,7 @@ class StatusActionBar extends ImmutablePureComponent { onMuteConversation: PropTypes.func, onPin: PropTypes.func, onBookmark: PropTypes.func, + onFilter: PropTypes.func, withDismiss: PropTypes.bool, withCounters: PropTypes.bool, scrollKey: PropTypes.string, @@ -207,6 +209,10 @@ class StatusActionBar extends ImmutablePureComponent { this.props.onMuteConversation(this.props.status); } + handleFilter = () => { + this.props.onFilter(); + } + handleCopy = () => { const url = this.props.status.get('url'); const textarea = document.createElement('textarea'); @@ -226,6 +232,11 @@ class StatusActionBar extends ImmutablePureComponent { } } + + handleFilterClick = () => { + this.props.onFilter(); + } + render () { const { status, relationship, intl, withDismiss, withCounters, scrollKey } = this.props; @@ -329,6 +340,10 @@ class StatusActionBar extends ImmutablePureComponent { ); + const filterButton = this.props.onFilter && ( + + ); + return (
@@ -337,6 +352,8 @@ class StatusActionBar extends ImmutablePureComponent { {shareButton} + {filterButton} +
this.props.dispatch(fetchFilters()), 500); + setTimeout(() => this.props.dispatch(fetchRules()), 3000); this.hotkeys.__mousetrap__.stopCallback = (e, element) => { diff --git a/app/javascript/mastodon/reducers/filters.js b/app/javascript/mastodon/reducers/filters.js index 33f0c673280..14b7040273e 100644 --- a/app/javascript/mastodon/reducers/filters.js +++ b/app/javascript/mastodon/reducers/filters.js @@ -1,10 +1,34 @@ -import { FILTERS_FETCH_SUCCESS } from '../actions/filters'; -import { List as ImmutableList, fromJS } from 'immutable'; +import { FILTERS_IMPORT } from '../actions/importer'; +import { Map as ImmutableMap, is, fromJS } from 'immutable'; -export default function filters(state = ImmutableList(), action) { +const normalizeFilter = (state, filter) => { + const normalizedFilter = fromJS({ + id: filter.id, + title: filter.title, + context: filter.context, + filter_action: filter.filter_action, + expires_at: filter.expires_at ? Date.parse(filter.expires_at) : null, + }); + + if (is(state.get(filter.id), normalizedFilter)) { + return state; + } else { + return state.set(filter.id, normalizedFilter); + } +}; + +const normalizeFilters = (state, filters) => { + filters.forEach(filter => { + state = normalizeFilter(state, filter); + }); + + return state; +}; + +export default function filters(state = ImmutableMap(), action) { switch(action.type) { - case FILTERS_FETCH_SUCCESS: - return fromJS(action.filters); + case FILTERS_IMPORT: + return normalizeFilters(state, action.filters); default: return state; } diff --git a/app/javascript/mastodon/selectors/index.js b/app/javascript/mastodon/selectors/index.js index fbd25b605a3..6aeb8b7bdde 100644 --- a/app/javascript/mastodon/selectors/index.js +++ b/app/javascript/mastodon/selectors/index.js @@ -40,15 +40,15 @@ const toServerSideType = columnType => { const escapeRegExp = string => string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string -const regexFromFilters = filters => { - if (filters.size === 0) { +const regexFromKeywords = keywords => { + if (keywords.size === 0) { return null; } - return new RegExp(filters.map(filter => { - let expr = escapeRegExp(filter.get('phrase')); + return new RegExp(keywords.map(keyword_filter => { + let expr = escapeRegExp(keyword_filter.get('keyword')); - if (filter.get('whole_word')) { + if (keyword_filter.get('whole_word')) { if (/^[\w]/.test(expr)) { expr = `\\b${expr}`; } @@ -62,27 +62,15 @@ const regexFromFilters = filters => { }).join('|'), 'i'); }; -// Memoize the filter regexps for each valid server contextType -const makeGetFiltersRegex = () => { - let memo = {}; +const getFilters = (state, { contextType }) => { + if (!contextType) return null; - return (state, { contextType }) => { - if (!contextType) return ImmutableList(); + const serverSideType = toServerSideType(contextType); + const now = new Date(); - const serverSideType = toServerSideType(contextType); - const filters = state.get('filters', ImmutableList()).filter(filter => filter.get('context').includes(serverSideType) && (filter.get('expires_at') === null || Date.parse(filter.get('expires_at')) > (new Date()))); - - if (!memo[serverSideType] || !is(memo[serverSideType].filters, filters)) { - const dropRegex = regexFromFilters(filters.filter(filter => filter.get('irreversible'))); - const regex = regexFromFilters(filters); - memo[serverSideType] = { filters: filters, results: [dropRegex, regex] }; - } - return memo[serverSideType].results; - }; + return state.get('filters').filter((filter) => filter.get('context').includes(serverSideType) && (filter.get('expires_at') === null || filter.get('expires_at') > now)); }; -export const getFiltersRegex = makeGetFiltersRegex(); - export const makeGetStatus = () => { return createSelector( [ @@ -90,10 +78,10 @@ export const makeGetStatus = () => { (state, { id }) => state.getIn(['statuses', state.getIn(['statuses', id, 'reblog'])]), (state, { id }) => state.getIn(['accounts', state.getIn(['statuses', id, 'account'])]), (state, { id }) => state.getIn(['accounts', state.getIn(['statuses', state.getIn(['statuses', id, 'reblog']), 'account'])]), - getFiltersRegex, + getFilters, ], - (statusBase, statusReblog, accountBase, accountReblog, filtersRegex) => { + (statusBase, statusReblog, accountBase, accountReblog, filters) => { if (!statusBase) { return null; } @@ -104,14 +92,17 @@ export const makeGetStatus = () => { statusReblog = null; } - const dropRegex = (accountReblog || accountBase).get('id') !== me && filtersRegex[0]; - if (dropRegex && dropRegex.test(statusBase.get('reblog') ? statusReblog.get('search_index') : statusBase.get('search_index'))) { - return null; + let filtered = false; + if ((accountReblog || accountBase).get('id') !== me && filters) { + let filterResults = statusReblog?.get('filtered') || statusBase.get('filtered') || ImmutableList(); + if (filterResults.some((result) => filters.getIn([result.get('filter'), 'filter_action']) === 'hide')) { + return null; + } + if (!filterResults.isEmpty()) { + filtered = filterResults.map(result => filters.getIn([result.get('filter'), 'title'])); + } } - const regex = (accountReblog || accountBase).get('id') !== me && filtersRegex[1]; - const filtered = regex && regex.test(statusBase.get('reblog') ? statusReblog.get('search_index') : statusBase.get('search_index')); - return statusBase.withMutations(map => { map.set('reblog', statusReblog); map.set('account', accountBase); diff --git a/app/javascript/packs/public.js b/app/javascript/packs/public.js index 3d0a937e1f8..e42468e0c69 100644 --- a/app/javascript/packs/public.js +++ b/app/javascript/packs/public.js @@ -4,6 +4,7 @@ import loadPolyfills from '../mastodon/load_polyfills'; import ready from '../mastodon/ready'; import { start } from '../mastodon/common'; import loadKeyboardExtensions from '../mastodon/load_keyboard_extensions'; +import 'cocoon-js-vanilla'; start(); diff --git a/app/javascript/styles/mastodon/admin.scss b/app/javascript/styles/mastodon/admin.scss index 66e2997f1f8..4ce5cd10139 100644 --- a/app/javascript/styles/mastodon/admin.scss +++ b/app/javascript/styles/mastodon/admin.scss @@ -915,7 +915,8 @@ a.name-tag, text-align: center; } -.applications-list__item { +.applications-list__item, +.filters-list__item { padding: 15px 0; background: $ui-base-color; border: 1px solid lighten($ui-base-color, 4%); @@ -923,7 +924,8 @@ a.name-tag, margin-top: 15px; } -.announcements-list { +.announcements-list, +.filters-list { border: 1px solid lighten($ui-base-color, 4%); border-radius: 4px; @@ -976,6 +978,33 @@ a.name-tag, } } +.filters-list__item { + &__title { + display: flex; + justify-content: space-between; + margin-bottom: 0; + } + + &__permissions { + margin-top: 0; + margin-bottom: 10px; + } + + .expiration { + font-size: 13px; + } + + &.expired { + .expiration { + color: lighten($error-red, 12%); + } + + .permissions-list__item__icon { + color: $dark-text-color; + } + } +} + .dashboard__counters.admin-account-counters { margin-top: 10px; } diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss index 7e3ce3de241..592ce91f3bd 100644 --- a/app/javascript/styles/mastodon/components.scss +++ b/app/javascript/styles/mastodon/components.scss @@ -959,6 +959,21 @@ width: 100%; clear: both; border-bottom: 1px solid lighten($ui-base-color, 8%); + + &__button { + display: inline; + color: lighten($ui-highlight-color, 8%); + border: 0; + background: transparent; + padding: 0; + font-size: inherit; + line-height: inherit; + + &:hover, + &:active { + text-decoration: underline; + } + } } .status__prepend-icon-wrapper { diff --git a/app/javascript/styles/mastodon/forms.scss b/app/javascript/styles/mastodon/forms.scss index d57eabc09ff..da699dd25ed 100644 --- a/app/javascript/styles/mastodon/forms.scss +++ b/app/javascript/styles/mastodon/forms.scss @@ -1070,3 +1070,34 @@ code { } } } + +.keywords-table { + thead { + th { + white-space: nowrap; + } + + th:first-child { + width: 100%; + } + } + + tfoot { + td { + border: 0; + } + } + + .input.string { + margin-bottom: 0; + } + + .label_input__wrapper { + margin-top: 10px; + } + + .table-action-link { + margin-top: 10px; + white-space: nowrap; + } +} diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index 4811ebbcc1b..2eb4ba2f4df 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -352,7 +352,6 @@ class FeedManager def filter_from_home?(status, receiver_id, crutches) return false if receiver_id == status.account_id return true if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?) - return true if phrase_filtered?(status, receiver_id, :home) check_for_blocks = crutches[:active_mentions][status.id] || [] check_for_blocks.concat([status.account_id]) @@ -388,7 +387,6 @@ class FeedManager # @return [Boolean] def filter_from_mentions?(status, receiver_id) return true if receiver_id == status.account_id - return true if phrase_filtered?(status, receiver_id, :notifications) # This filter is called from NotifyService, but already after the sender of # the notification has been checked for mute/block. Therefore, it's not @@ -418,34 +416,6 @@ class FeedManager false end - # Check if the status hits a phrase filter - # @param [Status] status - # @param [Integer] receiver_id - # @param [Symbol] context - # @return [Boolean] - def phrase_filtered?(status, receiver_id, context) - active_filters = Rails.cache.fetch("filters:#{receiver_id}") { CustomFilter.where(account_id: receiver_id).active_irreversible.to_a }.to_a - - active_filters.select! { |filter| filter.context.include?(context.to_s) && !filter.expired? } - - active_filters.map! do |filter| - if filter.whole_word - sb = /\A[[:word:]]/.match?(filter.phrase) ? '\b' : '' - eb = /[[:word:]]\z/.match?(filter.phrase) ? '\b' : '' - - /(?mix:#{sb}#{Regexp.escape(filter.phrase)}#{eb})/ - else - /#{Regexp.escape(filter.phrase)}/i - end - end - - return false if active_filters.empty? - - combined_regex = Regexp.union(active_filters) - - combined_regex.match?(status.proper.searchable_text) - end - # Adds a status to an account's feed, returning true if a status was # added, and false if it was not added to the feed. Note that this is # an internal helper: callers must call trim or push updates if diff --git a/app/models/concerns/account_interactions.rb b/app/models/concerns/account_interactions.rb index ad1665dc41e..a7401362f40 100644 --- a/app/models/concerns/account_interactions.rb +++ b/app/models/concerns/account_interactions.rb @@ -247,6 +247,19 @@ module AccountInteractions account_pins.where(target_account: account).exists? end + def status_matches_filters(status) + active_filters = CustomFilter.cached_filters_for(id) + + filter_matches = active_filters.filter_map do |filter, rules| + next if rules[:keywords].blank? + + match = rules[:keywords].match(status.proper.searchable_text) + FilterResultPresenter.new(filter: filter, keyword_matches: [match.to_s]) unless match.nil? + end + + filter_matches + end + def followers_for_local_distribution followers.local .joins(:user) diff --git a/app/models/custom_filter.rb b/app/models/custom_filter.rb index 8e347679414..e98ed7df9fb 100644 --- a/app/models/custom_filter.rb +++ b/app/models/custom_filter.rb @@ -3,18 +3,22 @@ # # Table name: custom_filters # -# id :bigint(8) not null, primary key -# account_id :bigint(8) -# expires_at :datetime -# phrase :text default(""), not null -# context :string default([]), not null, is an Array -# whole_word :boolean default(TRUE), not null -# irreversible :boolean default(FALSE), not null -# created_at :datetime not null -# updated_at :datetime not null +# id :bigint not null, primary key +# account_id :bigint +# expires_at :datetime +# phrase :text default(""), not null +# context :string default([]), not null, is an Array +# created_at :datetime not null +# updated_at :datetime not null +# action :integer default(0), not null # class CustomFilter < ApplicationRecord + self.ignored_columns = %w(whole_word irreversible) + + alias_attribute :title, :phrase + alias_attribute :filter_action, :action + VALID_CONTEXTS = %w( home notifications @@ -26,16 +30,20 @@ class CustomFilter < ApplicationRecord include Expireable include Redisable + enum action: [:warn, :hide], _suffix: :action + belongs_to :account + has_many :keywords, class_name: 'CustomFilterKeyword', foreign_key: :custom_filter_id, inverse_of: :custom_filter, dependent: :destroy + accepts_nested_attributes_for :keywords, reject_if: :all_blank, allow_destroy: true - validates :phrase, :context, presence: true + validates :title, :context, presence: true validate :context_must_be_valid - validate :irreversible_must_be_within_context - - scope :active_irreversible, -> { where(irreversible: true).where(Arel.sql('expires_at IS NULL OR expires_at > NOW()')) } before_validation :clean_up_contexts - after_commit :remove_cache + + before_save :prepare_cache_invalidation! + before_destroy :prepare_cache_invalidation! + after_commit :invalidate_cache! def expires_in return @expires_in if defined?(@expires_in) @@ -44,22 +52,55 @@ class CustomFilter < ApplicationRecord [30.minutes, 1.hour, 6.hours, 12.hours, 1.day, 1.week].find { |expires_in| expires_in.from_now >= expires_at } end + def irreversible=(value) + self.action = value ? :hide : :warn + end + + def irreversible? + hide_action? + end + + def self.cached_filters_for(account_id) + active_filters = Rails.cache.fetch("filters:v3:#{account_id}") do + scope = CustomFilterKeyword.includes(:custom_filter).where(custom_filter: { account_id: account_id }).where(Arel.sql('expires_at IS NULL OR expires_at > NOW()')) + scope.to_a.group_by(&:custom_filter).map do |filter, keywords| + keywords.map! do |keyword| + if keyword.whole_word + sb = /\A[[:word:]]/.match?(keyword.keyword) ? '\b' : '' + eb = /[[:word:]]\z/.match?(keyword.keyword) ? '\b' : '' + + /(?mix:#{sb}#{Regexp.escape(keyword.keyword)}#{eb})/ + else + /#{Regexp.escape(keyword.keyword)}/i + end + end + [filter, { keywords: Regexp.union(keywords) }] + end + end.to_a + + active_filters.select { |custom_filter, _| !custom_filter.expired? } + end + + def prepare_cache_invalidation! + @should_invalidate_cache = true + end + + def invalidate_cache! + return unless @should_invalidate_cache + @should_invalidate_cache = false + + Rails.cache.delete("filters:v3:#{account_id}") + redis.publish("timeline:#{account_id}", Oj.dump(event: :filters_changed)) + redis.publish("timeline:system:#{account_id}", Oj.dump(event: :filters_changed)) + end + private def clean_up_contexts self.context = Array(context).map(&:strip).filter_map(&:presence) end - def remove_cache - Rails.cache.delete("filters:#{account_id}") - redis.publish("timeline:#{account_id}", Oj.dump(event: :filters_changed)) - end - def context_must_be_valid errors.add(:context, I18n.t('filters.errors.invalid_context')) if context.empty? || context.any? { |c| !VALID_CONTEXTS.include?(c) } end - - def irreversible_must_be_within_context - errors.add(:irreversible, I18n.t('filters.errors.invalid_irreversible')) if irreversible? && !context.include?('home') && !context.include?('notifications') - end end diff --git a/app/models/custom_filter_keyword.rb b/app/models/custom_filter_keyword.rb new file mode 100644 index 00000000000..bf5c5574693 --- /dev/null +++ b/app/models/custom_filter_keyword.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: custom_filter_keywords +# +# id :bigint not null, primary key +# custom_filter_id :bigint not null +# keyword :text default(""), not null +# whole_word :boolean default(TRUE), not null +# created_at :datetime not null +# updated_at :datetime not null +# + +class CustomFilterKeyword < ApplicationRecord + belongs_to :custom_filter + + validates :keyword, presence: true + + alias_attribute :phrase, :keyword + + before_save :prepare_cache_invalidation! + before_destroy :prepare_cache_invalidation! + after_commit :invalidate_cache! + + private + + def prepare_cache_invalidation! + custom_filter.prepare_cache_invalidation! + end + + def invalidate_cache! + custom_filter.invalidate_cache! + end +end diff --git a/app/presenters/filter_result_presenter.rb b/app/presenters/filter_result_presenter.rb new file mode 100644 index 00000000000..677225f5ec2 --- /dev/null +++ b/app/presenters/filter_result_presenter.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +class FilterResultPresenter < ActiveModelSerializers::Model + attributes :filter, :keyword_matches +end diff --git a/app/presenters/status_relationships_presenter.rb b/app/presenters/status_relationships_presenter.rb index 4163bb098c0..d7ffb1954af 100644 --- a/app/presenters/status_relationships_presenter.rb +++ b/app/presenters/status_relationships_presenter.rb @@ -2,7 +2,7 @@ class StatusRelationshipsPresenter attr_reader :reblogs_map, :favourites_map, :mutes_map, :pins_map, - :bookmarks_map + :bookmarks_map, :filters_map def initialize(statuses, current_account_id = nil, **options) if current_account_id.nil? @@ -11,12 +11,14 @@ class StatusRelationshipsPresenter @bookmarks_map = {} @mutes_map = {} @pins_map = {} + @filters_map = {} else statuses = statuses.compact status_ids = statuses.flat_map { |s| [s.id, s.reblog_of_id] }.uniq.compact conversation_ids = statuses.filter_map(&:conversation_id).uniq pinnable_status_ids = statuses.map(&:proper).filter_map { |s| s.id if s.account_id == current_account_id && %w(public unlisted private).include?(s.visibility) } + @filters_map = build_filters_map(statuses, current_account_id).merge(options[:filters_map] || {}) @reblogs_map = Status.reblogs_map(status_ids, current_account_id).merge(options[:reblogs_map] || {}) @favourites_map = Status.favourites_map(status_ids, current_account_id).merge(options[:favourites_map] || {}) @bookmarks_map = Status.bookmarks_map(status_ids, current_account_id).merge(options[:bookmarks_map] || {}) @@ -24,4 +26,24 @@ class StatusRelationshipsPresenter @pins_map = Status.pins_map(pinnable_status_ids, current_account_id).merge(options[:pins_map] || {}) end end + + private + + def build_filters_map(statuses, current_account_id) + active_filters = CustomFilter.cached_filters_for(current_account_id) + + @filters_map = statuses.each_with_object({}) do |status, h| + filter_matches = active_filters.filter_map do |filter, rules| + next if rules[:keywords].blank? + + match = rules[:keywords].match(status.proper.searchable_text) + FilterResultPresenter.new(filter: filter, keyword_matches: [match.to_s]) unless match.nil? + end + + unless filter_matches.empty? + h[status.id] = filter_matches + h[status.reblog_of_id] = filter_matches if status.reblog? + end + end + end end diff --git a/app/serializers/rest/filter_keyword_serializer.rb b/app/serializers/rest/filter_keyword_serializer.rb new file mode 100644 index 00000000000..dd2ebac6eae --- /dev/null +++ b/app/serializers/rest/filter_keyword_serializer.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class REST::FilterKeywordSerializer < ActiveModel::Serializer + attributes :id, :keyword, :whole_word + + def id + object.id.to_s + end +end diff --git a/app/serializers/rest/filter_result_serializer.rb b/app/serializers/rest/filter_result_serializer.rb new file mode 100644 index 00000000000..0ef4db79a87 --- /dev/null +++ b/app/serializers/rest/filter_result_serializer.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +class REST::FilterResultSerializer < ActiveModel::Serializer + belongs_to :filter, serializer: REST::FilterSerializer + has_many :keyword_matches +end diff --git a/app/serializers/rest/filter_serializer.rb b/app/serializers/rest/filter_serializer.rb index 57205630bbb..98d7edb175a 100644 --- a/app/serializers/rest/filter_serializer.rb +++ b/app/serializers/rest/filter_serializer.rb @@ -1,10 +1,14 @@ # frozen_string_literal: true class REST::FilterSerializer < ActiveModel::Serializer - attributes :id, :phrase, :context, :whole_word, :expires_at, - :irreversible + attributes :id, :title, :context, :expires_at, :filter_action + has_many :keywords, serializer: REST::FilterKeywordSerializer, if: :rules_requested? def id object.id.to_s end + + def rules_requested? + instance_options[:rules_requested] + end end diff --git a/app/serializers/rest/status_serializer.rb b/app/serializers/rest/status_serializer.rb index 6bd6a23e51b..e0b8f32a68f 100644 --- a/app/serializers/rest/status_serializer.rb +++ b/app/serializers/rest/status_serializer.rb @@ -13,6 +13,7 @@ class REST::StatusSerializer < ActiveModel::Serializer attribute :muted, if: :current_user? attribute :bookmarked, if: :current_user? attribute :pinned, if: :pinnable? + has_many :filtered, serializer: REST::FilterResultSerializer, if: :current_user? attribute :content, unless: :source_requested? attribute :text, if: :source_requested? @@ -120,6 +121,14 @@ class REST::StatusSerializer < ActiveModel::Serializer end end + def filtered + if instance_options && instance_options[:relationships] + instance_options[:relationships].filters_map[object.id] || [] + else + current_user.account.status_matches_filters(object) + end + end + def pinnable? current_user? && current_user.account_id == object.account_id && diff --git a/app/serializers/rest/v1/filter_serializer.rb b/app/serializers/rest/v1/filter_serializer.rb new file mode 100644 index 00000000000..455f17efdb3 --- /dev/null +++ b/app/serializers/rest/v1/filter_serializer.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class REST::V1::FilterSerializer < ActiveModel::Serializer + attributes :id, :phrase, :context, :whole_word, :expires_at, + :irreversible + + delegate :context, :expires_at, to: :custom_filter + + def id + object.id.to_s + end + + def phrase + object.keyword + end + + def irreversible + custom_filter.irreversible? + end + + private + + def custom_filter + object.custom_filter + end +end diff --git a/app/views/filters/_fields.html.haml b/app/views/filters/_fields.html.haml deleted file mode 100644 index 84dcdcca515..00000000000 --- a/app/views/filters/_fields.html.haml +++ /dev/null @@ -1,16 +0,0 @@ -.fields-row - .fields-row__column.fields-row__column-6.fields-group - = f.input :phrase, as: :string, wrapper: :with_label, hint: false - .fields-row__column.fields-row__column-6.fields-group - = f.input :expires_in, wrapper: :with_label, collection: [30.minutes, 1.hour, 6.hours, 12.hours, 1.day, 1.week].map(&:to_i), label_method: lambda { |i| I18n.t("invites.expires_in.#{i}") }, include_blank: I18n.t('invites.expires_in_prompt') - -.fields-group - = f.input :context, wrapper: :with_block_label, collection: CustomFilter::VALID_CONTEXTS, as: :check_boxes, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li', label_method: lambda { |context| I18n.t("filters.contexts.#{context}") }, include_blank: false - -%hr.spacer/ - -.fields-group - = f.input :irreversible, wrapper: :with_label - -.fields-group - = f.input :whole_word, wrapper: :with_label diff --git a/app/views/filters/_filter.html.haml b/app/views/filters/_filter.html.haml new file mode 100644 index 00000000000..2ab014081c8 --- /dev/null +++ b/app/views/filters/_filter.html.haml @@ -0,0 +1,32 @@ +.filters-list__item{ class: [filter.expired? && 'expired'] } + = link_to edit_filter_path(filter), class: 'filters-list__item__title' do + = filter.title + + - if filter.expires? + .expiration{ title: t('filters.index.expires_on', date: l(filter.expires_at)) } + - if filter.expired? + = t('invites.expired') + - else + = t('filters.index.expires_in', distance: distance_of_time_in_words_to_now(filter.expires_at)) + + .filters-list__item__permissions + %ul.permissions-list + - unless filter.keywords.empty? + %li.permissions-list__item + .permissions-list__item__icon + = fa_icon('paragraph') + .permissions-list__item__text + .permissions-list__item__text__title + = t('filters.index.keywords', count: filter.keywords.size) + .permissions-list__item__text__type + - keywords = filter.keywords.map(&:keyword) + - keywords = keywords.take(5) + ['…'] if keywords.size > 5 # TODO + = keywords.join(', ') + + .announcements-list__item__action-bar + .announcements-list__item__meta + = t('filters.index.contexts', contexts: filter.context.map { |context| I18n.t("filters.contexts.#{context}") }.join(', ')) + + %div + = table_link_to 'pencil', t('filters.edit.title'), edit_filter_path(filter) + = table_link_to 'times', t('filters.index.delete'), filter_path(filter), method: :delete, data: { confirm: t('admin.accounts.are_you_sure') } diff --git a/app/views/filters/_filter_fields.html.haml b/app/views/filters/_filter_fields.html.haml new file mode 100644 index 00000000000..1a52faa7af5 --- /dev/null +++ b/app/views/filters/_filter_fields.html.haml @@ -0,0 +1,33 @@ +.fields-row + .fields-row__column.fields-row__column-6.fields-group + = f.input :title, as: :string, wrapper: :with_label, hint: false + .fields-row__column.fields-row__column-6.fields-group + = f.input :expires_in, wrapper: :with_label, collection: [30.minutes, 1.hour, 6.hours, 12.hours, 1.day, 1.week].map(&:to_i), label_method: lambda { |i| I18n.t("invites.expires_in.#{i}") }, include_blank: I18n.t('invites.expires_in_prompt') + +.fields-group + = f.input :context, wrapper: :with_block_label, collection: CustomFilter::VALID_CONTEXTS, as: :check_boxes, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li', label_method: lambda { |context| I18n.t("filters.contexts.#{context}") }, include_blank: false + +%hr.spacer/ + +.fields-group + = f.input :filter_action, as: :radio_buttons, collection: %i(warn hide), include_blank: false, wrapper: :with_block_label, label_method: ->(action) { safe_join([t("simple_form.labels.filters.actions.#{action}"), content_tag(:span, t("simple_form.hints.filters.actions.#{action}"), class: 'hint')]) }, hint: t('simple_form.hints.filters.action'), required: true + +%hr.spacer/ + +%h4= t('filters.edit.keywords') + +.table-wrapper + %table.table.keywords-table + %thead + %tr + %th= t('simple_form.labels.defaults.phrase') + %th= t('simple_form.labels.defaults.whole_word') + %th + %tbody + = f.simple_fields_for :keywords do |keyword| + = render 'keyword_fields', f: keyword + %tfoot + %tr + %td{ colspan: 3} + = link_to_add_association f, :keywords, class: 'table-action-link', partial: 'keyword_fields', 'data-association-insertion-node': '.keywords-table tbody', 'data-association-insertion-method': 'append' do + = safe_join([fa_icon('plus'), t('filters.edit.add_keyword')]) diff --git a/app/views/filters/_keyword_fields.html.haml b/app/views/filters/_keyword_fields.html.haml new file mode 100644 index 00000000000..eedd514ef52 --- /dev/null +++ b/app/views/filters/_keyword_fields.html.haml @@ -0,0 +1,8 @@ +%tr.nested-fields + %td= f.input :keyword, as: :string + %td + .label_input__wrapper= f.input_field :whole_word + %td + = f.hidden_field :id if f.object&.persisted? # Required so Rails doesn't put the field outside of the + = link_to_remove_association(f, class: 'table-action-link') do + = safe_join([fa_icon('times'), t('filters.index.delete')]) diff --git a/app/views/filters/edit.html.haml b/app/views/filters/edit.html.haml index e971215ac6b..3dc3f07b72c 100644 --- a/app/views/filters/edit.html.haml +++ b/app/views/filters/edit.html.haml @@ -2,7 +2,7 @@ = t('filters.edit.title') = simple_form_for @filter, url: filter_path(@filter), method: :put do |f| - = render 'fields', f: f + = render 'filter_fields', f: f .actions = f.button :button, t('generic.save_changes'), type: :submit diff --git a/app/views/filters/index.html.haml b/app/views/filters/index.html.haml index b4d5333aa10..0227526a47e 100644 --- a/app/views/filters/index.html.haml +++ b/app/views/filters/index.html.haml @@ -7,18 +7,5 @@ - if @filters.empty? %div.muted-hint.center-text= t 'filters.index.empty' - else - .table-wrapper - %table.table - %thead - %tr - %th= t('simple_form.labels.defaults.phrase') - %th= t('simple_form.labels.defaults.context') - %th - %tbody - - @filters.each do |filter| - %tr - %td= filter.phrase - %td= filter.context.map { |context| I18n.t("filters.contexts.#{context}") }.join(', ') - %td - = table_link_to 'pencil', t('filters.edit.title'), edit_filter_path(filter) - = table_link_to 'times', t('filters.index.delete'), filter_path(filter), method: :delete + .applications-list + = render partial: 'filter', collection: @filters diff --git a/app/views/filters/new.html.haml b/app/views/filters/new.html.haml index 05bec343f85..5f400e604a7 100644 --- a/app/views/filters/new.html.haml +++ b/app/views/filters/new.html.haml @@ -2,7 +2,7 @@ = t('filters.new.title') = simple_form_for @filter, url: filters_path do |f| - = render 'fields', f: f + = render 'filter_fields', f: f .actions - = f.button :button, t('filters.new.title'), type: :submit + = f.button :button, t('filters.new.save'), type: :submit diff --git a/config/locales/en.yml b/config/locales/en.yml index 5a0fc3da88e..91ae3a3bce6 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1124,15 +1124,24 @@ en: public: Public timelines thread: Conversations edit: + add_keyword: Add keyword + keywords: Keywords title: Edit filter errors: + deprecated_api_multiple_keywords: These parameters cannot be changed from this application because they apply to more than one filter keyword. Use a more recent application or the web interface. invalid_context: None or invalid context supplied - invalid_irreversible: Irreversible filtering only works with home or notifications context index: + contexts: Filters in %{contexts} delete: Delete empty: You have no filters. + expires_in: Expires in %{distance} + expires_on: Expires on %{date} + keywords: + one: "%{count} keyword" + other: "%{count} keywords" title: Filters new: + save: Save new filter title: Add new filter footer: developers: Developers diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml index 7e4f52849a1..ea4f68562a0 100644 --- a/config/locales/simple_form.en.yml +++ b/config/locales/simple_form.en.yml @@ -68,6 +68,11 @@ en: with_dns_records: An attempt to resolve the given domain's DNS records will be made and the results will also be blocked featured_tag: name: 'You might want to use one of these:' + filters: + action: Chose which action to perform when a post matches the filter + actions: + hide: Completely hide the filtered content, behaving as if it did not exist + warn: Hide the filtered content behind a warning mentioning the filter's title form_challenge: current_password: You are entering a secure area imports: @@ -181,6 +186,7 @@ en: setting_use_pending_items: Slow mode severity: Severity sign_in_token_attempt: Security code + title: Title type: Import type username: Username username_or_email: Username or Email @@ -189,6 +195,10 @@ en: with_dns_records: Include MX records and IPs of the domain featured_tag: name: Hashtag + filters: + actions: + hide: Hide completely + warn: Hide with a warning interactions: must_be_follower: Block notifications from non-followers must_be_following: Block notifications from people you don't follow diff --git a/config/routes.rb b/config/routes.rb index 1b9c5079970..4abf55655d6 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -451,10 +451,16 @@ Rails.application.routes.draw do resources :bookmarks, only: [:index] resources :reports, only: [:create] resources :trends, only: [:index], controller: 'trends/tags' - resources :filters, only: [:index, :create, :show, :update, :destroy] + resources :filters, only: [:index, :create, :show, :update, :destroy] do + resources :keywords, only: [:index, :create], controller: 'filters/keywords' + end resources :endorsements, only: [:index] resources :markers, only: [:index, :create] + namespace :filters do + resources :keywords, only: [:show, :update, :destroy] + end + namespace :apps do get :verify_credentials, to: 'credentials#show' end @@ -589,6 +595,7 @@ Rails.application.routes.draw do resources :media, only: [:create] get '/search', to: 'search#index', as: :search resources :suggestions, only: [:index] + resources :filters, only: [:index, :create, :show, :update, :destroy] namespace :admin do resources :accounts, only: [:index] diff --git a/db/migrate/20220613110628_create_custom_filter_keywords.rb b/db/migrate/20220613110628_create_custom_filter_keywords.rb new file mode 100644 index 00000000000..353fc334f0a --- /dev/null +++ b/db/migrate/20220613110628_create_custom_filter_keywords.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class CreateCustomFilterKeywords < ActiveRecord::Migration[6.1] + def change + create_table :custom_filter_keywords do |t| + t.belongs_to :custom_filter, foreign_key: { on_delete: :cascade }, null: false + t.text :keyword, null: false, default: '' + t.boolean :whole_word, null: false, default: true + + t.timestamps + end + end +end diff --git a/db/migrate/20220613110711_migrate_custom_filters.rb b/db/migrate/20220613110711_migrate_custom_filters.rb new file mode 100644 index 00000000000..ea6a9b8c6d1 --- /dev/null +++ b/db/migrate/20220613110711_migrate_custom_filters.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +class MigrateCustomFilters < ActiveRecord::Migration[6.1] + def up + # Preserve IDs as much as possible to not confuse existing clients. + # As long as this migration is irreversible, we do not have to deal with conflicts. + safety_assured do + execute <<-SQL.squish + INSERT INTO custom_filter_keywords (id, custom_filter_id, keyword, whole_word, created_at, updated_at) + SELECT id, id, phrase, whole_word, created_at, updated_at + FROM custom_filters + SQL + end + end + + def down + # Copy back changes from custom filters guaranteed to be from the old API + safety_assured do + execute <<-SQL.squish + UPDATE custom_filters + SET phrase = custom_filter_keywords.keyword, whole_word = custom_filter_keywords.whole_word + FROM custom_filter_keywords + WHERE custom_filters.id = custom_filter_keywords.id AND custom_filters.id = custom_filter_keywords.custom_filter_id + SQL + end + + # Drop every keyword as we can't safely provide a 1:1 mapping + safety_assured do + execute <<-SQL.squish + TRUNCATE custom_filter_keywords RESTART IDENTITY + SQL + end + end +end diff --git a/db/migrate/20220613110834_add_action_to_custom_filters.rb b/db/migrate/20220613110834_add_action_to_custom_filters.rb new file mode 100644 index 00000000000..9427a66fc48 --- /dev/null +++ b/db/migrate/20220613110834_add_action_to_custom_filters.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true +require Rails.root.join('lib', 'mastodon', 'migration_helpers') + +class AddActionToCustomFilters < ActiveRecord::Migration[6.1] + include Mastodon::MigrationHelpers + + disable_ddl_transaction! + + def up + safety_assured do + add_column_with_default :custom_filters, :action, :integer, allow_null: false, default: 0 + execute 'UPDATE custom_filters SET action = 1 WHERE irreversible IS TRUE' + end + end + + def down + execute 'UPDATE custom_filters SET irreversible = (action = 1)' + remove_column :custom_filters, :action + end +end diff --git a/db/post_migrate/20220613110802_remove_whole_word_from_custom_filters.rb b/db/post_migrate/20220613110802_remove_whole_word_from_custom_filters.rb new file mode 100644 index 00000000000..7ef0749e542 --- /dev/null +++ b/db/post_migrate/20220613110802_remove_whole_word_from_custom_filters.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true +require Rails.root.join('lib', 'mastodon', 'migration_helpers') + +class RemoveWholeWordFromCustomFilters < ActiveRecord::Migration[6.1] + include Mastodon::MigrationHelpers + + disable_ddl_transaction! + + def up + safety_assured do + remove_column :custom_filters, :whole_word + end + end + + def down + safety_assured do + add_column_with_default :custom_filters, :whole_word, :boolean, default: true, allow_null: false + end + end +end diff --git a/db/post_migrate/20220613110903_remove_irreversible_from_custom_filters.rb b/db/post_migrate/20220613110903_remove_irreversible_from_custom_filters.rb new file mode 100644 index 00000000000..6ed8bcfeee9 --- /dev/null +++ b/db/post_migrate/20220613110903_remove_irreversible_from_custom_filters.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true +require Rails.root.join('lib', 'mastodon', 'migration_helpers') + +class RemoveIrreversibleFromCustomFilters < ActiveRecord::Migration[6.1] + include Mastodon::MigrationHelpers + + disable_ddl_transaction! + + def up + safety_assured do + remove_column :custom_filters, :irreversible + end + end + + def down + safety_assured do + add_column_with_default :custom_filters, :irreversible, :boolean, allow_null: false, default: false + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 5d8aea60106..759dc712bfe 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2022_06_06_044941) do +ActiveRecord::Schema.define(version: 2022_06_13_110903) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -339,15 +339,23 @@ ActiveRecord::Schema.define(version: 2022_06_06_044941) do t.index ["shortcode", "domain"], name: "index_custom_emojis_on_shortcode_and_domain", unique: true end + create_table "custom_filter_keywords", force: :cascade do |t| + t.bigint "custom_filter_id", null: false + t.text "keyword", default: "", null: false + t.boolean "whole_word", default: true, null: false + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["custom_filter_id"], name: "index_custom_filter_keywords_on_custom_filter_id" + end + create_table "custom_filters", force: :cascade do |t| t.bigint "account_id" t.datetime "expires_at" t.text "phrase", default: "", null: false t.string "context", default: [], null: false, array: true - t.boolean "irreversible", default: false, null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.boolean "whole_word", default: true, null: false + t.integer "action", default: 0, null: false t.index ["account_id"], name: "index_custom_filters_on_account_id" end @@ -1082,6 +1090,7 @@ ActiveRecord::Schema.define(version: 2022_06_06_044941) do add_foreign_key "canonical_email_blocks", "accounts", column: "reference_account_id", on_delete: :cascade add_foreign_key "conversation_mutes", "accounts", name: "fk_225b4212bb", on_delete: :cascade add_foreign_key "conversation_mutes", "conversations", on_delete: :cascade + add_foreign_key "custom_filter_keywords", "custom_filters", on_delete: :cascade add_foreign_key "custom_filters", "accounts", on_delete: :cascade add_foreign_key "devices", "accounts", on_delete: :cascade add_foreign_key "devices", "oauth_access_tokens", column: "access_token_id", on_delete: :cascade diff --git a/lib/tasks/tests.rake b/lib/tasks/tests.rake index 0f3b44a7443..65bff6a8e8f 100644 --- a/lib/tasks/tests.rake +++ b/lib/tasks/tests.rake @@ -38,10 +38,26 @@ namespace :tests do puts 'Instance actor does not have a private key' exit(1) end + + unless Account.find_by(username: 'user', domain: nil).custom_filters.map { |filter| filter.keywords.pluck(:keyword) } == [['test'], ['take']] + puts 'CustomFilterKeyword records not created as expected' + exit(1) + end + end + + desc 'Populate the database with test data for 2.4.3' + task populate_v2_4_3: :environment do # rubocop:disable Naming/VariableNumber + ActiveRecord::Base.connection.execute(<<~SQL) + INSERT INTO "custom_filters" + (id, account_id, phrase, context, whole_word, irreversible, created_at, updated_at) + VALUES + (1, 2, 'test', '{ "home", "public" }', true, true, now(), now()), + (2, 2, 'take', '{ "home" }', false, false, now(), now()); + SQL end desc 'Populate the database with test data for 2.4.0' - task populate_v2_4: :environment do + task populate_v2_4: :environment do # rubocop:disable Naming/VariableNumber ActiveRecord::Base.connection.execute(<<~SQL) INSERT INTO "settings" (id, thing_type, thing_id, var, value, created_at, updated_at) diff --git a/package.json b/package.json index 30fe8b67252..343a8267e55 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "babel-plugin-transform-react-remove-prop-types": "^0.4.24", "blurhash": "^1.1.5", "classnames": "^2.3.1", + "cocoon-js-vanilla": "^1.2.0", "color-blend": "^3.0.1", "compression-webpack-plugin": "^6.1.1", "cross-env": "^7.0.3", @@ -71,6 +72,7 @@ "intl-relativeformat": "^6.4.3", "is-nan": "^1.3.2", "js-yaml": "^4.1.0", + "jsdom": "^20.0.0", "lodash": "^4.17.21", "mark-loader": "^0.1.6", "marky": "^1.2.4", diff --git a/spec/controllers/api/v1/filters/keywords_controller_spec.rb b/spec/controllers/api/v1/filters/keywords_controller_spec.rb new file mode 100644 index 00000000000..aecb4e41c93 --- /dev/null +++ b/spec/controllers/api/v1/filters/keywords_controller_spec.rb @@ -0,0 +1,142 @@ +require 'rails_helper' + +RSpec.describe Api::V1::Filters::KeywordsController, type: :controller do + render_views + + let(:user) { Fabricate(:user) } + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: scopes) } + let(:filter) { Fabricate(:custom_filter, account: user.account) } + let(:other_user) { Fabricate(:user) } + let(:other_filter) { Fabricate(:custom_filter, account: other_user.account) } + + before do + allow(controller).to receive(:doorkeeper_token) { token } + end + + describe 'GET #index' do + let(:scopes) { 'read:filters' } + let!(:keyword) { Fabricate(:custom_filter_keyword, custom_filter: filter) } + + it 'returns http success' do + get :index, params: { filter_id: filter.id } + expect(response).to have_http_status(200) + end + + context "when trying to access another's user filters" do + it 'returns http not found' do + get :index, params: { filter_id: other_filter.id } + expect(response).to have_http_status(404) + end + end + end + + describe 'POST #create' do + let(:scopes) { 'write:filters' } + let(:filter_id) { filter.id } + + before do + post :create, params: { filter_id: filter_id, keyword: 'magic', whole_word: false } + end + + it 'returns http success' do + expect(response).to have_http_status(200) + end + + it 'returns a keyword' do + json = body_as_json + expect(json[:keyword]).to eq 'magic' + expect(json[:whole_word]).to eq false + end + + it 'creates a keyword' do + filter = user.account.custom_filters.first + expect(filter).to_not be_nil + expect(filter.keywords.pluck(:keyword)).to eq ['magic'] + end + + context "when trying to add to another another's user filters" do + let(:filter_id) { other_filter.id } + + it 'returns http not found' do + expect(response).to have_http_status(404) + end + end + end + + describe 'GET #show' do + let(:scopes) { 'read:filters' } + let(:keyword) { Fabricate(:custom_filter_keyword, keyword: 'foo', whole_word: false, custom_filter: filter) } + + before do + get :show, params: { id: keyword.id } + end + + it 'returns http success' do + expect(response).to have_http_status(200) + end + + it 'returns expected data' do + json = body_as_json + expect(json[:keyword]).to eq 'foo' + expect(json[:whole_word]).to eq false + end + + context "when trying to access another user's filter keyword" do + let(:keyword) { Fabricate(:custom_filter_keyword, custom_filter: other_filter) } + + it 'returns http not found' do + expect(response).to have_http_status(404) + end + end + end + + describe 'PUT #update' do + let(:scopes) { 'write:filters' } + let(:keyword) { Fabricate(:custom_filter_keyword, custom_filter: filter) } + + before do + get :update, params: { id: keyword.id, keyword: 'updated' } + end + + it 'returns http success' do + expect(response).to have_http_status(200) + end + + it 'updates the keyword' do + expect(keyword.reload.keyword).to eq 'updated' + end + + context "when trying to update another user's filter keyword" do + let(:keyword) { Fabricate(:custom_filter_keyword, custom_filter: other_filter) } + + it 'returns http not found' do + expect(response).to have_http_status(404) + end + end + end + + describe 'DELETE #destroy' do + let(:scopes) { 'write:filters' } + let(:keyword) { Fabricate(:custom_filter_keyword, custom_filter: filter) } + + before do + delete :destroy, params: { id: keyword.id } + end + + it 'returns http success' do + expect(response).to have_http_status(200) + end + + it 'removes the filter' do + expect { keyword.reload }.to raise_error ActiveRecord::RecordNotFound + end + + context "when trying to update another user's filter keyword" do + let(:keyword) { Fabricate(:custom_filter_keyword, custom_filter: other_filter) } + + it 'returns http not found' do + expect(response).to have_http_status(404) + end + end + end +end diff --git a/spec/controllers/api/v1/filters_controller_spec.rb b/spec/controllers/api/v1/filters_controller_spec.rb index 5948809e3f1..af1951f0ba6 100644 --- a/spec/controllers/api/v1/filters_controller_spec.rb +++ b/spec/controllers/api/v1/filters_controller_spec.rb @@ -34,7 +34,7 @@ RSpec.describe Api::V1::FiltersController, type: :controller do it 'creates a filter' do filter = user.account.custom_filters.first expect(filter).to_not be_nil - expect(filter.phrase).to eq 'magic' + expect(filter.keywords.pluck(:keyword)).to eq ['magic'] expect(filter.context).to eq %w(home) expect(filter.irreversible?).to be true expect(filter.expires_at).to be_nil @@ -42,21 +42,23 @@ RSpec.describe Api::V1::FiltersController, type: :controller do end describe 'GET #show' do - let(:scopes) { 'read:filters' } - let(:filter) { Fabricate(:custom_filter, account: user.account) } + let(:scopes) { 'read:filters' } + let(:filter) { Fabricate(:custom_filter, account: user.account) } + let(:keyword) { Fabricate(:custom_filter_keyword, custom_filter: filter) } it 'returns http success' do - get :show, params: { id: filter.id } + get :show, params: { id: keyword.id } expect(response).to have_http_status(200) end end describe 'PUT #update' do - let(:scopes) { 'write:filters' } - let(:filter) { Fabricate(:custom_filter, account: user.account) } + let(:scopes) { 'write:filters' } + let(:filter) { Fabricate(:custom_filter, account: user.account) } + let(:keyword) { Fabricate(:custom_filter_keyword, custom_filter: filter) } before do - put :update, params: { id: filter.id, phrase: 'updated' } + put :update, params: { id: keyword.id, phrase: 'updated' } end it 'returns http success' do @@ -64,16 +66,17 @@ RSpec.describe Api::V1::FiltersController, type: :controller do end it 'updates the filter' do - expect(filter.reload.phrase).to eq 'updated' + expect(keyword.reload.phrase).to eq 'updated' end end describe 'DELETE #destroy' do - let(:scopes) { 'write:filters' } - let(:filter) { Fabricate(:custom_filter, account: user.account) } + let(:scopes) { 'write:filters' } + let(:filter) { Fabricate(:custom_filter, account: user.account) } + let(:keyword) { Fabricate(:custom_filter_keyword, custom_filter: filter) } before do - delete :destroy, params: { id: filter.id } + delete :destroy, params: { id: keyword.id } end it 'returns http success' do @@ -81,7 +84,7 @@ RSpec.describe Api::V1::FiltersController, type: :controller do end it 'removes the filter' do - expect { filter.reload }.to raise_error ActiveRecord::RecordNotFound + expect { keyword.reload }.to raise_error ActiveRecord::RecordNotFound end end end diff --git a/spec/controllers/api/v1/statuses_controller_spec.rb b/spec/controllers/api/v1/statuses_controller_spec.rb index 2eb30af74be..4d104a198db 100644 --- a/spec/controllers/api/v1/statuses_controller_spec.rb +++ b/spec/controllers/api/v1/statuses_controller_spec.rb @@ -20,6 +20,58 @@ RSpec.describe Api::V1::StatusesController, type: :controller do get :show, params: { id: status.id } expect(response).to have_http_status(200) end + + context 'when post includes filtered terms' do + let(:status) { Fabricate(:status, text: 'this toot is about that banned word') } + + before do + user.account.custom_filters.create!(phrase: 'filter1', context: %w(home), action: :hide, keywords_attributes: [{ keyword: 'banned' }, { keyword: 'irrelevant' }]) + end + + it 'returns http success' do + get :show, params: { id: status.id } + expect(response).to have_http_status(200) + end + + it 'returns filter information' do + get :show, params: { id: status.id } + json = body_as_json + expect(json[:filtered][0]).to include({ + filter: a_hash_including({ + id: user.account.custom_filters.first.id.to_s, + title: 'filter1', + filter_action: 'hide', + }), + keyword_matches: ['banned'], + }) + end + end + + context 'when reblog includes filtered terms' do + let(:status) { Fabricate(:status, reblog: Fabricate(:status, text: 'this toot is about that banned word')) } + + before do + user.account.custom_filters.create!(phrase: 'filter1', context: %w(home), action: :hide, keywords_attributes: [{ keyword: 'banned' }, { keyword: 'irrelevant' }]) + end + + it 'returns http success' do + get :show, params: { id: status.id } + expect(response).to have_http_status(200) + end + + it 'returns filter information' do + get :show, params: { id: status.id } + json = body_as_json + expect(json[:reblog][:filtered][0]).to include({ + filter: a_hash_including({ + id: user.account.custom_filters.first.id.to_s, + title: 'filter1', + filter_action: 'hide', + }), + keyword_matches: ['banned'], + }) + end + end end describe 'GET #context' do diff --git a/spec/controllers/api/v2/filters_controller_spec.rb b/spec/controllers/api/v2/filters_controller_spec.rb new file mode 100644 index 00000000000..cc0070d577e --- /dev/null +++ b/spec/controllers/api/v2/filters_controller_spec.rb @@ -0,0 +1,121 @@ +require 'rails_helper' + +RSpec.describe Api::V2::FiltersController, type: :controller do + render_views + + let(:user) { Fabricate(:user) } + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: scopes) } + + before do + allow(controller).to receive(:doorkeeper_token) { token } + end + + describe 'GET #index' do + let(:scopes) { 'read:filters' } + let!(:filter) { Fabricate(:custom_filter, account: user.account) } + + it 'returns http success' do + get :index + expect(response).to have_http_status(200) + end + end + + describe 'POST #create' do + let(:scopes) { 'write:filters' } + + before do + post :create, params: { title: 'magic', context: %w(home), filter_action: 'hide', keywords_attributes: [keyword: 'magic'] } + end + + it 'returns http success' do + expect(response).to have_http_status(200) + end + + it 'returns a filter with keywords' do + json = body_as_json + expect(json[:title]).to eq 'magic' + expect(json[:filter_action]).to eq 'hide' + expect(json[:context]).to eq ['home'] + expect(json[:keywords].map { |keyword| keyword.slice(:keyword, :whole_word) }).to eq [{ keyword: 'magic', whole_word: true }] + end + + it 'creates a filter' do + filter = user.account.custom_filters.first + expect(filter).to_not be_nil + expect(filter.keywords.pluck(:keyword)).to eq ['magic'] + expect(filter.context).to eq %w(home) + expect(filter.irreversible?).to be true + expect(filter.expires_at).to be_nil + end + end + + describe 'GET #show' do + let(:scopes) { 'read:filters' } + let(:filter) { Fabricate(:custom_filter, account: user.account) } + + it 'returns http success' do + get :show, params: { id: filter.id } + expect(response).to have_http_status(200) + end + end + + describe 'PUT #update' do + let(:scopes) { 'write:filters' } + let!(:filter) { Fabricate(:custom_filter, account: user.account) } + let!(:keyword) { Fabricate(:custom_filter_keyword, custom_filter: filter) } + + context 'updating filter parameters' do + before do + put :update, params: { id: filter.id, title: 'updated', context: %w(home public) } + end + + it 'returns http success' do + expect(response).to have_http_status(200) + end + + it 'updates the filter title' do + expect(filter.reload.title).to eq 'updated' + end + + it 'updates the filter context' do + expect(filter.reload.context).to eq %w(home public) + end + end + + context 'updating keywords in bulk' do + before do + allow(redis).to receive_messages(publish: nil) + put :update, params: { id: filter.id, keywords_attributes: [{ id: keyword.id, keyword: 'updated' }] } + end + + it 'returns http success' do + expect(response).to have_http_status(200) + end + + it 'updates the keyword' do + expect(keyword.reload.keyword).to eq 'updated' + end + + it 'sends exactly one filters_changed event' do + expect(redis).to have_received(:publish).with("timeline:#{user.account.id}", Oj.dump(event: :filters_changed)).once + end + end + end + + describe 'DELETE #destroy' do + let(:scopes) { 'write:filters' } + let(:filter) { Fabricate(:custom_filter, account: user.account) } + + before do + delete :destroy, params: { id: filter.id } + end + + it 'returns http success' do + expect(response).to have_http_status(200) + end + + it 'removes the filter' do + expect { filter.reload }.to raise_error ActiveRecord::RecordNotFound + end + end +end diff --git a/spec/fabricators/custom_filter_keyword_fabricator.rb b/spec/fabricators/custom_filter_keyword_fabricator.rb new file mode 100644 index 00000000000..0f101dcd1ab --- /dev/null +++ b/spec/fabricators/custom_filter_keyword_fabricator.rb @@ -0,0 +1,4 @@ +Fabricator(:custom_filter_keyword) do + custom_filter + keyword 'discourse' +end diff --git a/spec/lib/feed_manager_spec.rb b/spec/lib/feed_manager_spec.rb index 3ba8aaa9fa9..48c57b86e11 100644 --- a/spec/lib/feed_manager_spec.rb +++ b/spec/lib/feed_manager_spec.rb @@ -127,38 +127,6 @@ RSpec.describe FeedManager do reblog = Fabricate(:status, reblog: status, account: jeff) expect(FeedManager.instance.filter?(:home, reblog, alice)).to be true end - - context 'for irreversibly muted phrases' do - it 'considers word boundaries when matching' do - alice.custom_filters.create!(phrase: 'bob', context: %w(home), irreversible: true) - alice.follow!(jeff) - status = Fabricate(:status, text: 'bobcats', account: jeff) - expect(FeedManager.instance.filter?(:home, status, alice)).to be_falsy - end - - it 'returns true if phrase is contained' do - alice.custom_filters.create!(phrase: 'farts', context: %w(home public), irreversible: true) - alice.custom_filters.create!(phrase: 'pop tarts', context: %w(home), irreversible: true) - alice.follow!(jeff) - status = Fabricate(:status, text: 'i sure like POP TARts', account: jeff) - expect(FeedManager.instance.filter?(:home, status, alice)).to be true - end - - it 'matches substrings if whole_word is false' do - alice.custom_filters.create!(phrase: 'take', context: %w(home), whole_word: false, irreversible: true) - alice.follow!(jeff) - status = Fabricate(:status, text: 'shiitake', account: jeff) - expect(FeedManager.instance.filter?(:home, status, alice)).to be true - end - - it 'returns true if phrase is contained in a poll option' do - alice.custom_filters.create!(phrase: 'farts', context: %w(home public), irreversible: true) - alice.custom_filters.create!(phrase: 'pop tarts', context: %w(home), irreversible: true) - alice.follow!(jeff) - status = Fabricate(:status, text: 'what do you prefer', poll: Fabricate(:poll, options: %w(farts POP TARts)), account: jeff) - expect(FeedManager.instance.filter?(:home, status, alice)).to be true - end - end end context 'for mentions feed' do diff --git a/spec/models/custom_filter_keyword_spec.rb b/spec/models/custom_filter_keyword_spec.rb new file mode 100644 index 00000000000..e15b9dad507 --- /dev/null +++ b/spec/models/custom_filter_keyword_spec.rb @@ -0,0 +1,4 @@ +require 'rails_helper' + +RSpec.describe CustomFilterKeyword, type: :model do +end diff --git a/spec/presenters/status_relationships_presenter_spec.rb b/spec/presenters/status_relationships_presenter_spec.rb index 03296bd179c..5cd4929a63c 100644 --- a/spec/presenters/status_relationships_presenter_spec.rb +++ b/spec/presenters/status_relationships_presenter_spec.rb @@ -5,7 +5,7 @@ require 'rails_helper' RSpec.describe StatusRelationshipsPresenter do describe '.initialize' do before do - allow(Status).to receive(:reblogs_map).with(status_ids, current_account_id).and_return(default_map) + allow(Status).to receive(:reblogs_map).with(match_array(status_ids), current_account_id).and_return(default_map) allow(Status).to receive(:favourites_map).with(status_ids, current_account_id).and_return(default_map) allow(Status).to receive(:bookmarks_map).with(status_ids, current_account_id).and_return(default_map) allow(Status).to receive(:mutes_map).with(anything, current_account_id).and_return(default_map) @@ -15,7 +15,7 @@ RSpec.describe StatusRelationshipsPresenter do let(:presenter) { StatusRelationshipsPresenter.new(statuses, current_account_id, **options) } let(:current_account_id) { Fabricate(:account).id } let(:statuses) { [Fabricate(:status)] } - let(:status_ids) { statuses.map(&:id) } + let(:status_ids) { statuses.map(&:id) + statuses.map(&:reblog_of_id).compact } let(:default_map) { { 1 => true } } context 'options are not set' do @@ -69,5 +69,30 @@ RSpec.describe StatusRelationshipsPresenter do expect(presenter.pins_map).to eq default_map.merge(options[:pins_map]) end end + + context 'when post includes filtered terms' do + let(:statuses) { [Fabricate(:status, text: 'this toot is about that banned word'), Fabricate(:status, reblog: Fabricate(:status, text: 'this toot is about an irrelevant word'))] } + let(:options) { {} } + + before do + Account.find(current_account_id).custom_filters.create!(phrase: 'filter1', context: %w(home), action: :hide, keywords_attributes: [{ keyword: 'banned' }, { keyword: 'irrelevant' }]) + end + + it 'sets @filters_map to filter top-level status' do + matched_filters = presenter.filters_map[statuses[0].id] + expect(matched_filters.size).to eq 1 + + expect(matched_filters[0].filter.title).to eq 'filter1' + expect(matched_filters[0].keyword_matches).to eq ['banned'] + end + + it 'sets @filters_map to filter reblogged status' do + matched_filters = presenter.filters_map[statuses[1].reblog_of_id] + expect(matched_filters.size).to eq 1 + + expect(matched_filters[0].filter.title).to eq 'filter1' + expect(matched_filters[0].keyword_matches).to eq ['irrelevant'] + end + end end end diff --git a/streaming/index.js b/streaming/index.js index 6935c47645c..792ec5a445e 100644 --- a/streaming/index.js +++ b/streaming/index.js @@ -12,6 +12,7 @@ const url = require('url'); const uuid = require('uuid'); const fs = require('fs'); const WebSocket = require('ws'); +const { JSDOM } = require('jsdom'); const env = process.env.NODE_ENV || 'development'; const alwaysRequireAuth = process.env.LIMITED_FEDERATION_MODE === 'true' || process.env.WHITELIST_MODE === 'true' || process.env.AUTHORIZED_FETCH === 'true'; @@ -503,6 +504,9 @@ const startWorker = async (workerId) => { if (event === 'kill') { log.verbose(req.requestId, `Closing connection for ${req.accountId} due to expired access token`); eventHandlers.onKill(); + } else if (event === 'filters_changed') { + log.verbose(req.requestId, `Invalidating filters cache for ${req.accountId}`); + req.cachedFilters = null; } }; }; @@ -512,7 +516,8 @@ const startWorker = async (workerId) => { * @param {any} res */ const subscribeHttpToSystemChannel = (req, res) => { - const systemChannelId = `timeline:access_token:${req.accessTokenId}`; + const accessTokenChannelId = `timeline:access_token:${req.accessTokenId}`; + const systemChannelId = `timeline:system:${req.accountId}`; const listener = createSystemMessageListener(req, { @@ -523,9 +528,11 @@ const startWorker = async (workerId) => { }); res.on('close', () => { + unsubscribe(`${redisPrefix}${accessTokenChannelId}`, listener); unsubscribe(`${redisPrefix}${systemChannelId}`, listener); }); + subscribe(`${redisPrefix}${accessTokenChannelId}`, listener); subscribe(`${redisPrefix}${systemChannelId}`, listener); }; @@ -674,17 +681,84 @@ const startWorker = async (workerId) => { queries.push(client.query('SELECT 1 FROM account_domain_blocks WHERE account_id = $1 AND domain = $2', [req.accountId, accountDomain])); } + if (!unpackedPayload.filter_results && !req.cachedFilters) { + queries.push(client.query('SELECT filter.id AS id, filter.phrase AS title, filter.context AS context, filter.expires_at AS expires_at, filter.action AS filter_action, keyword.keyword AS keyword, keyword.whole_word AS whole_word FROM custom_filter_keywords keyword JOIN custom_filters filter ON keyword.custom_filter_id = filter.id WHERE filter.account_id = $1 AND filter.expires_at IS NULL OR filter.expires_at > NOW()', [req.accountId])); + } + Promise.all(queries).then(values => { done(); - if (values[0].rows.length > 0 || (values.length > 1 && values[1].rows.length > 0)) { + if (values[0].rows.length > 0 || (accountDomain && values[1].rows.length > 0)) { return; } + if (!unpackedPayload.filter_results && !req.cachedFilters) { + const filterRows = values[accountDomain ? 2 : 1].rows; + + req.cachedFilters = filterRows.reduce((cache, row) => { + if (cache[row.id]) { + cache[row.id].keywords.push([row.keyword, row.whole_word]); + } else { + cache[row.id] = { + keywords: [[row.keyword, row.whole_word]], + expires_at: row.expires_at, + repr: { + id: row.id, + title: row.title, + context: row.context, + expires_at: row.expires_at, + filter_action: row.filter_action, + }, + }; + } + + return cache; + }, {}); + + Object.keys(req.cachedFilters).forEach((key) => { + req.cachedFilters[key].regexp = new RegExp(req.cachedFilters[key].keywords.map(([keyword, whole_word]) => { + let expr = keyword.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');; + + if (whole_word) { + if (/^[\w]/.test(expr)) { + expr = `\\b${expr}`; + } + + if (/[\w]$/.test(expr)) { + expr = `${expr}\\b`; + } + } + + return expr; + }).join('|'), 'i'); + }); + } + + // Check filters + if (req.cachedFilters && !unpackedPayload.filter_results) { + const status = unpackedPayload; + const searchContent = ([status.spoiler_text || '', status.content].concat((status.poll && status.poll.options) ? status.poll.options.map(option => option.title) : [])).concat(status.media_attachments.map(att => att.description)).join('\n\n').replace(//g, '\n').replace(/<\/p>

/g, '\n\n'); + const searchIndex = JSDOM.fragment(searchContent).textContent; + + const now = new Date(); + payload.filter_results = []; + Object.values(req.cachedFilters).forEach((cachedFilter) => { + if ((cachedFilter.expires_at === null || cachedFilter.expires_at > now)) { + const keyword_matches = searchIndex.match(cachedFilter.regexp); + if (keyword_matches) { + payload.filter_results.push({ + filter: cachedFilter.repr, + keyword_matches, + }); + } + } + }); + } + transmit(); }).catch(err => { - done(); log.error(err); + done(); }); }); }; @@ -1009,7 +1083,8 @@ const startWorker = async (workerId) => { * @param {WebSocketSession} session */ const subscribeWebsocketToSystemChannel = ({ socket, request, subscriptions }) => { - const systemChannelId = `timeline:access_token:${request.accessTokenId}`; + const accessTokenChannelId = `timeline:access_token:${request.accessTokenId}`; + const systemChannelId = `timeline:system:${request.accountId}`; const listener = createSystemMessageListener(request, { @@ -1019,8 +1094,15 @@ const startWorker = async (workerId) => { }); + subscribe(`${redisPrefix}${accessTokenChannelId}`, listener); subscribe(`${redisPrefix}${systemChannelId}`, listener); + subscriptions[accessTokenChannelId] = { + listener, + stopHeartbeat: () => { + }, + }; + subscriptions[systemChannelId] = { listener, stopHeartbeat: () => { diff --git a/yarn.lock b/yarn.lock index 42d877453f9..7901b9fd595 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1486,7 +1486,7 @@ "@types/yargs" "^16.0.0" chalk "^4.0.0" -"@jest/types@^28.1.0", "@jest/types@^28.1.1": +"@jest/types@^28.1.1": version "28.1.1" resolved "https://registry.yarnpkg.com/@jest/types/-/types-28.1.1.tgz#d059bbc80e6da6eda9f081f293299348bd78ee0b" integrity sha512-vRXVqSg1VhDnB8bWcmvLzmg0Bt9CRKVgHPXqYwvWMX3TvAjeO+nRuK6+VdTKCtWOvYlmkF/HqNAL/z+N3B53Kw== @@ -2153,7 +2153,7 @@ acorn@^8.0.4: resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.3.0.tgz#1193f9b96c4e8232f00b11a9edff81b2c8b98b88" integrity sha512-tqPKHZ5CaBJw0Xmy0ZZvLs1qTV+BNFSyvn77ASXkpBNfIRk8ev26fKrD9iLGwGA9zedPao52GSHzq8lyZG0NUw== -acorn@^8.5.0: +acorn@^8.5.0, acorn@^8.7.1: version "8.7.1" resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.7.1.tgz#0197122c843d1bf6d0a5e83220a788f278f63c30" integrity sha512-Xx54uLJQZ19lKygFXOWsscKUbsBZW0CPykPhVQdhIeIwrbPmJzqeASDInc8nKBnp/JT6igTs82qPXz069H8I/A== @@ -3304,6 +3304,11 @@ coa@^2.0.2: chalk "^2.4.1" q "^1.1.2" +cocoon-js-vanilla@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/cocoon-js-vanilla/-/cocoon-js-vanilla-1.2.0.tgz#595348499d315d3b5828dd77a20974756cf59321" + integrity sha512-qLomIVL0Krfc983WLgaYPPktMjMtBN+F/CV15NPVDc9U9BCe2OL5WyAIYkPrVhDRphoYBmHCdIlZkq+vSBI4xg== + collect-v8-coverage@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/collect-v8-coverage/-/collect-v8-coverage-1.0.1.tgz#cc2c8e94fc18bbdffe64d6534570c8a673b27f59" @@ -3892,7 +3897,7 @@ damerau-levenshtein@^1.0.7: resolved "https://registry.yarnpkg.com/damerau-levenshtein/-/damerau-levenshtein-1.0.7.tgz#64368003512a1a6992593741a09a9d31a836f55d" integrity sha512-VvdQIPGdWP0SqFXghj79Wf/5LArmreyMsGLa6FG6iC4t3j7j5s71TrwWmT/4akbDQIqjfACkLZmjXhA7g2oUZw== -data-urls@^3.0.1: +data-urls@^3.0.1, data-urls@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/data-urls/-/data-urls-3.0.2.tgz#9cf24a477ae22bcef5cd5f6f0bfbc1d2d3be9143" integrity sha512-Jy/tj3ldjZJo63sVAvg6LHt2mHvl4V6AgRAmNDtLdm7faqtsx+aJG42rsyCo9JCoRVKwPFzKlIPx3DIibwSIaQ== @@ -4338,6 +4343,11 @@ entities@^2.0.0: resolved "https://registry.yarnpkg.com/entities/-/entities-2.0.3.tgz#5c487e5742ab93c15abb5da22759b8590ec03b7f" integrity sha512-MyoZ0jgnLvB2X3Lg5HqpFmn1kybDiIfEQmKzTb5apr51Rb+T3KdmMiqa70T+bhGnyv7bQ6WMj2QMHpGMmlrUYQ== +entities@^4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/entities/-/entities-4.3.0.tgz#62915f08d67353bb4eb67e3d62641a4059aec656" + integrity sha512-/iP1rZrSEJ0DTlPiX+jbzlA3eVkY/e8L8SozroF395fIqE3TYF/Nz7YOMAawta+vLmyJ/hkGNNPcSbMADCCXbg== + errno@^0.1.3, errno@~0.1.7: version "0.1.7" resolved "https://registry.yarnpkg.com/errno/-/errno-0.1.7.tgz#4684d71779ad39af177e3f007996f7c67c852618" @@ -5700,7 +5710,7 @@ https-browserify@^1.0.0: resolved "https://registry.yarnpkg.com/https-browserify/-/https-browserify-1.0.0.tgz#ec06c10e0a34c0f2faf199f7fd7fc78fffd03c73" integrity sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM= -https-proxy-agent@^5.0.0: +https-proxy-agent@^5.0.0, https-proxy-agent@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz#c59ef224a04fe8b754f3db0063a25ea30d0005d6" integrity sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA== @@ -6728,7 +6738,7 @@ jest-snapshot@^28.1.1: pretty-format "^28.1.1" semver "^7.3.5" -jest-util@^28.1.0, jest-util@^28.1.1: +jest-util@^28.1.1: version "28.1.1" resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-28.1.1.tgz#ff39e436a1aca397c0ab998db5a51ae2b7080d05" integrity sha512-FktOu7ca1DZSyhPAxgxB6hfh2+9zMoJ7aEQA759Z6p45NuO8mWcqujH+UdHlCm/V6JTWwDztM2ITCzU1ijJAfw== @@ -6852,6 +6862,39 @@ jsdom@^19.0.0: ws "^8.2.3" xml-name-validator "^4.0.0" +jsdom@^20.0.0: + version "20.0.0" + resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-20.0.0.tgz#882825ac9cc5e5bbee704ba16143e1fa78361ebf" + integrity sha512-x4a6CKCgx00uCmP+QakBDFXwjAJ69IkkIWHmtmjd3wvXPcdOS44hfX2vqkOQrVrq8l9DhNNADZRXaCEWvgXtVA== + dependencies: + abab "^2.0.6" + acorn "^8.7.1" + acorn-globals "^6.0.0" + cssom "^0.5.0" + cssstyle "^2.3.0" + data-urls "^3.0.2" + decimal.js "^10.3.1" + domexception "^4.0.0" + escodegen "^2.0.0" + form-data "^4.0.0" + html-encoding-sniffer "^3.0.0" + http-proxy-agent "^5.0.0" + https-proxy-agent "^5.0.1" + is-potential-custom-element-name "^1.0.1" + nwsapi "^2.2.0" + parse5 "^7.0.0" + saxes "^6.0.0" + symbol-tree "^3.2.4" + tough-cookie "^4.0.0" + w3c-hr-time "^1.0.2" + w3c-xmlserializer "^3.0.0" + webidl-conversions "^7.0.0" + whatwg-encoding "^2.0.0" + whatwg-mimetype "^3.0.0" + whatwg-url "^11.0.0" + ws "^8.8.0" + xml-name-validator "^4.0.0" + jsesc@^2.5.1: version "2.5.2" resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" @@ -8141,6 +8184,13 @@ parse5@6.0.1: resolved "https://registry.yarnpkg.com/parse5/-/parse5-6.0.1.tgz#e1a1c085c569b3dc08321184f19a39cc27f7c30b" integrity sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw== +parse5@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/parse5/-/parse5-7.0.0.tgz#51f74a5257f5fcc536389e8c2d0b3802e1bfa91a" + integrity sha512-y/t8IXSPWTuRZqXc0ajH/UwDj4mnqLEbSttNbThcFhGrZuOyoyvNBO85PBp2jQa55wY9d07PBNjsK8ZP3K5U6g== + dependencies: + entities "^4.3.0" + parseurl@~1.3.2, parseurl@~1.3.3: version "1.3.3" resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4" @@ -9771,6 +9821,13 @@ saxes@^5.0.1: dependencies: xmlchars "^2.2.0" +saxes@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/saxes/-/saxes-6.0.0.tgz#fe5b4a4768df4f14a201b1ba6a65c1f3d9988cc5" + integrity sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA== + dependencies: + xmlchars "^2.2.0" + scheduler@^0.19.1: version "0.19.1" resolved "https://registry.yarnpkg.com/scheduler/-/scheduler-0.19.1.tgz#4f3e2ed2c1a7d65681f4c854fa8c5a1ccb40f196"