diff --git a/Gemfile b/Gemfile index deef23af57..cc48302732 100644 --- a/Gemfile +++ b/Gemfile @@ -65,6 +65,7 @@ gem 'nsa', '~> 0.2' gem 'oj', '~> 3.8' gem 'ostatus2', '~> 2.0' gem 'ox', '~> 2.11' +gem 'parslet' gem 'posix-spawn', git: 'https://github.com/rtomayko/posix-spawn', ref: '58465d2e213991f8afb13b984854a49fcdcc980c' gem 'pundit', '~> 2.0' gem 'premailer-rails' @@ -92,7 +93,7 @@ gem 'tzinfo-data', '~> 1.2019' gem 'webpacker', '~> 4.0' gem 'webpush' -gem 'json-ld', '~> 3.0' +gem 'json-ld', git: 'https://github.com/ruby-rdf/json-ld.git', ref: '345b7a5733308af827e8491d284dbafa9128d7a2' gem 'json-ld-preloaded', '~> 3.0' gem 'rdf-normalize', '~> 0.3' diff --git a/Gemfile.lock b/Gemfile.lock index b091612953..d2e74f82e9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -5,6 +5,19 @@ GIT specs: posix-spawn (0.3.13) +GIT + remote: https://github.com/ruby-rdf/json-ld.git + revision: 345b7a5733308af827e8491d284dbafa9128d7a2 + ref: 345b7a5733308af827e8491d284dbafa9128d7a2 + specs: + json-ld (3.0.2) + htmlentities (~> 4.3) + json-canonicalization (~> 0.1) + link_header (~> 0.0, >= 0.0.8) + multi_json (~> 1.13) + rack (>= 1.6, < 3.0) + rdf (~> 3.0, >= 3.0.8) + GIT remote: https://github.com/tmm1/http_parser.rb revision: 54b17ba8c7d8d20a16dfc65d1775241833219cf2 @@ -299,10 +312,8 @@ GEM jaro_winkler (1.5.3) jmespath (1.4.0) json (2.2.0) - json-ld (3.0.2) - multi_json (~> 1.12) - rdf (>= 2.2.8, < 4.0) - json-ld-preloaded (3.0.2) + json-canonicalization (0.1.0) + json-ld-preloaded (3.0.3) json-ld (~> 3.0) multi_json (~> 1.12) rdf (~> 3.0) @@ -406,6 +417,7 @@ GEM parallel parser (2.6.3.0) ast (~> 2.4.0) + parslet (1.8.2) pastel (0.7.2) equatable (~> 0.5.0) tty-color (~> 0.4.0) @@ -480,7 +492,7 @@ GEM thor (>= 0.19.0, < 2.0) rainbow (3.0.0) rake (12.3.2) - rdf (3.0.9) + rdf (3.0.12) hamster (~> 3.0) link_header (~> 0.0, >= 0.0.8) rdf-normalize (0.3.3) @@ -703,7 +715,7 @@ DEPENDENCIES i18n-tasks (~> 0.9) idn-ruby iso-639 - json-ld (~> 3.0) + json-ld! json-ld-preloaded (~> 3.0) kaminari (~> 1.1) letter_opener (~> 1.7) @@ -728,6 +740,7 @@ DEPENDENCIES paperclip (~> 6.0) paperclip-av-transcoder (~> 0.6) parallel_tests (~> 2.29) + parslet pg (~> 1.1) pghero (~> 2.2) pkg-config (~> 1.3) diff --git a/app/controllers/invites_controller.rb b/app/controllers/invites_controller.rb index 52cddc4044..639002964e 100644 --- a/app/controllers/invites_controller.rb +++ b/app/controllers/invites_controller.rb @@ -44,7 +44,7 @@ class InvitesController < ApplicationController end def invites - Invite.where(user: current_user).order(id: :desc) + current_user.invites.order(id: :desc) end def resource_params diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb index 83a5b2462e..1c473efa3f 100644 --- a/app/helpers/jsonld_helper.rb +++ b/app/helpers/jsonld_helper.rb @@ -130,7 +130,7 @@ module JsonLdHelper end end - doc = JSON::LD::API::RemoteDocument.new(url, json) + doc = JSON::LD::API::RemoteDocument.new(json, documentUrl: url) block_given? ? yield(doc) : doc end diff --git a/app/javascript/mastodon/actions/compose.js b/app/javascript/mastodon/actions/compose.js index fbf97d3747..cd9955505f 100644 --- a/app/javascript/mastodon/actions/compose.js +++ b/app/javascript/mastodon/actions/compose.js @@ -11,7 +11,7 @@ import { showAlertForError } from './alerts'; import { showAlert } from './alerts'; import { defineMessages } from 'react-intl'; -let cancelFetchComposeSuggestionsAccounts; +let cancelFetchComposeSuggestionsAccounts, cancelFetchComposeSuggestionsTags; export const COMPOSE_CHANGE = 'COMPOSE_CHANGE'; export const COMPOSE_SUBMIT_REQUEST = 'COMPOSE_SUBMIT_REQUEST'; @@ -325,10 +325,12 @@ const fetchComposeSuggestionsAccounts = throttle((dispatch, getState, token) => if (cancelFetchComposeSuggestionsAccounts) { cancelFetchComposeSuggestionsAccounts(); } + api(getState).get('/api/v1/accounts/search', { cancelToken: new CancelToken(cancel => { cancelFetchComposeSuggestionsAccounts = cancel; }), + params: { q: token.slice(1), resolve: false, @@ -349,9 +351,30 @@ const fetchComposeSuggestionsEmojis = (dispatch, getState, token) => { dispatch(readyComposeSuggestionsEmojis(token, results)); }; -const fetchComposeSuggestionsTags = (dispatch, getState, token) => { - dispatch(updateSuggestionTags(token)); -}; +const fetchComposeSuggestionsTags = throttle((dispatch, getState, token) => { + if (cancelFetchComposeSuggestionsTags) { + cancelFetchComposeSuggestionsTags(); + } + + api(getState).get('/api/v2/search', { + cancelToken: new CancelToken(cancel => { + cancelFetchComposeSuggestionsTags = cancel; + }), + + params: { + type: 'hashtags', + q: token.slice(1), + resolve: false, + limit: 4, + }, + }).then(({ data }) => { + dispatch(readyComposeSuggestionsTags(token, data.hashtags)); + }).catch(error => { + if (!isCancel(error)) { + dispatch(showAlertForError(error)); + } + }); +}, 200, { leading: true, trailing: true }); export function fetchComposeSuggestions(token) { return (dispatch, getState) => { @@ -385,6 +408,12 @@ export function readyComposeSuggestionsAccounts(token, accounts) { }; }; +export const readyComposeSuggestionsTags = (token, tags) => ({ + type: COMPOSE_SUGGESTIONS_READY, + token, + tags, +}); + export function selectComposeSuggestion(position, token, suggestion, path) { return (dispatch, getState) => { let completion, startPosition; @@ -394,8 +423,8 @@ export function selectComposeSuggestion(position, token, suggestion, path) { startPosition = position - 1; dispatch(useEmoji(suggestion)); - } else if (suggestion[0] === '#') { - completion = suggestion; + } else if (typeof suggestion === 'object' && suggestion.name) { + completion = `#${suggestion.name}`; startPosition = position - 1; } else { completion = getState().getIn(['accounts', suggestion, 'acct']); diff --git a/app/javascript/mastodon/actions/domain_blocks.js b/app/javascript/mastodon/actions/domain_blocks.js index 0445a5e10c..34a33a6546 100644 --- a/app/javascript/mastodon/actions/domain_blocks.js +++ b/app/javascript/mastodon/actions/domain_blocks.js @@ -23,6 +23,7 @@ export function blockDomain(domain) { api(getState).post('/api/v1/domain_blocks', { domain }).then(() => { const at_domain = '@' + domain; const accounts = getState().get('accounts').filter(item => item.get('acct').endsWith(at_domain)).valueSeq().map(item => item.get('id')); + dispatch(blockDomainSuccess(domain, accounts)); }).catch(err => { dispatch(blockDomainFail(domain, err)); diff --git a/app/javascript/mastodon/actions/search.js b/app/javascript/mastodon/actions/search.js index 0974fdd15e..a178faeadd 100644 --- a/app/javascript/mastodon/actions/search.js +++ b/app/javascript/mastodon/actions/search.js @@ -10,6 +10,10 @@ export const SEARCH_FETCH_REQUEST = 'SEARCH_FETCH_REQUEST'; export const SEARCH_FETCH_SUCCESS = 'SEARCH_FETCH_SUCCESS'; export const SEARCH_FETCH_FAIL = 'SEARCH_FETCH_FAIL'; +export const SEARCH_EXPAND_REQUEST = 'SEARCH_EXPAND_REQUEST'; +export const SEARCH_EXPAND_SUCCESS = 'SEARCH_EXPAND_SUCCESS'; +export const SEARCH_EXPAND_FAIL = 'SEARCH_EXPAND_FAIL'; + export function changeSearch(value) { return { type: SEARCH_CHANGE, @@ -77,8 +81,50 @@ export function fetchSearchFail(error) { }; }; -export function showSearch() { - return { - type: SEARCH_SHOW, - }; +export const expandSearch = type => (dispatch, getState) => { + const value = getState().getIn(['search', 'value']); + const offset = getState().getIn(['search', 'results', type]).size; + + dispatch(expandSearchRequest()); + + api(getState).get('/api/v2/search', { + params: { + q: value, + type, + offset, + }, + }).then(({ data }) => { + if (data.accounts) { + dispatch(importFetchedAccounts(data.accounts)); + } + + if (data.statuses) { + dispatch(importFetchedStatuses(data.statuses)); + } + + dispatch(expandSearchSuccess(data, value, type)); + dispatch(fetchRelationships(data.accounts.map(item => item.id))); + }).catch(error => { + dispatch(expandSearchFail(error)); + }); }; + +export const expandSearchRequest = () => ({ + type: SEARCH_EXPAND_REQUEST, +}); + +export const expandSearchSuccess = (results, searchTerm, searchType) => ({ + type: SEARCH_EXPAND_SUCCESS, + results, + searchTerm, + searchType, +}); + +export const expandSearchFail = error => ({ + type: SEARCH_EXPAND_FAIL, + error, +}); + +export const showSearch = () => ({ + type: SEARCH_SHOW, +}); diff --git a/app/javascript/mastodon/components/autosuggest_hashtag.js b/app/javascript/mastodon/components/autosuggest_hashtag.js new file mode 100644 index 0000000000..eabb8b1785 --- /dev/null +++ b/app/javascript/mastodon/components/autosuggest_hashtag.js @@ -0,0 +1,28 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import { shortNumberFormat } from 'mastodon/utils/numbers'; +import { FormattedMessage } from 'react-intl'; + +export default class AutosuggestHashtag extends React.PureComponent { + + static propTypes = { + tag: PropTypes.shape({ + name: PropTypes.string.isRequired, + url: PropTypes.string, + history: PropTypes.array.isRequired, + }).isRequired, + }; + + render () { + const { tag } = this.props; + const weeklyUses = shortNumberFormat(tag.history.reduce((total, day) => total + (day.uses * 1), 0)); + + return ( +
+
#{tag.name}
+
+
+ ); + } + +} diff --git a/app/javascript/mastodon/components/autosuggest_input.js b/app/javascript/mastodon/components/autosuggest_input.js index c7d965b53a..0df8bf64e1 100644 --- a/app/javascript/mastodon/components/autosuggest_input.js +++ b/app/javascript/mastodon/components/autosuggest_input.js @@ -1,6 +1,7 @@ import React from 'react'; import AutosuggestAccountContainer from '../features/compose/containers/autosuggest_account_container'; import AutosuggestEmoji from './autosuggest_emoji'; +import AutosuggestHashtag from './autosuggest_hashtag'; import ImmutablePropTypes from 'react-immutable-proptypes'; import PropTypes from 'prop-types'; import { isRtl } from '../rtl'; @@ -167,12 +168,12 @@ export default class AutosuggestInput extends ImmutablePureComponent { const { selectedSuggestion } = this.state; let inner, key; - if (typeof suggestion === 'object') { + if (typeof suggestion === 'object' && suggestion.shortcode) { inner = ; key = suggestion.id; - } else if (suggestion[0] === '#') { - inner = suggestion; - key = suggestion; + } else if (typeof suggestion === 'object' && suggestion.name) { + inner = ; + key = suggestion.name; } else { inner = ; key = suggestion; diff --git a/app/javascript/mastodon/components/autosuggest_textarea.js b/app/javascript/mastodon/components/autosuggest_textarea.js index b070fe3e51..2bd06d28a9 100644 --- a/app/javascript/mastodon/components/autosuggest_textarea.js +++ b/app/javascript/mastodon/components/autosuggest_textarea.js @@ -1,6 +1,7 @@ import React from 'react'; import AutosuggestAccountContainer from '../features/compose/containers/autosuggest_account_container'; import AutosuggestEmoji from './autosuggest_emoji'; +import AutosuggestHashtag from './autosuggest_hashtag'; import ImmutablePropTypes from 'react-immutable-proptypes'; import PropTypes from 'prop-types'; import { isRtl } from '../rtl'; @@ -173,12 +174,12 @@ export default class AutosuggestTextarea extends ImmutablePureComponent { const { selectedSuggestion } = this.state; let inner, key; - if (typeof suggestion === 'object') { + if (typeof suggestion === 'object' && suggestion.shortcode) { inner = ; key = suggestion.id; - } else if (suggestion[0] === '#') { - inner = suggestion; - key = suggestion; + } else if (typeof suggestion === 'object' && suggestion.name) { + inner = ; + key = suggestion.name; } else { inner = ; key = suggestion; diff --git a/app/javascript/mastodon/components/status_content.js b/app/javascript/mastodon/components/status_content.js index 8a05415afd..549de95fc1 100644 --- a/app/javascript/mastodon/components/status_content.js +++ b/app/javascript/mastodon/components/status_content.js @@ -55,6 +55,7 @@ export default class StatusContent extends React.PureComponent { link.addEventListener('click', this.onHashtagClick.bind(this, link.text), false); } else { link.setAttribute('title', link.href); + link.classList.add('unhandled-link'); } link.setAttribute('target', '_blank'); @@ -233,46 +234,23 @@ export default class StatusContent extends React.PureComponent { ); } else if (this.props.onClick) { - const output = [ -
, - ]; + return ( +
+
- if (this.state.collapsed) { - output.push(readMoreButton); - } + {!!this.state.collapsed && readMoreButton} - if (status.get('poll')) { - output.push(); - } - - return output; + {!!status.get('poll') && } +
+ ); } else { - const output = [ -
, - ]; + return ( +
+
- if (status.get('poll')) { - output.push(); - } - - return output; + {!!status.get('poll') && } +
+ ); } } diff --git a/app/javascript/mastodon/features/compose/components/search_results.js b/app/javascript/mastodon/features/compose/components/search_results.js index 2f338dd246..4b4cdff743 100644 --- a/app/javascript/mastodon/features/compose/components/search_results.js +++ b/app/javascript/mastodon/features/compose/components/search_results.js @@ -8,6 +8,7 @@ import ImmutablePureComponent from 'react-immutable-pure-component'; import Hashtag from '../../../components/hashtag'; import Icon from 'mastodon/components/icon'; import { searchEnabled } from '../../../initial_state'; +import LoadMore from 'mastodon/components/load_more'; const messages = defineMessages({ dismissSuggestion: { id: 'suggestions.dismiss', defaultMessage: 'Dismiss suggestion' }, @@ -20,15 +21,24 @@ class SearchResults extends ImmutablePureComponent { results: ImmutablePropTypes.map.isRequired, suggestions: ImmutablePropTypes.list.isRequired, fetchSuggestions: PropTypes.func.isRequired, + expandSearch: PropTypes.func.isRequired, dismissSuggestion: PropTypes.func.isRequired, searchTerm: PropTypes.string, intl: PropTypes.object.isRequired, }; componentDidMount () { - this.props.fetchSuggestions(); + if (this.props.searchTerm === '') { + this.props.fetchSuggestions(); + } } + handleLoadMoreAccounts = () => this.props.expandSearch('accounts'); + + handleLoadMoreStatuses = () => this.props.expandSearch('statuses'); + + handleLoadMoreHashtags = () => this.props.expandSearch('hashtags'); + render () { const { intl, results, suggestions, dismissSuggestion, searchTerm } = this.props; @@ -65,6 +75,8 @@ class SearchResults extends ImmutablePureComponent {
{results.get('accounts').map(accountId => )} + + {results.get('accounts').size >= 5 && }
); } @@ -76,6 +88,8 @@ class SearchResults extends ImmutablePureComponent {
{results.get('statuses').map(statusId => )} + + {results.get('statuses').size >= 5 && }
); } else if(results.get('statuses') && results.get('statuses').size === 0 && !searchEnabled && !(searchTerm.startsWith('@') || searchTerm.startsWith('#') || searchTerm.includes(' '))) { @@ -97,6 +111,8 @@ class SearchResults extends ImmutablePureComponent {
{results.get('hashtags').map(hashtag => )} + + {results.get('hashtags').size >= 5 && }
); } diff --git a/app/javascript/mastodon/features/compose/containers/search_results_container.js b/app/javascript/mastodon/features/compose/containers/search_results_container.js index e4d5f34207..1f714ff834 100644 --- a/app/javascript/mastodon/features/compose/containers/search_results_container.js +++ b/app/javascript/mastodon/features/compose/containers/search_results_container.js @@ -1,6 +1,7 @@ import { connect } from 'react-redux'; import SearchResults from '../components/search_results'; -import { fetchSuggestions, dismissSuggestion } from '../../../actions/suggestions'; +import { fetchSuggestions, dismissSuggestion } from 'mastodon/actions/suggestions'; +import { expandSearch } from 'mastodon/actions/search'; const mapStateToProps = state => ({ results: state.getIn(['search', 'results']), @@ -10,6 +11,7 @@ const mapStateToProps = state => ({ const mapDispatchToProps = dispatch => ({ fetchSuggestions: () => dispatch(fetchSuggestions()), + expandSearch: type => dispatch(expandSearch(type)), dismissSuggestion: account => dispatch(dismissSuggestion(account.get('id'))), }); diff --git a/app/javascript/mastodon/reducers/compose.js b/app/javascript/mastodon/reducers/compose.js index fae7522b26..fc6d5f30f1 100644 --- a/app/javascript/mastodon/reducers/compose.js +++ b/app/javascript/mastodon/reducers/compose.js @@ -17,7 +17,6 @@ import { COMPOSE_SUGGESTIONS_CLEAR, COMPOSE_SUGGESTIONS_READY, COMPOSE_SUGGESTION_SELECT, - COMPOSE_SUGGESTION_TAGS_UPDATE, COMPOSE_TAG_HISTORY_UPDATE, COMPOSE_SENSITIVITY_CHANGE, COMPOSE_SPOILERNESS_CHANGE, @@ -144,15 +143,20 @@ const insertSuggestion = (state, position, token, completion, path) => { }); }; -const updateSuggestionTags = (state, token) => { - const prefix = token.slice(1); +const sortHashtagsByUse = (state, tags) => { + const personalHistory = state.get('tagHistory'); - return state.merge({ - suggestions: state.get('tagHistory') - .filter(tag => tag.toLowerCase().startsWith(prefix.toLowerCase())) - .slice(0, 4) - .map(tag => '#' + tag), - suggestion_token: token, + return tags.sort((a, b) => { + const usedA = personalHistory.includes(a.name); + const usedB = personalHistory.includes(b.name); + + if (usedA === usedB) { + return 0; + } else if (usedA && !usedB) { + return 1; + } else { + return -1; + } }); }; @@ -201,6 +205,16 @@ const expiresInFromExpiresAt = expires_at => { return [300, 1800, 3600, 21600, 86400, 259200, 604800].find(expires_in => expires_in >= delta) || 24 * 3600; }; +const normalizeSuggestions = (state, { accounts, emojis, tags }) => { + if (accounts) { + return accounts.map(item => item.id); + } else if (emojis) { + return emojis; + } else { + return sortHashtagsByUse(state, tags); + } +}; + export default function compose(state = initialState, action) { switch(action.type) { case STORE_HYDRATE: @@ -311,11 +325,9 @@ export default function compose(state = initialState, action) { case COMPOSE_SUGGESTIONS_CLEAR: return state.update('suggestions', ImmutableList(), list => list.clear()).set('suggestion_token', null); case COMPOSE_SUGGESTIONS_READY: - return state.set('suggestions', ImmutableList(action.accounts ? action.accounts.map(item => item.id) : action.emojis)).set('suggestion_token', action.token); + return state.set('suggestions', ImmutableList(normalizeSuggestions(state, action))).set('suggestion_token', action.token); case COMPOSE_SUGGESTION_SELECT: return insertSuggestion(state, action.position, action.token, action.completion, action.path); - case COMPOSE_SUGGESTION_TAGS_UPDATE: - return updateSuggestionTags(state, action.token); case COMPOSE_TAG_HISTORY_UPDATE: return state.set('tagHistory', fromJS(action.tags)); case TIMELINE_DELETE: diff --git a/app/javascript/mastodon/reducers/conversations.js b/app/javascript/mastodon/reducers/conversations.js index 9564bffcd2..3906582392 100644 --- a/app/javascript/mastodon/reducers/conversations.js +++ b/app/javascript/mastodon/reducers/conversations.js @@ -8,6 +8,8 @@ import { CONVERSATIONS_UPDATE, CONVERSATIONS_READ, } from '../actions/conversations'; +import { ACCOUNT_BLOCK_SUCCESS, ACCOUNT_MUTE_SUCCESS } from 'mastodon/actions/accounts'; +import { DOMAIN_BLOCK_SUCCESS } from 'mastodon/actions/domain_blocks'; import compareId from '../compare_id'; const initialState = ImmutableMap({ @@ -74,6 +76,10 @@ const expandNormalizedConversations = (state, conversations, next, isLoadingRece }); }; +const filterConversations = (state, accountIds) => { + return state.update('items', list => list.filterNot(item => item.get('accounts').some(accountId => accountIds.includes(accountId)))); +}; + export default function conversations(state = initialState, action) { switch (action.type) { case CONVERSATIONS_FETCH_REQUEST: @@ -96,6 +102,11 @@ export default function conversations(state = initialState, action) { return item; })); + case ACCOUNT_BLOCK_SUCCESS: + case ACCOUNT_MUTE_SUCCESS: + return filterConversations(state, [action.relationship.id]); + case DOMAIN_BLOCK_SUCCESS: + return filterConversations(state, action.accounts); default: return state; } diff --git a/app/javascript/mastodon/reducers/notifications.js b/app/javascript/mastodon/reducers/notifications.js index e94a4946b8..049c70cb41 100644 --- a/app/javascript/mastodon/reducers/notifications.js +++ b/app/javascript/mastodon/reducers/notifications.js @@ -12,6 +12,7 @@ import { ACCOUNT_BLOCK_SUCCESS, ACCOUNT_MUTE_SUCCESS, } from '../actions/accounts'; +import { DOMAIN_BLOCK_SUCCESS } from 'mastodon/actions/domain_blocks'; import { TIMELINE_DELETE, TIMELINE_DISCONNECT } from '../actions/timelines'; import { Map as ImmutableMap, List as ImmutableList } from 'immutable'; import compareId from '../compare_id'; @@ -83,8 +84,8 @@ const expandNormalizedNotifications = (state, notifications, next, usePendingIte }); }; -const filterNotifications = (state, relationship) => { - const helper = list => list.filterNot(item => item !== null && item.get('account') === relationship.id); +const filterNotifications = (state, accountIds) => { + const helper = list => list.filterNot(item => item !== null && accountIds.includes(item.get('account'))); return state.update('items', helper).update('pendingItems', helper); }; @@ -118,9 +119,11 @@ export default function notifications(state = initialState, action) { case NOTIFICATIONS_EXPAND_SUCCESS: return expandNormalizedNotifications(state, action.notifications, action.next, action.usePendingItems); case ACCOUNT_BLOCK_SUCCESS: - return filterNotifications(state, action.relationship); + return filterNotifications(state, [action.relationship.id]); case ACCOUNT_MUTE_SUCCESS: - return action.relationship.muting_notifications ? filterNotifications(state, action.relationship) : state; + return action.relationship.muting_notifications ? filterNotifications(state, [action.relationship.id]) : state; + case DOMAIN_BLOCK_SUCCESS: + return filterNotifications(state, action.accounts); case NOTIFICATIONS_CLEAR: return state.set('items', ImmutableList()).set('pendingItems', ImmutableList()).set('hasMore', false); case TIMELINE_DELETE: diff --git a/app/javascript/mastodon/reducers/search.js b/app/javascript/mastodon/reducers/search.js index 77b7f588c5..f437ff2171 100644 --- a/app/javascript/mastodon/reducers/search.js +++ b/app/javascript/mastodon/reducers/search.js @@ -3,6 +3,7 @@ import { SEARCH_CLEAR, SEARCH_FETCH_SUCCESS, SEARCH_SHOW, + SEARCH_EXPAND_SUCCESS, } from '../actions/search'; import { COMPOSE_MENTION, @@ -42,6 +43,8 @@ export default function search(state = initialState, action) { statuses: ImmutableList(action.results.statuses.map(item => item.id)), hashtags: fromJS(action.results.hashtags), })).set('submitted', true).set('searchTerm', action.searchTerm); + case SEARCH_EXPAND_SUCCESS: + return state.updateIn(['results', action.searchType], list => list.concat(action.results[action.searchType].map(item => item.id))); default: return state; } diff --git a/app/javascript/mastodon/reducers/suggestions.js b/app/javascript/mastodon/reducers/suggestions.js index 9f4b89d586..834be728f1 100644 --- a/app/javascript/mastodon/reducers/suggestions.js +++ b/app/javascript/mastodon/reducers/suggestions.js @@ -4,6 +4,8 @@ import { SUGGESTIONS_FETCH_FAIL, SUGGESTIONS_DISMISS, } from '../actions/suggestions'; +import { ACCOUNT_BLOCK_SUCCESS, ACCOUNT_MUTE_SUCCESS } from 'mastodon/actions/accounts'; +import { DOMAIN_BLOCK_SUCCESS } from 'mastodon/actions/domain_blocks'; import { Map as ImmutableMap, List as ImmutableList, fromJS } from 'immutable'; const initialState = ImmutableMap({ @@ -24,6 +26,11 @@ export default function suggestionsReducer(state = initialState, action) { return state.set('isLoading', false); case SUGGESTIONS_DISMISS: return state.update('items', list => list.filterNot(id => id === action.id)); + case ACCOUNT_BLOCK_SUCCESS: + case ACCOUNT_MUTE_SUCCESS: + return state.update('items', list => list.filterNot(id => id === action.relationship.id)); + case DOMAIN_BLOCK_SUCCESS: + return state.update('items', list => list.filterNot(id => action.accounts.includes(id))); default: return state; } diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss index 4d0adc2293..1853cd2e5a 100644 --- a/app/javascript/styles/mastodon/components.scss +++ b/app/javascript/styles/mastodon/components.scss @@ -445,7 +445,8 @@ } .autosuggest-account, - .autosuggest-emoji { + .autosuggest-emoji, + .autosuggest-hashtag { display: flex; flex-direction: row; align-items: center; @@ -454,6 +455,14 @@ font-size: 14px; } + .autosuggest-hashtag { + justify-content: space-between; + + strong { + font-weight: 500; + } + } + .autosuggest-account-icon, .autosuggest-emoji img { display: block; @@ -753,6 +762,10 @@ } } + a.unhandled-link { + color: lighten($ui-highlight-color, 8%); + } + .status__content__spoiler-link { background: $action-button-color; @@ -1936,6 +1949,9 @@ a.account__display-name { background: lighten($ui-base-color, 8%); flex: 0 0 auto; overflow-y: auto; + position: sticky; + top: 0; + z-index: 3; } .tabs-bar__link { @@ -4006,8 +4022,9 @@ a.status-card.compact:hover { } .search-results__info { - padding: 10px; - color: $secondary-text-color; + padding: 20px; + color: $darker-text-color; + text-align: center; } .modal-root { diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 56c24680a7..000b77df5f 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -148,12 +148,9 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def process_hashtag(tag) return if tag['name'].blank? - hashtag = tag['name'].gsub(/\A#/, '').mb_chars.downcase - hashtag = Tag.where(name: hashtag).first_or_create!(name: hashtag) - - return if @tags.include?(hashtag) - - @tags << hashtag + Tag.find_or_create_by_names(tag['name']) do |hashtag| + @tags << hashtag unless @tags.include?(hashtag) + end rescue ActiveRecord::RecordInvalid nil end diff --git a/app/lib/search_query_parser.rb b/app/lib/search_query_parser.rb new file mode 100644 index 0000000000..405ad15b89 --- /dev/null +++ b/app/lib/search_query_parser.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class SearchQueryParser < Parslet::Parser + rule(:term) { match('[^\s":]').repeat(1).as(:term) } + rule(:quote) { str('"') } + rule(:colon) { str(':') } + rule(:space) { match('\s').repeat(1) } + rule(:operator) { (str('+') | str('-')).as(:operator) } + rule(:prefix) { (term >> colon).as(:prefix) } + rule(:phrase) { (quote >> (term >> space.maybe).repeat >> quote).as(:phrase) } + rule(:clause) { (prefix.maybe >> operator.maybe >> (phrase | term)).as(:clause) } + rule(:query) { (clause >> space.maybe).repeat.as(:query) } + root(:query) +end diff --git a/app/lib/search_query_transformer.rb b/app/lib/search_query_transformer.rb new file mode 100644 index 0000000000..2c4144790b --- /dev/null +++ b/app/lib/search_query_transformer.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +class SearchQueryTransformer < Parslet::Transform + class Query + attr_reader :should_clauses, :must_not_clauses, :must_clauses + + def initialize(clauses) + grouped = clauses.chunk(&:operator).to_h + @should_clauses = grouped.fetch(:should, []) + @must_not_clauses = grouped.fetch(:must_not, []) + @must_clauses = grouped.fetch(:must, []) + end + + def apply(search) + should_clauses.each { |clause| search = search.query.should(clause_to_query(clause)) } + must_clauses.each { |clause| search = search.query.must(clause_to_query(clause)) } + must_not_clauses.each { |clause| search = search.query.must_not(clause_to_query(clause)) } + search.query.minimum_should_match(1) + end + + private + + def clause_to_query(clause) + case clause + when TermClause + { multi_match: { type: 'most_fields', query: clause.term, fields: ['text', 'text.stemmed'] } } + when PhraseClause + { match_phrase: { text: { query: clause.phrase } } } + else + raise "Unexpected clause type: #{clause}" + end + end + end + + class Operator + class << self + def symbol(str) + case str + when '+' + :must + when '-' + :must_not + when nil + :should + else + raise "Unknown operator: #{str}" + end + end + end + end + + class TermClause + attr_reader :prefix, :operator, :term + + def initialize(prefix, operator, term) + @prefix = prefix + @operator = Operator.symbol(operator) + @term = term + end + end + + class PhraseClause + attr_reader :prefix, :operator, :phrase + + def initialize(prefix, operator, phrase) + @prefix = prefix + @operator = Operator.symbol(operator) + @phrase = phrase + end + end + + rule(clause: subtree(:clause)) do + prefix = clause[:prefix][:term].to_s if clause[:prefix] + operator = clause[:operator]&.to_s + + if clause[:term] + TermClause.new(prefix, operator, clause[:term].to_s) + elsif clause[:phrase] + PhraseClause.new(prefix, operator, clause[:phrase].map { |p| p[:term].to_s }.join(' ')) + else + raise "Unexpected clause type: #{clause}" + end + end + + rule(query: sequence(:clauses)) { Query.new(clauses) } +end diff --git a/app/models/invite.rb b/app/models/invite.rb index fe23224625..02ab8e0b21 100644 --- a/app/models/invite.rb +++ b/app/models/invite.rb @@ -17,7 +17,7 @@ class Invite < ApplicationRecord include Expireable - belongs_to :user + belongs_to :user, inverse_of: :invites has_many :users, inverse_of: :invite scope :available, -> { where(expires_at: nil).or(where('expires_at >= ?', Time.now.utc)) } @@ -25,7 +25,7 @@ class Invite < ApplicationRecord before_validation :set_code def valid_for_use? - (max_uses.nil? || uses < max_uses) && !expired? + (max_uses.nil? || uses < max_uses) && !expired? && !(user.nil? || user.disabled?) end private diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index 189d80e77a..be762889cf 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -113,7 +113,7 @@ class MediaAttachment < ApplicationRecord has_attached_file :file, styles: ->(f) { file_styles f }, processors: ->(f) { file_processors f }, - convert_options: { all: '-quality 90 -strip' } + convert_options: { all: '-quality 90 -strip +set modify-date +set create-date' } validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES + AUDIO_MIME_TYPES validates_attachment_size :file, less_than: IMAGE_LIMIT, unless: :larger_media_format? diff --git a/app/models/tag.rb b/app/models/tag.rb index b371d59c1b..972242064b 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -20,7 +20,7 @@ class Tag < ApplicationRecord HASHTAG_NAME_RE = '([[:word:]_][[:word:]_·]*[[:alpha:]_·][[:word:]_·]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)' HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i - validates :name, presence: true, uniqueness: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i } + validates :name, presence: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i } scope :discoverable, -> { joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).where(account_tag_stats: { hidden: false }).order(Arel.sql('account_tag_stats.accounts_count desc')) } scope :hidden, -> { where(account_tag_stats: { hidden: true }) } @@ -64,22 +64,48 @@ class Tag < ApplicationRecord end class << self - def search_for(term, limit = 5, offset = 0) - pattern = sanitize_sql_like(term.strip) + '%' + def find_or_create_by_names(name_or_names) + Array(name_or_names).map(&method(:normalize)).uniq.map do |normalized_name| + tag = matching_name(normalized_name).first || create(name: normalized_name) - Tag.where('lower(name) like lower(?)', pattern) + yield tag if block_given? + + tag + end + end + + def search_for(term, limit = 5, offset = 0) + pattern = sanitize_sql_like(normalize(term.strip)) + '%' + + Tag.where(arel_table[:name].lower.matches(pattern.downcase)) .order(:name) .limit(limit) .offset(offset) end def find_normalized(name) - find_by(name: name.mb_chars.downcase.to_s) + matching_name(name).first end def find_normalized!(name) find_normalized(name) || raise(ActiveRecord::RecordNotFound) end + + def matching_name(name_or_names) + names = Array(name_or_names).map { |name| normalize(name).downcase } + + if names.size == 1 + where(arel_table[:name].lower.eq(names.first)) + else + where(arel_table[:name].lower.in(names)) + end + end + + private + + def normalize(str) + str.gsub(/\A#/, '').mb_chars.to_s + end end private diff --git a/app/models/user.rb b/app/models/user.rb index 1548e1ea0f..2a7fffca5d 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -73,6 +73,7 @@ class User < ApplicationRecord has_many :applications, class_name: 'Doorkeeper::Application', as: :owner has_many :backups, inverse_of: :user + has_many :invites, inverse_of: :user has_one :invite_request, class_name: 'UserInviteRequest', inverse_of: :user, dependent: :destroy accepts_nested_attributes_for :invite_request, reject_if: ->(attributes) { attributes['text'].blank? } diff --git a/app/services/after_block_domain_from_account_service.rb b/app/services/after_block_domain_from_account_service.rb index a87c2e792c..f50bde261d 100644 --- a/app/services/after_block_domain_from_account_service.rb +++ b/app/services/after_block_domain_from_account_service.rb @@ -10,12 +10,24 @@ class AfterBlockDomainFromAccountService < BaseService @account = account @domain = domain + clear_notifications! + remove_follows! reject_existing_followers! reject_pending_follow_requests! end private + def remove_follows! + @account.active_relationships.where(account: Account.where(domain: @domain)).includes(:target_account).reorder(nil).find_each do |follow| + UnfollowService.new.call(@account, follow.target_account) + end + end + + def clear_notifications! + Notification.where(account: @account).where(from_account: Account.where(domain: @domain)).in_batches.delete_all + end + def reject_existing_followers! @account.passive_relationships.where(account: Account.where(domain: @domain)).includes(:account).reorder(nil).find_each do |follow| reject_follow!(follow) diff --git a/app/services/after_block_service.rb b/app/services/after_block_service.rb index 706db0d633..2a0e10a79a 100644 --- a/app/services/after_block_service.rb +++ b/app/services/after_block_service.rb @@ -2,43 +2,25 @@ class AfterBlockService < BaseService def call(account, target_account) - clear_home_feed(account, target_account) - clear_notifications(account, target_account) - clear_conversations(account, target_account) + @account = account + @target_account = target_account + + clear_home_feed! + clear_notifications! + clear_conversations! end private - def clear_home_feed(account, target_account) - FeedManager.instance.clear_from_timeline(account, target_account) + def clear_home_feed! + FeedManager.instance.clear_from_timeline(@account, @target_account) end - def clear_conversations(account, target_account) - AccountConversation.where(account: account) - .where('? = ANY(participant_account_ids)', target_account.id) - .in_batches - .destroy_all + def clear_conversations! + AccountConversation.where(account: @account).where('? = ANY(participant_account_ids)', @target_account.id).in_batches.destroy_all end - def clear_notifications(account, target_account) - Notification.where(account: account) - .joins(:follow) - .where(activity_type: 'Follow', follows: { account_id: target_account.id }) - .delete_all - - Notification.where(account: account) - .joins(mention: :status) - .where(activity_type: 'Mention', statuses: { account_id: target_account.id }) - .delete_all - - Notification.where(account: account) - .joins(:favourite) - .where(activity_type: 'Favourite', favourites: { account_id: target_account.id }) - .delete_all - - Notification.where(account: account) - .joins(:status) - .where(activity_type: 'Status', statuses: { account_id: target_account.id }) - .delete_all + def clear_notifications! + Notification.where(account: @account).where(from_account: @target_account).in_batches.delete_all end end diff --git a/app/services/follow_service.rb b/app/services/follow_service.rb index 8e118f5d34..101acdaf96 100644 --- a/app/services/follow_service.rb +++ b/app/services/follow_service.rb @@ -13,7 +13,7 @@ class FollowService < BaseService target_account = ResolveAccountService.new.call(target_account, skip_webfinger: true) raise ActiveRecord::RecordNotFound if target_account.nil? || target_account.id == source_account.id || target_account.suspended? - raise Mastodon::NotPermittedError if target_account.blocking?(source_account) || source_account.blocking?(target_account) || target_account.moved? || (!target_account.local? && target_account.ostatus?) + raise Mastodon::NotPermittedError if target_account.blocking?(source_account) || source_account.blocking?(target_account) || target_account.moved? || (!target_account.local? && target_account.ostatus?) || source_account.domain_blocking?(target_account.domain) if source_account.following?(target_account) # We're already following this account, but we'll call follow! again to diff --git a/app/services/hashtag_query_service.rb b/app/services/hashtag_query_service.rb index 5773d78c6e..282821710a 100644 --- a/app/services/hashtag_query_service.rb +++ b/app/services/hashtag_query_service.rb @@ -14,7 +14,7 @@ class HashtagQueryService < BaseService private - def tags_for(tags) - Tag.where(name: tags.map(&:downcase)) if tags.presence + def tags_for(names) + Tag.matching_name(names) if names.presence end end diff --git a/app/services/process_hashtags_service.rb b/app/services/process_hashtags_service.rb index b6974e598b..e8e139b058 100644 --- a/app/services/process_hashtags_service.rb +++ b/app/services/process_hashtags_service.rb @@ -5,9 +5,7 @@ class ProcessHashtagsService < BaseService tags = Extractor.extract_hashtags(status.text) if status.local? records = [] - tags.map { |str| str.mb_chars.downcase }.uniq(&:to_s).each do |name| - tag = Tag.where(name: name).first_or_create(name: name) - + Tag.find_or_create_by_names(tags) do |tag| status.tags << tag records << tag diff --git a/app/services/search_service.rb b/app/services/search_service.rb index e0da61dac5..769d1ac7a4 100644 --- a/app/services/search_service.rb +++ b/app/services/search_service.rb @@ -33,8 +33,7 @@ class SearchService < BaseService end def perform_statuses_search! - definition = StatusesIndex.filter(term: { searchable_by: @account.id }) - .query(multi_match: { type: 'most_fields', query: @query, operator: 'and', fields: %w(text text.stemmed) }) + definition = parsed_query.apply(StatusesIndex.filter(term: { searchable_by: @account.id })) if @options[:account_id].present? definition = definition.filter(term: { account_id: @options[:account_id] }) @@ -70,7 +69,7 @@ class SearchService < BaseService end def url_query? - @options[:type].blank? && @query =~ /\Ahttps?:\/\// + @resolve && @options[:type].blank? && @query =~ /\Ahttps?:\/\// end def url_resource_results @@ -120,4 +119,8 @@ class SearchService < BaseService domain_blocking_by_domain: Account.domain_blocking_map_by_domain(domains, account.id), } end + + def parsed_query + SearchQueryTransformer.new.apply(SearchQueryParser.new.parse(@query)) + end end diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb index 00cffcdfc8..902af376c8 100644 --- a/app/services/suspend_account_service.rb +++ b/app/services/suspend_account_service.rb @@ -64,6 +64,7 @@ class SuspendAccountService < BaseService @account.user.destroy else @account.user.disable! + @account.user.invites.where(uses: 0).destroy_all end end diff --git a/app/views/accounts/show.html.haml b/app/views/accounts/show.html.haml index 0dc984dcc2..6846abeb69 100644 --- a/app/views/accounts/show.html.haml +++ b/app/views/accounts/show.html.haml @@ -5,7 +5,7 @@ %meta{ name: 'description', content: account_description(@account) }/ - if @account.user&.setting_noindex - %meta{ name: 'robots', content: 'noindex' }/ + %meta{ name: 'robots', content: 'noindex, noarchive' }/ %link{ rel: 'alternate', type: 'application/atom+xml', href: account_url(@account, format: 'atom') }/ %link{ rel: 'alternate', type: 'application/rss+xml', href: account_url(@account, format: 'rss') }/ diff --git a/app/views/statuses/show.html.haml b/app/views/statuses/show.html.haml index 704e37a3df..0f22d106b5 100644 --- a/app/views/statuses/show.html.haml +++ b/app/views/statuses/show.html.haml @@ -3,7 +3,7 @@ - content_for :header_tags do - if @account.user&.setting_noindex - %meta{ name: 'robots', content: 'noindex' }/ + %meta{ name: 'robots', content: 'noindex, noarchive' }/ %link{ rel: 'alternate', type: 'application/json+oembed', href: api_oembed_url(url: short_account_status_url(@account, @status), format: 'json') }/ %link{ rel: 'alternate', type: 'application/activity+json', href: ActivityPub::TagManager.instance.uri_for(@status) }/ diff --git a/app/workers/scheduler/preview_cards_cleanup_scheduler.rb b/app/workers/scheduler/preview_cards_cleanup_scheduler.rb deleted file mode 100644 index 2b38792f03..0000000000 --- a/app/workers/scheduler/preview_cards_cleanup_scheduler.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -class Scheduler::PreviewCardsCleanupScheduler - include Sidekiq::Worker - - sidekiq_options unique: :until_executed, retry: 0 - - def perform - Maintenance::UncachePreviewWorker.push_bulk(recent_link_preview_cards.pluck(:id)) - Maintenance::UncachePreviewWorker.push_bulk(older_preview_cards.pluck(:id)) - end - - private - - def recent_link_preview_cards - PreviewCard.where(type: :link).where('updated_at < ?', 1.month.ago) - end - - def older_preview_cards - PreviewCard.where('updated_at < ?', 6.months.ago) - end -end diff --git a/config/navigation.rb b/config/navigation.rb index 52d41f72f9..244fe03d3e 100644 --- a/config/navigation.rb +++ b/config/navigation.rb @@ -4,13 +4,13 @@ SimpleNavigation::Configuration.run do |navigation| navigation.items do |n| n.item :web, safe_join([fa_icon('chevron-left fw'), t('settings.back')]), root_url - n.item :profile, safe_join([fa_icon('user fw'), t('settings.profile')]), settings_profile_url do |s| + n.item :profile, safe_join([fa_icon('user fw'), t('settings.profile')]), settings_profile_url, if: -> { current_user.functional? } do |s| s.item :profile, safe_join([fa_icon('pencil fw'), t('settings.appearance')]), settings_profile_url, highlights_on: %r{/settings/profile|/settings/migration} s.item :featured_tags, safe_join([fa_icon('hashtag fw'), t('settings.featured_tags')]), settings_featured_tags_url s.item :identity_proofs, safe_join([fa_icon('key fw'), t('settings.identity_proofs')]), settings_identity_proofs_path, highlights_on: %r{/settings/identity_proofs*}, if: proc { current_account.identity_proofs.exists? } end - n.item :preferences, safe_join([fa_icon('cog fw'), t('settings.preferences')]), settings_preferences_url do |s| + n.item :preferences, safe_join([fa_icon('cog fw'), t('settings.preferences')]), settings_preferences_url, if: -> { current_user.functional? } do |s| s.item :appearance, safe_join([fa_icon('desktop fw'), t('settings.appearance')]), settings_preferences_appearance_url s.item :notifications, safe_join([fa_icon('bell fw'), t('settings.notifications')]), settings_preferences_notifications_url s.item :other, safe_join([fa_icon('cog fw'), t('preferences.other')]), settings_preferences_other_url @@ -22,8 +22,8 @@ SimpleNavigation::Configuration.run do |navigation| end end - n.item :relationships, safe_join([fa_icon('users fw'), t('settings.relationships')]), relationships_url - n.item :filters, safe_join([fa_icon('filter fw'), t('filters.index.title')]), filters_path, highlights_on: %r{/filters} + n.item :relationships, safe_join([fa_icon('users fw'), t('settings.relationships')]), relationships_url, if: -> { current_user.functional? } + n.item :filters, safe_join([fa_icon('filter fw'), t('filters.index.title')]), filters_path, highlights_on: %r{/filters}, if: -> { current_user.functional? } n.item :security, safe_join([fa_icon('lock fw'), t('settings.account')]), edit_user_registration_url do |s| s.item :password, safe_join([fa_icon('lock fw'), t('settings.account_settings')]), edit_user_registration_url, highlights_on: %r{/auth/edit|/settings/delete} @@ -31,13 +31,13 @@ SimpleNavigation::Configuration.run do |navigation| s.item :authorized_apps, safe_join([fa_icon('list fw'), t('settings.authorized_apps')]), oauth_authorized_applications_url end - n.item :data, safe_join([fa_icon('cloud-download fw'), t('settings.import_and_export')]), settings_export_url do |s| + n.item :data, safe_join([fa_icon('cloud-download fw'), t('settings.import_and_export')]), settings_export_url, if: -> { current_user.functional? } do |s| s.item :import, safe_join([fa_icon('cloud-upload fw'), t('settings.import')]), settings_import_url s.item :export, safe_join([fa_icon('cloud-download fw'), t('settings.export')]), settings_export_url end - n.item :invites, safe_join([fa_icon('user-plus fw'), t('invites.title')]), invites_path, if: proc { Setting.min_invite_role == 'user' } - n.item :development, safe_join([fa_icon('code fw'), t('settings.development')]), settings_applications_url + n.item :invites, safe_join([fa_icon('user-plus fw'), t('invites.title')]), invites_path, if: proc { Setting.min_invite_role == 'user' && current_user.functional? } + n.item :development, safe_join([fa_icon('code fw'), t('settings.development')]), settings_applications_url, if: -> { current_user.functional? } n.item :moderation, safe_join([fa_icon('gavel fw'), t('moderation.title')]), admin_reports_url, if: proc { current_user.staff? } do |s| s.item :action_logs, safe_join([fa_icon('bars fw'), t('admin.action_logs.title')]), admin_action_logs_url diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 7f41b6607b..6ebe450b00 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -24,9 +24,6 @@ ip_cleanup_scheduler: cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *' class: Scheduler::IpCleanupScheduler - preview_cards_cleanup_scheduler: - cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *' - class: Scheduler::PreviewCardsCleanupScheduler email_scheduler: cron: '0 10 * * 2' class: Scheduler::EmailScheduler diff --git a/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb b/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb new file mode 100644 index 0000000000..6fa8c0ec43 --- /dev/null +++ b/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb @@ -0,0 +1,15 @@ +class AddCaseInsensitiveIndexToTags < ActiveRecord::Migration[5.2] + disable_ddl_transaction! + + def up + safety_assured { execute 'CREATE UNIQUE INDEX CONCURRENTLY index_tags_on_name_lower ON tags (lower(name))' } + remove_index :tags, name: 'index_tags_on_name' + remove_index :tags, name: 'hashtag_search_index' + end + + def down + add_index :tags, :name, unique: true, algorithm: :concurrently + safety_assured { execute 'CREATE INDEX CONCURRENTLY hashtag_search_index ON tags (name text_pattern_ops)' } + remove_index :tags, name: 'index_tags_on_name_lower' + end +end diff --git a/db/schema.rb b/db/schema.rb index b7da26e35c..22cea1507c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_07_15_164535) do +ActiveRecord::Schema.define(version: 2019_07_26_175042) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -665,8 +665,7 @@ ActiveRecord::Schema.define(version: 2019_07_15_164535) do t.string "name", default: "", null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.index "lower((name)::text) text_pattern_ops", name: "hashtag_search_index" - t.index ["name"], name: "index_tags_on_name", unique: true + t.index "lower((name)::text)", name: "index_tags_on_name_lower", unique: true end create_table "tombstones", force: :cascade do |t| diff --git a/lib/cli.rb b/lib/cli.rb index be276583d2..fbdf49fc34 100644 --- a/lib/cli.rb +++ b/lib/cli.rb @@ -9,6 +9,7 @@ require_relative 'mastodon/search_cli' require_relative 'mastodon/settings_cli' require_relative 'mastodon/statuses_cli' require_relative 'mastodon/domains_cli' +require_relative 'mastodon/preview_cards_cli' require_relative 'mastodon/cache_cli' require_relative 'mastodon/version' @@ -42,6 +43,9 @@ module Mastodon desc 'domains SUBCOMMAND ...ARGS', 'Manage account domains' subcommand 'domains', Mastodon::DomainsCLI + desc 'preview_cards SUBCOMMAND ...ARGS', 'Manage preview cards' + subcommand 'preview_cards', Mastodon::PreviewCardsCLI + desc 'cache SUBCOMMAND ...ARGS', 'Manage cache' subcommand 'cache', Mastodon::CacheCLI diff --git a/lib/json_ld/security.rb b/lib/json_ld/security.rb index 1230206f03..a6fbce95f5 100644 --- a/lib/json_ld/security.rb +++ b/lib/json_ld/security.rb @@ -1,9 +1,9 @@ # -*- encoding: utf-8 -*- # frozen_string_literal: true -# This file generated automatically from https://w3id.org/security/v1 +# This file generated automatically from http://w3id.org/security/v1 require 'json/ld' class JSON::LD::Context - add_preloaded("https://w3id.org/security/v1") do + add_preloaded("http://w3id.org/security/v1") do new(processingMode: "json-ld-1.0", term_definitions: { "CryptographicKey" => TermDefinition.new("CryptographicKey", id: "https://w3id.org/security#Key", simple: true), "EcdsaKoblitzSignature2016" => TermDefinition.new("EcdsaKoblitzSignature2016", id: "https://w3id.org/security#EcdsaKoblitzSignature2016", simple: true), @@ -47,4 +47,5 @@ class JSON::LD::Context "xsd" => TermDefinition.new("xsd", id: "http://www.w3.org/2001/XMLSchema#", simple: true, prefix: true) }) end + alias_preloaded("https://w3id.org/security/v1", "http://w3id.org/security/v1") end diff --git a/lib/mastodon/preview_cards_cli.rb b/lib/mastodon/preview_cards_cli.rb new file mode 100644 index 0000000000..465fe7d0b0 --- /dev/null +++ b/lib/mastodon/preview_cards_cli.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require 'tty-prompt' +require_relative '../../config/boot' +require_relative '../../config/environment' +require_relative 'cli_helper' + +module Mastodon + class PreviewCardsCLI < Thor + include ActionView::Helpers::NumberHelper + + def self.exit_on_failure? + true + end + + option :days, type: :numeric, default: 180 + option :background, type: :boolean, default: false + option :verbose, type: :boolean, default: false + option :dry_run, type: :boolean, default: false + option :link, type: :boolean, default: false + desc 'remove', 'Remove preview cards' + long_desc <<-DESC + Removes locally thumbnails for previews. + + The --days option specifies how old preview cards have to be before + they are removed. It defaults to 180 days. + + With the --background option, instead of deleting the files sequentially, + they will be queued into Sidekiq and the command will exit as soon as + possible. In Sidekiq they will be processed with higher concurrency, but + it may impact other operations of the Mastodon server, and it may overload + the underlying file storage. + + With the --dry-run option, no work will be done. + + With the --verbose option, when preview cards are processed sequentially in the + foreground, the IDs of the preview cards will be printed. + + With the --link option, delete only link-type preview cards. + DESC + def remove + prompt = TTY::Prompt.new + time_ago = options[:days].days.ago + queued = 0 + processed = 0 + size = 0 + dry_run = options[:dry_run] ? '(DRY RUN)' : '' + link = options[:link] ? 'link-type ' : '' + scope = PreviewCard.where.not(image_file_name: nil) + scope = scope.where.not(image_file_name: '') + scope = scope.where(type: :link) if options[:link] + scope = scope.where('updated_at < ?', time_ago) + + if time_ago > 2.weeks.ago + prompt.say "\n" + prompt.say('The preview cards less than the past two weeks will not be re-acquired even when needed.') + prompt.say "\n" + + unless prompt.yes?('Are you sure you want to delete the preview cards?', default: false) + prompt.say "\n" + prompt.warn 'Nothing execute. Bye!' + prompt.say "\n" + exit(1) + end + end + + if options[:background] + scope.select(:id, :image_file_size).reorder(nil).find_in_batches do |preview_cards| + queued += preview_cards.size + size += preview_cards.reduce(0) { |sum, p| sum + (p.image_file_size || 0) } + Maintenance::UncachePreviewWorker.push_bulk(preview_cards.map(&:id)) unless options[:dry_run] + end + + else + scope.select(:id, :image_file_size).reorder(nil).find_in_batches do |preview_cards| + preview_cards.each do |p| + size += p.image_file_size || 0 + Maintenance::UncachePreviewWorker.new.perform(p.id) unless options[:dry_run] + options[:verbose] ? say(p.id) : say('.', :green, false) + processed += 1 + end + end + end + + say + + if options[:background] + say("Scheduled the deletion of #{queued} #{link}preview cards (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true) + else + say("Removed #{processed} #{link}preview cards (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true) + end + end + end +end diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index cd216b92df..4645feb2f3 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -16,22 +16,18 @@ module Mastodon 2 end - def pre - nil - end - def flags '' end - def to_a - [major, minor, patch, pre].compact - end - def suffix '+glitch' end + def to_a + [major, minor, patch].compact + end + def to_s [to_a.join('.'), flags, suffix].join end diff --git a/spec/lib/spam_check_spec.rb b/spec/lib/spam_check_spec.rb index c722dc6427..9e0989216a 100644 --- a/spec/lib/spam_check_spec.rb +++ b/spec/lib/spam_check_spec.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'rails_helper' RSpec.describe SpamCheck do @@ -133,7 +135,31 @@ RSpec.describe SpamCheck do end describe '#remember!' do - pending + let(:status) { status_with_html('@alice') } + let(:spam_check) { described_class.new(status) } + let(:redis_key) { spam_check.send(:redis_key) } + + it 'remembers' do + expect do + spam_check.remember! + end.to change { Redis.current.exists(redis_key) }.from(false).to(true) + end + end + + describe '#reset!' do + let(:status) { status_with_html('@alice') } + let(:spam_check) { described_class.new(status) } + let(:redis_key) { spam_check.send(:redis_key) } + + before do + spam_check.remember! + end + + it 'resets' do + expect do + spam_check.reset! + end.to change { Redis.current.exists(redis_key) }.from(true).to(false) + end end describe '#flag!' do diff --git a/spec/models/invite_spec.rb b/spec/models/invite_spec.rb index 0ba1dccb33..30abfb86bf 100644 --- a/spec/models/invite_spec.rb +++ b/spec/models/invite_spec.rb @@ -3,27 +3,33 @@ require 'rails_helper' RSpec.describe Invite, type: :model do describe '#valid_for_use?' do it 'returns true when there are no limitations' do - invite = Invite.new(max_uses: nil, expires_at: nil) + invite = Fabricate(:invite, max_uses: nil, expires_at: nil) expect(invite.valid_for_use?).to be true end it 'returns true when not expired' do - invite = Invite.new(max_uses: nil, expires_at: 1.hour.from_now) + invite = Fabricate(:invite, max_uses: nil, expires_at: 1.hour.from_now) expect(invite.valid_for_use?).to be true end it 'returns false when expired' do - invite = Invite.new(max_uses: nil, expires_at: 1.hour.ago) + invite = Fabricate(:invite, max_uses: nil, expires_at: 1.hour.ago) expect(invite.valid_for_use?).to be false end it 'returns true when uses still available' do - invite = Invite.new(max_uses: 250, uses: 249, expires_at: nil) + invite = Fabricate(:invite, max_uses: 250, uses: 249, expires_at: nil) expect(invite.valid_for_use?).to be true end it 'returns false when maximum uses reached' do - invite = Invite.new(max_uses: 250, uses: 250, expires_at: nil) + invite = Fabricate(:invite, max_uses: 250, uses: 250, expires_at: nil) + expect(invite.valid_for_use?).to be false + end + + it 'returns false when invite creator has been disabled' do + invite = Fabricate(:invite, max_uses: nil, expires_at: nil) + SuspendAccountService.new.call(invite.user.account) expect(invite.valid_for_use?).to be false end end diff --git a/spec/models/poll_vote_spec.rb b/spec/models/poll_vote_spec.rb index 354afd5350..563f346993 100644 --- a/spec/models/poll_vote_spec.rb +++ b/spec/models/poll_vote_spec.rb @@ -1,5 +1,13 @@ +# frozen_string_literal: true + require 'rails_helper' RSpec.describe PollVote, type: :model do - pending "add some examples to (or delete) #{__FILE__}" + describe '#object_type' do + let(:poll_vote) { Fabricate.build(:poll_vote) } + + it 'returns :vote' do + expect(poll_vote.object_type).to eq :vote + end + end end diff --git a/spec/services/search_service_spec.rb b/spec/services/search_service_spec.rb index d064cd9b85..ade306ed28 100644 --- a/spec/services/search_service_spec.rb +++ b/spec/services/search_service_spec.rb @@ -27,7 +27,7 @@ describe SearchService, type: :service do it 'returns the empty results' do service = double(call: nil) allow(ResolveURLService).to receive(:new).and_return(service) - results = subject.call(@query, nil, 10) + results = subject.call(@query, nil, 10, resolve: true) expect(service).to have_received(:call).with(@query, on_behalf_of: nil) expect(results).to eq empty_results @@ -40,7 +40,7 @@ describe SearchService, type: :service do service = double(call: account) allow(ResolveURLService).to receive(:new).and_return(service) - results = subject.call(@query, nil, 10) + results = subject.call(@query, nil, 10, resolve: true) expect(service).to have_received(:call).with(@query, on_behalf_of: nil) expect(results).to eq empty_results.merge(accounts: [account]) end @@ -52,7 +52,7 @@ describe SearchService, type: :service do service = double(call: status) allow(ResolveURLService).to receive(:new).and_return(service) - results = subject.call(@query, nil, 10) + results = subject.call(@query, nil, 10, resolve: true) expect(service).to have_received(:call).with(@query, on_behalf_of: nil) expect(results).to eq empty_results.merge(statuses: [status]) end