treehouse
/
mastodon
22
8
Fork 5
Code Issues 32 Pull Requests Projects 1 Releases Wiki Activity

Merge pull request #1653 from ClearlyClaire/glitch-soc/merge-upstream 

Merge upstream changes
remotes/1727458204337373841/tmp_refs/heads/signup-info-prompt
Claire 2021-12-27 16:46:38 +01:00 committed by GitHub
parent 3b9a6049ad f011787003
commit e2933854ac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 911 additions and 964 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Dockerfile
View File

@ -56,8 +56,8 @@ RUN npm install -g npm@latest && \
COPY Gemfile* package.json yarn.lock /opt/mastodon/ COPY Gemfile* package.json yarn.lock /opt/mastodon/
RUN cd /opt/mastodon && \ RUN cd /opt/mastodon && \
bundle config set deployment 'true' && \ bundle config set --local deployment 'true' && \
bundle config set without 'development test' && \ bundle config set --local without 'development test' && \
bundle config set silence_root_warning true && \ bundle config set silence_root_warning true && \
bundle install -j"$(nproc)" && \ bundle install -j"$(nproc)" && \
yarn install --pure-lockfile yarn install --pure-lockfile

8
Gemfile
View File

@ -18,7 +18,7 @@ gem 'makara', '~> 0.5'
gem 'pghero', '~> 2.8' gem 'pghero', '~> 2.8'
gem 'dotenv-rails', '~> 2.7' gem 'dotenv-rails', '~> 2.7'
gem 'aws-sdk-s3', '~> 1.107', require: false gem 'aws-sdk-s3', '~> 1.109', require: false
gem 'fog-core', '<= 2.1.0' gem 'fog-core', '<= 2.1.0'
gem 'fog-openstack', '~> 0.3', require: false gem 'fog-openstack', '~> 0.3', require: false
gem 'kt-paperclip', '~> 7.0' gem 'kt-paperclip', '~> 7.0'
@ -102,7 +102,7 @@ gem 'rdf-normalize', '~> 0.4'
gem 'redcarpet', '~> 3.5' gem 'redcarpet', '~> 3.5'
group :development, :test do group :development, :test do
gem 'fabrication', '~> 2.22' gem 'fabrication', '~> 2.23'
gem 'fuubar', '~> 2.5' gem 'fuubar', '~> 2.5'
gem 'i18n-tasks', '~> 0.9', require: false gem 'i18n-tasks', '~> 0.9', require: false
gem 'pry-byebug', '~> 3.9' gem 'pry-byebug', '~> 3.9'
@ -131,13 +131,13 @@ group :development do
gem 'annotate', '~> 3.1' gem 'annotate', '~> 3.1'
gem 'better_errors', '~> 2.9' gem 'better_errors', '~> 2.9'
gem 'binding_of_caller', '~> 1.0' gem 'binding_of_caller', '~> 1.0'
gem 'bullet', '~> 6.1' gem 'bullet', '~> 7.0'
gem 'letter_opener', '~> 1.7' gem 'letter_opener', '~> 1.7'
gem 'letter_opener_web', '~> 2.0' gem 'letter_opener_web', '~> 2.0'
gem 'memory_profiler' gem 'memory_profiler'
gem 'rubocop', '~> 1.23', require: false gem 'rubocop', '~> 1.23', require: false
gem 'rubocop-rails', '~> 2.12', require: false gem 'rubocop-rails', '~> 2.12', require: false
gem 'brakeman', '~> 5.1', require: false gem 'brakeman', '~> 5.2', require: false
gem 'bundler-audit', '~> 0.9', require: false gem 'bundler-audit', '~> 0.9', require: false
gem 'capistrano', '~> 3.16' gem 'capistrano', '~> 3.16'

44
Gemfile.lock
View File

@ -79,16 +79,16 @@ GEM
encryptor (~> 3.0.0) encryptor (~> 3.0.0)
awrence (1.1.1) awrence (1.1.1)
aws-eventstream (1.2.0) aws-eventstream (1.2.0)
aws-partitions (1.534.0) aws-partitions (1.539.0)
aws-sdk-core (3.123.0) aws-sdk-core (3.124.0)
aws-eventstream (~> 1, >= 1.0.2) aws-eventstream (~> 1, >= 1.0.2)
aws-partitions (~> 1, >= 1.525.0) aws-partitions (~> 1, >= 1.525.0)
aws-sigv4 (~> 1.1) aws-sigv4 (~> 1.1)
jmespath (~> 1.0) jmespath (~> 1.0)
aws-sdk-kms (1.51.0) aws-sdk-kms (1.52.0)
aws-sdk-core (~> 3, >= 3.122.0) aws-sdk-core (~> 3, >= 3.122.0)
aws-sigv4 (~> 1.1) aws-sigv4 (~> 1.1)
aws-sdk-s3 (1.107.0) aws-sdk-s3 (1.109.0)
aws-sdk-core (~> 3, >= 3.122.0) aws-sdk-core (~> 3, >= 3.122.0)
aws-sdk-kms (~> 1) aws-sdk-kms (~> 1)
aws-sigv4 (~> 1.4) aws-sigv4 (~> 1.4)
@ -106,13 +106,13 @@ GEM
ffi (~> 1.14) ffi (~> 1.14)
bootsnap (1.9.3) bootsnap (1.9.3)
msgpack (~> 1.0) msgpack (~> 1.0)
brakeman (5.1.2) brakeman (5.2.0)
browser (4.2.0) browser (4.2.0)
brpoplpush-redis_script (0.1.2) brpoplpush-redis_script (0.1.2)
concurrent-ruby (~> 1.0, >= 1.0.5) concurrent-ruby (~> 1.0, >= 1.0.5)
redis (>= 1.0, <= 5.0) redis (>= 1.0, <= 5.0)
builder (3.2.4) builder (3.2.4)
bullet (6.1.5) bullet (7.0.0)
activesupport (>= 3.0.0) activesupport (>= 3.0.0)
uniform_notifier (~> 1.11) uniform_notifier (~> 1.11)
bundler-audit (0.9.0.1) bundler-audit (0.9.0.1)
@ -168,7 +168,7 @@ GEM
css_parser (1.7.1) css_parser (1.7.1)
addressable addressable
debug_inspector (1.0.0) debug_inspector (1.0.0)
devise (4.8.0) devise (4.8.1)
bcrypt (~> 3.0) bcrypt (~> 3.0)
orm_adapter (~> 0.1) orm_adapter (~> 0.1)
railties (>= 4.1.0) railties (>= 4.1.0)
@ -184,8 +184,8 @@ GEM
devise (>= 4.0.0) devise (>= 4.0.0)
rpam2 (~> 4.0) rpam2 (~> 4.0)
diff-lcs (1.4.4) diff-lcs (1.4.4)
discard (1.2.0) discard (1.2.1)
activerecord (>= 4.2, < 7) activerecord (>= 4.2, < 8)
docile (1.3.4) docile (1.3.4)
domain_name (0.5.20190701) domain_name (0.5.20190701)
unf (>= 0.0.5, < 1.0.0) unf (>= 0.0.5, < 1.0.0)
@ -211,7 +211,7 @@ GEM
et-orbi (1.2.4) et-orbi (1.2.4)
tzinfo tzinfo
excon (0.76.0) excon (0.76.0)
fabrication (2.22.0) fabrication (2.23.1)
faker (2.19.0) faker (2.19.0)
i18n (>= 1.6, < 2) i18n (>= 1.6, < 2)
faraday (1.8.0) faraday (1.8.0)
@ -234,7 +234,7 @@ GEM
faraday-patron (1.0.0) faraday-patron (1.0.0)
faraday-rack (1.0.0) faraday-rack (1.0.0)
fast_blank (1.0.1) fast_blank (1.0.1)
fastimage (2.2.5) fastimage (2.2.6)
ffi (1.15.4) ffi (1.15.4)
ffi-compiler (1.0.1) ffi-compiler (1.0.1)
ffi (>= 1.0.0) ffi (>= 1.0.0)
@ -355,7 +355,7 @@ GEM
activesupport (>= 4) activesupport (>= 4)
railties (>= 4) railties (>= 4)
request_store (~> 1.0) request_store (~> 1.0)
loofah (2.12.0) loofah (2.13.0)
crass (~> 1.0.2) crass (~> 1.0.2)
nokogiri (>= 1.5.9) nokogiri (>= 1.5.9)
mail (2.7.1) mail (2.7.1)
@ -376,7 +376,7 @@ GEM
mime-types-data (3.2021.1115) mime-types-data (3.2021.1115)
mini_mime (1.1.2) mini_mime (1.1.2)
mini_portile2 (2.6.1) mini_portile2 (2.6.1)
minitest (5.14.4) minitest (5.15.0)
msgpack (1.4.2) msgpack (1.4.2)
multi_json (1.15.0) multi_json (1.15.0)
multipart-post (2.1.1) multipart-post (2.1.1)
@ -393,7 +393,7 @@ GEM
concurrent-ruby (~> 1.0, >= 1.0.2) concurrent-ruby (~> 1.0, >= 1.0.2)
sidekiq (>= 3.5) sidekiq (>= 3.5)
statsd-ruby (~> 1.4, >= 1.4.0) statsd-ruby (~> 1.4, >= 1.4.0)
oj (3.13.9) oj (3.13.10)
omniauth (1.9.1) omniauth (1.9.1)
hashie (>= 3.4.6) hashie (>= 3.4.6)
rack (>= 1.6.2, < 3) rack (>= 1.6.2, < 3)
@ -418,7 +418,7 @@ GEM
pastel (0.8.0) pastel (0.8.0)
tty-color (~> 0.5) tty-color (~> 0.5)
pg (1.2.3) pg (1.2.3)
pghero (2.8.1) pghero (2.8.2)
activerecord (>= 5) activerecord (>= 5)
pkg-config (1.4.6) pkg-config (1.4.6)
posix-spawn (0.3.15) posix-spawn (0.3.15)
@ -580,7 +580,7 @@ GEM
sidekiq (>= 3) sidekiq (>= 3)
thwait thwait
tilt (>= 1.4.0) tilt (>= 1.4.0)
sidekiq-unique-jobs (7.1.8) sidekiq-unique-jobs (7.1.12)
brpoplpush-redis_script (> 0.1.1, <= 2.0.0) brpoplpush-redis_script (> 0.1.1, <= 2.0.0)
concurrent-ruby (~> 1.0, >= 1.0.5) concurrent-ruby (~> 1.0, >= 1.0.5)
sidekiq (>= 5.0, < 8.0) sidekiq (>= 5.0, < 8.0)
@ -599,7 +599,7 @@ GEM
sprockets (3.7.2) sprockets (3.7.2)
concurrent-ruby (~> 1.0) concurrent-ruby (~> 1.0)
rack (> 1, < 3) rack (> 1, < 3)
sprockets-rails (3.4.1) sprockets-rails (3.4.2)
actionpack (>= 5.2) actionpack (>= 5.2)
activesupport (>= 5.2) activesupport (>= 5.2)
sprockets (>= 3.0.0) sprockets (>= 3.0.0)
@ -609,7 +609,7 @@ GEM
stackprof (0.2.17) stackprof (0.2.17)
statsd-ruby (1.5.0) statsd-ruby (1.5.0)
stoplight (2.2.1) stoplight (2.2.1)
strong_migrations (0.7.8) strong_migrations (0.7.9)
activerecord (>= 5) activerecord (>= 5)
temple (0.8.2) temple (0.8.2)
terminal-table (3.0.2) terminal-table (3.0.2)
@ -686,14 +686,14 @@ DEPENDENCIES
active_record_query_trace (~> 1.8) active_record_query_trace (~> 1.8)
addressable (~> 2.8) addressable (~> 2.8)
annotate (~> 3.1) annotate (~> 3.1)
aws-sdk-s3 (~> 1.107) aws-sdk-s3 (~> 1.109)
better_errors (~> 2.9) better_errors (~> 2.9)
binding_of_caller (~> 1.0) binding_of_caller (~> 1.0)
blurhash (~> 0.1) blurhash (~> 0.1)
bootsnap (~> 1.9.2) bootsnap (~> 1.9.2)
brakeman (~> 5.1) brakeman (~> 5.2)
browser browser
bullet (~> 6.1) bullet (~> 7.0)
bundler-audit (~> 0.9) bundler-audit (~> 0.9)
capistrano (~> 3.16) capistrano (~> 3.16)
capistrano-rails (~> 1.6) capistrano-rails (~> 1.6)
@ -714,7 +714,7 @@ DEPENDENCIES
doorkeeper (~> 5.5) doorkeeper (~> 5.5)
dotenv-rails (~> 2.7) dotenv-rails (~> 2.7)
ed25519 (~> 1.2) ed25519 (~> 1.2)
fabrication (~> 2.22) fabrication (~> 2.23)
faker (~> 2.19) faker (~> 2.19)
fast_blank (~> 1.0) fast_blank (~> 1.0)
fastimage fastimage

4
app/javascript/flavours/glitch/features/hashtag_timeline/components/column_settings.js
View File

@ -33,8 +33,8 @@ class ColumnSettings extends React.PureComponent {
tags (mode) { tags (mode) {
let tags = this.props.settings.getIn(['tags', mode]) || []; let tags = this.props.settings.getIn(['tags', mode]) || [];
if (tags.toJSON) { if (tags.toJS) {
return tags.toJSON(); return tags.toJS();
} else { } else {
return tags; return tags;
} }

4
app/javascript/mastodon/features/hashtag_timeline/components/column_settings.js
View File

@ -33,8 +33,8 @@ class ColumnSettings extends React.PureComponent {
tags (mode) { tags (mode) {
let tags = this.props.settings.getIn(['tags', mode]) || []; let tags = this.props.settings.getIn(['tags', mode]) || [];
if (tags.toJSON) { if (tags.toJS) {
return tags.toJSON(); return tags.toJS();
} else { } else {
return tags; return tags;
} }

2
app/models/user.rb
View File

@ -107,7 +107,7 @@ class User < ApplicationRecord
scope :inactive, -> { where(arel_table[:current_sign_in_at].lt(ACTIVE_DURATION.ago)) } scope :inactive, -> { where(arel_table[:current_sign_in_at].lt(ACTIVE_DURATION.ago)) }
scope :active, -> { confirmed.where(arel_table[:current_sign_in_at].gteq(ACTIVE_DURATION.ago)).joins(:account).where(accounts: { suspended_at: nil }) } scope :active, -> { confirmed.where(arel_table[:current_sign_in_at].gteq(ACTIVE_DURATION.ago)).joins(:account).where(accounts: { suspended_at: nil }) }
scope :matches_email, ->(value) { where(arel_table[:email].matches("#{value}%")) } scope :matches_email, ->(value) { where(arel_table[:email].matches("#{value}%")) }
scope :matches_ip, ->(value) { left_joins(:session_activations).where('users.current_sign_in_ip <<= ?', value).or(left_joins(:session_activations).where('users.sign_up_ip <<= ?', value)).or(left_joins(:session_activations).where('users.last_sign_in_ip <<= ?', value)).or(left_joins(:session_activations).where('session_activations.ip <<= ?', value)) } scope :matches_ip, ->(value) { where('current_sign_in_ip <<= ?', value).or(where('users.sign_up_ip <<= ?', value)).or(where('users.last_sign_in_ip <<= ?', value)).or(where(id: SessionActivation.select(:user_id).where('ip <<= ?', value))) }
scope :emailable, -> { confirmed.enabled.joins(:account).merge(Account.searchable) } scope :emailable, -> { confirmed.enabled.joins(:account).merge(Account.searchable) }
before_validation :sanitize_languages before_validation :sanitize_languages

2
config/environments/development.rb
View File

@ -81,7 +81,7 @@ Rails.application.configure do
Bullet.bullet_logger = true Bullet.bullet_logger = true
Bullet.rails_logger = false Bullet.rails_logger = false
Bullet.add_whitelist type: :n_plus_one_query, class_name: 'User', association: :account Bullet.add_safelist type: :n_plus_one_query, class_name: 'User', association: :account
end end
config.x.otp_secret = ENV.fetch('OTP_SECRET', '1fc2b87989afa6351912abeebe31ffc5c476ead9bf8b3d74cbc4a302c7b69a45b40b1bbef3506ddad73e942e15ed5ca4b402bf9a66423626051104f4b5f05109') config.x.otp_secret = ENV.fetch('OTP_SECRET', '1fc2b87989afa6351912abeebe31ffc5c476ead9bf8b3d74cbc4a302c7b69a45b40b1bbef3506ddad73e942e15ed5ca4b402bf9a66423626051104f4b5f05109')

2
config/initializers/sidekiq.rb
View File

@ -1,5 +1,7 @@
# frozen_string_literal: true # frozen_string_literal: true
require_relative '../../lib/sidekiq_error_handler'
Sidekiq.configure_server do |config| Sidekiq.configure_server do |config|
config.redis = REDIS_SIDEKIQ_PARAMS config.redis = REDIS_SIDEKIQ_PARAMS

1
dist/mastodon-sidekiq.service vendored
View File

@ -47,6 +47,7 @@ SystemCallFilter=~@cpu-emulation @debug @keyring @ipc @mount @obsolete @privileg
SystemCallFilter=@chown SystemCallFilter=@chown
SystemCallFilter=pipe SystemCallFilter=pipe
SystemCallFilter=pipe2 SystemCallFilter=pipe2
ReadWritePaths=/home/mastodon/live
[Install] [Install]
WantedBy=multi-user.target WantedBy=multi-user.target

1
dist/mastodon-streaming.service vendored
View File

@ -45,6 +45,7 @@ SystemCallArchitectures=native
SystemCallFilter=~@cpu-emulation @debug @keyring @ipc @memlock @mount @obsolete @privileged @resources @setuid SystemCallFilter=~@cpu-emulation @debug @keyring @ipc @memlock @mount @obsolete @privileged @resources @setuid
SystemCallFilter=pipe SystemCallFilter=pipe
SystemCallFilter=pipe2 SystemCallFilter=pipe2
ReadWritePaths=/home/mastodon/live
[Install] [Install]
WantedBy=multi-user.target WantedBy=multi-user.target

1
dist/mastodon-web.service vendored
View File

@ -47,6 +47,7 @@ SystemCallFilter=~@cpu-emulation @debug @keyring @ipc @mount @obsolete @privileg
SystemCallFilter=@chown SystemCallFilter=@chown
SystemCallFilter=pipe SystemCallFilter=pipe
SystemCallFilter=pipe2 SystemCallFilter=pipe2
ReadWritePaths=/home/mastodon/live
[Install] [Install]
WantedBy=multi-user.target WantedBy=multi-user.target

8
app/lib/sidekiq_error_handler.rb → lib/sidekiq_error_handler.rb
View File

@ -17,10 +17,8 @@ class SidekiqErrorHandler
private private
# rubocop:disable Naming/MethodParameterName def limit_backtrace_and_raise(exception)
def limit_backtrace_and_raise(e) exception.set_backtrace(exception.backtrace.first(BACKTRACE_LIMIT))
e.set_backtrace(e.backtrace.first(BACKTRACE_LIMIT)) raise exception
raise e
end end
# rubocop:enable Naming/MethodParameterName
end end

30
package.json
View File

@ -61,13 +61,13 @@
}, },
"private": true, "private": true,
"dependencies": { "dependencies": {
"@babel/core": "^7.16.0", "@babel/core": "^7.16.5",
"@babel/plugin-proposal-decorators": "^7.16.4", "@babel/plugin-proposal-decorators": "^7.16.5",
"@babel/plugin-transform-react-inline-elements": "^7.16.0", "@babel/plugin-transform-react-inline-elements": "^7.16.5",
"@babel/plugin-transform-runtime": "^7.16.4", "@babel/plugin-transform-runtime": "^7.16.5",
"@babel/preset-env": "^7.16.4", "@babel/preset-env": "^7.16.5",
"@babel/preset-react": "^7.16.0", "@babel/preset-react": "^7.16.5",
"@babel/runtime": "^7.16.3", "@babel/runtime": "^7.16.5",
"@gamestdio/websocket": "^0.3.2", "@gamestdio/websocket": "^0.3.2",
"@github/webauthn-json": "^0.5.7", "@github/webauthn-json": "^0.5.7",
"@rails/ujs": "^6.1.4", "@rails/ujs": "^6.1.4",
@ -115,7 +115,7 @@
"marky": "^1.2.2", "marky": "^1.2.2",
"mini-css-extract-plugin": "^1.6.2", "mini-css-extract-plugin": "^1.6.2",
"mkdirp": "^1.0.4", "mkdirp": "^1.0.4",
"npmlog": "^5.0.1", "npmlog": "^6.0.0",
"object-assign": "^4.1.1", "object-assign": "^4.1.1",
"object-fit-images": "^3.2.3", "object-fit-images": "^3.2.3",
"object.values": "^1.1.5", "object.values": "^1.1.5",
@ -146,7 +146,7 @@
"react-swipeable-views": "^0.14.0", "react-swipeable-views": "^0.14.0",
"react-textarea-autosize": "^8.3.3", "react-textarea-autosize": "^8.3.3",
"react-toggle": "^4.1.2", "react-toggle": "^4.1.2",
"redis": "^3.1.2", "redis": "^4.0.1",
"redux": "^4.1.2", "redux": "^4.1.2",
"redux-immutable": "^4.0.0", "redux-immutable": "^4.0.0",
"redux-thunk": "^2.4.1", "redux-thunk": "^2.4.1",
@ -155,7 +155,7 @@
"requestidlecallback": "^0.3.0", "requestidlecallback": "^0.3.0",
"reselect": "^4.1.5", "reselect": "^4.1.5",
"rimraf": "^3.0.2", "rimraf": "^3.0.2",
"sass": "^1.43.5", "sass": "^1.45.1",
"sass-loader": "^10.2.0", "sass-loader": "^10.2.0",
"stacktrace-js": "^2.0.2", "stacktrace-js": "^2.0.2",
"stringz": "^2.1.0", "stringz": "^2.1.0",
@ -175,22 +175,22 @@
"ws": "^8.3.0" "ws": "^8.3.0"
}, },
"devDependencies": { "devDependencies": {
"@testing-library/jest-dom": "^5.16.0", "@testing-library/jest-dom": "^5.16.1",
"@testing-library/react": "^12.1.2", "@testing-library/react": "^12.1.2",
"babel-eslint": "^10.1.0", "babel-eslint": "^10.1.0",
"babel-jest": "^27.4.0", "babel-jest": "^27.4.5",
"eslint": "^7.32.0", "eslint": "^7.32.0",
"eslint-plugin-import": "~2.25.3", "eslint-plugin-import": "~2.25.3",
"eslint-plugin-jsx-a11y": "~6.5.1", "eslint-plugin-jsx-a11y": "~6.5.1",
"eslint-plugin-promise": "~5.1.1", "eslint-plugin-promise": "~6.0.0",
"eslint-plugin-react": "~7.27.1", "eslint-plugin-react": "~7.27.1",
"jest": "^27.4.3", "jest": "^27.4.5",
"raf": "^3.4.1", "raf": "^3.4.1",
"react-intl-translations-manager": "^5.0.3", "react-intl-translations-manager": "^5.0.3",
"react-test-renderer": "^16.14.0", "react-test-renderer": "^16.14.0",
"sass-lint": "^1.13.1", "sass-lint": "^1.13.1",
"webpack-dev-server": "^3.11.3", "webpack-dev-server": "^3.11.3",
"yargs": "^17.2.1" "yargs": "^17.3.0"
}, },
"resolutions": { "resolutions": {
"kind-of": "^6.0.3" "kind-of": "^6.0.3"

135
streaming/index.js
View File

@ -63,20 +63,29 @@ const dbUrlToConfig = (dbUrl) => {
* @param {Object.<string, any>} defaultConfig * @param {Object.<string, any>} defaultConfig
* @param {string} redisUrl * @param {string} redisUrl
*/ */
const redisUrlToClient = (defaultConfig, redisUrl) => { const redisUrlToClient = async (defaultConfig, redisUrl) => {
const config = defaultConfig; const config = defaultConfig;
let client;
if (!redisUrl) { if (!redisUrl) {
return redis.createClient(config); client = redis.createClient(config);
} else if (redisUrl.startsWith('unix://')) {
client = redis.createClient(Object.assign(config, {
socket: {
path: redisUrl.slice(7),
},
}));
} else {
client = redis.createClient(Object.assign(config, {
url: redisUrl,
}));
} }
if (redisUrl.startsWith('unix://')) { client.on('error', (err) => log.error('Redis Client Error!', err));
return redis.createClient(redisUrl.slice(7), config); await client.connect();
}
return redis.createClient(Object.assign(config, { return client;
url: redisUrl,
}));
}; };
const numWorkers = +process.env.STREAMING_CLUSTER_NUM || (env === 'development' ? 1 : Math.max(os.cpus().length - 1, 1)); const numWorkers = +process.env.STREAMING_CLUSTER_NUM || (env === 'development' ? 1 : Math.max(os.cpus().length - 1, 1));
@ -102,7 +111,7 @@ const startMaster = () => {
log.warn(`Starting streaming API server master with ${numWorkers} workers`); log.warn(`Starting streaming API server master with ${numWorkers} workers`);
}; };
const startWorker = (workerId) => { const startWorker = async (workerId) => {
log.warn(`Starting worker ${workerId}`); log.warn(`Starting worker ${workerId}`);
const pgConfigs = { const pgConfigs = {
@ -127,7 +136,7 @@ const startWorker = (workerId) => {
if (!!process.env.DB_SSLMODE && process.env.DB_SSLMODE !== 'disable') { if (!!process.env.DB_SSLMODE && process.env.DB_SSLMODE !== 'disable') {
pgConfigs.development.ssl = true; pgConfigs.development.ssl = true;
pgConfigs.production.ssl = true; pgConfigs.production.ssl = true;
} }
const app = express(); const app = express();
@ -139,9 +148,11 @@ const startWorker = (workerId) => {
const redisNamespace = process.env.REDIS_NAMESPACE || null; const redisNamespace = process.env.REDIS_NAMESPACE || null;
const redisParams = { const redisParams = {
host: process.env.REDIS_HOST || '127.0.0.1', socket: {
port: process.env.REDIS_PORT || 6379, host: process.env.REDIS_HOST || '127.0.0.1',
db: process.env.REDIS_DB || 0, port: process.env.REDIS_PORT || 6379,
},
database: process.env.REDIS_DB || 0,
password: process.env.REDIS_PASSWORD || undefined, password: process.env.REDIS_PASSWORD || undefined,
}; };
@ -151,25 +162,8 @@ const startWorker = (workerId) => {
const redisPrefix = redisNamespace ? `${redisNamespace}:` : ''; const redisPrefix = redisNamespace ? `${redisNamespace}:` : '';
const redisSubscribeClient = redisUrlToClient(redisParams, process.env.REDIS_URL); const redisSubscribeClient = await redisUrlToClient(redisParams, process.env.REDIS_URL);
const redisClient = redisUrlToClient(redisParams, process.env.REDIS_URL); const redisClient = await redisUrlToClient(redisParams, process.env.REDIS_URL);
/**
* @type {Object.<string, Array.<function(string): void>>}
*/
const subs = {};
redisSubscribeClient.on('message', (channel, message) => {
const callbacks = subs[channel];
log.silly(`New message on channel ${channel}`);
if (!callbacks) {
return;
}
callbacks.forEach(callback => callback(message));
});
/** /**
* @param {string[]} channels * @param {string[]} channels
@ -197,34 +191,16 @@ const startWorker = (workerId) => {
*/ */
const subscribe = (channel, callback) => { const subscribe = (channel, callback) => {
log.silly(`Adding listener for ${channel}`); log.silly(`Adding listener for ${channel}`);
subs[channel] = subs[channel] || [];
if (subs[channel].length === 0) { redisSubscribeClient.subscribe(channel, callback);
log.verbose(`Subscribe ${channel}`);
redisSubscribeClient.subscribe(channel);
}
subs[channel].push(callback);
}; };
/** /**
* @param {string} channel * @param {string} channel
* @param {function(string): void} callback
*/ */
const unsubscribe = (channel, callback) => { const unsubscribe = (channel) => {
log.silly(`Removing listener for ${channel}`);
if (!subs[channel]) { redisSubscribeClient.unsubscribe(channel);
return;
}
subs[channel] = subs[channel].filter(item => item !== callback);
if (subs[channel].length === 0) {
log.verbose(`Unsubscribe ${channel}`);
redisSubscribeClient.unsubscribe(channel);
delete subs[channel];
}
}; };
const FALSE_VALUES = [ const FALSE_VALUES = [
@ -366,7 +342,7 @@ const startWorker = (workerId) => {
const onlyMedia = isTruthy(query.only_media); const onlyMedia = isTruthy(query.only_media);
const allowLocalOnly = isTruthy(query.allow_local_only); const allowLocalOnly = isTruthy(query.allow_local_only);
switch(path) { switch (path) {
case '/api/v1/streaming/user': case '/api/v1/streaming/user':
return 'user'; return 'user';
case '/api/v1/streaming/user/notification': case '/api/v1/streaming/user/notification':
@ -497,7 +473,7 @@ const startWorker = (workerId) => {
const listener = createSystemMessageListener(req, { const listener = createSystemMessageListener(req, {
onKill () { onKill() {
res.end(); res.end();
}, },
@ -549,7 +525,7 @@ const startWorker = (workerId) => {
}; };
/** /**
* @param {array} * @param {array} arr
* @param {number=} shift * @param {number=} shift
* @return {string} * @return {string}
*/ */
@ -592,7 +568,7 @@ const startWorker = (workerId) => {
* @return {function(string): void} * @return {function(string): void}
*/ */
const streamFrom = (ids, req, output, attachCloseHandler, needsFiltering = false, allowLocalOnly = false) => { const streamFrom = (ids, req, output, attachCloseHandler, needsFiltering = false, allowLocalOnly = false) => {
const accountId = req.accountId || req.remoteAddress; const accountId = req.accountId || req.remoteAddress;
log.verbose(req.requestId, `Starting stream from ${ids.join(', ')} for ${accountId}`); log.verbose(req.requestId, `Starting stream from ${ids.join(', ')} for ${accountId}`);
@ -604,8 +580,8 @@ const startWorker = (workerId) => {
const { event, payload, queued_at } = json; const { event, payload, queued_at } = json;
const transmit = () => { const transmit = () => {
const now = new Date().getTime(); const now = new Date().getTime();
const delta = now - queued_at; const delta = now - queued_at;
const encodedPayload = typeof payload === 'object' ? JSON.stringify(payload) : payload; const encodedPayload = typeof payload === 'object' ? JSON.stringify(payload) : payload;
log.silly(req.requestId, `Transmitting for ${accountId}: ${event} ${encodedPayload} Delay: ${delta}ms`); log.silly(req.requestId, `Transmitting for ${accountId}: ${event} ${encodedPayload} Delay: ${delta}ms`);
@ -625,9 +601,9 @@ const startWorker = (workerId) => {
return; return;
} }
const unpackedPayload = payload; const unpackedPayload = payload;
const targetAccountIds = [unpackedPayload.account.id].concat(unpackedPayload.mentions.map(item => item.id)); const targetAccountIds = [unpackedPayload.account.id].concat(unpackedPayload.mentions.map(item => item.id));
const accountDomain = unpackedPayload.account.acct.split('@')[1]; const accountDomain = unpackedPayload.account.acct.split('@')[1];
if (Array.isArray(req.chosenLanguages) && unpackedPayload.language !== null && req.chosenLanguages.indexOf(unpackedPayload.language) === -1) { if (Array.isArray(req.chosenLanguages) && unpackedPayload.language !== null && req.chosenLanguages.indexOf(unpackedPayload.language) === -1) {
log.silly(req.requestId, `Message ${unpackedPayload.id} filtered by language (${unpackedPayload.language})`); log.silly(req.requestId, `Message ${unpackedPayload.id} filtered by language (${unpackedPayload.language})`);
@ -647,7 +623,15 @@ const startWorker = (workerId) => {
} }
const queries = [ const queries = [
client.query(`SELECT 1 FROM blocks WHERE (account_id = $1 AND target_account_id IN (${placeholders(targetAccountIds, 2)})) OR (account_id = $2 AND target_account_id = $1) UNION SELECT 1 FROM mutes WHERE account_id = $1 AND target_account_id IN (${placeholders(targetAccountIds, 2)})`, [req.accountId, unpackedPayload.account.id].concat(targetAccountIds)), client.query(`SELECT 1
FROM blocks
WHERE (account_id = $1 AND target_account_id IN (${placeholders(targetAccountIds, 2)}))
OR (account_id = $2 AND target_account_id = $1)
UNION
SELECT 1
FROM mutes
WHERE account_id = $1
AND target_account_id IN (${placeholders(targetAccountIds, 2)})`, [req.accountId, unpackedPayload.account.id].concat(targetAccountIds)),
]; ];
if (accountDomain) { if (accountDomain) {
@ -710,12 +694,12 @@ const startWorker = (workerId) => {
/** /**
* @param {any} req * @param {any} req
* @param {function(): void} [closeHandler] * @param {function(): void} [closeHandler]
* @return {function(string[], function(string): void)} * @return {function(string[]): void}
*/ */
const streamHttpEnd = (req, closeHandler = undefined) => (ids, listener) => { const streamHttpEnd = (req, closeHandler = undefined) => (ids) => {
req.on('close', () => { req.on('close', () => {
ids.forEach(id => { ids.forEach(id => {
unsubscribe(id, listener); unsubscribe(id);
}); });
if (closeHandler) { if (closeHandler) {
@ -762,7 +746,7 @@ const startWorker = (workerId) => {
app.get('/api/v1/streaming/*', (req, res) => { app.get('/api/v1/streaming/*', (req, res) => {
channelNameToIds(req, channelNameFromPath(req), req.query).then(({ channelIds, options }) => { channelNameToIds(req, channelNameFromPath(req), req.query).then(({ channelIds, options }) => {
const onSend = streamToHttp(req, res); const onSend = streamToHttp(req, res);
const onEnd = streamHttpEnd(req, subscriptionHeartbeat(channelIds)); const onEnd = streamHttpEnd(req, subscriptionHeartbeat(channelIds));
streamFrom(channelIds, req, onSend, onEnd, options.needsFiltering, options.allowLocalOnly); streamFrom(channelIds, req, onSend, onEnd, options.needsFiltering, options.allowLocalOnly);
}).catch(err => { }).catch(err => {
@ -805,7 +789,7 @@ const startWorker = (workerId) => {
* @return {Promise.<{ channelIds: string[], options: { needsFiltering: boolean } }>} * @return {Promise.<{ channelIds: string[], options: { needsFiltering: boolean } }>}
*/ */
const channelNameToIds = (req, name, params) => new Promise((resolve, reject) => { const channelNameToIds = (req, name, params) => new Promise((resolve, reject) => {
switch(name) { switch (name) {
case 'user': case 'user':
resolve({ resolve({
channelIds: channelsForUserStream(req), channelIds: channelsForUserStream(req),
@ -949,15 +933,17 @@ const startWorker = (workerId) => {
* @param {StreamParams} params * @param {StreamParams} params
*/ */
const subscribeWebsocketToChannel = ({ socket, request, subscriptions }, channelName, params) => const subscribeWebsocketToChannel = ({ socket, request, subscriptions }, channelName, params) =>
checkScopes(request, channelName).then(() => channelNameToIds(request, channelName, params)).then(({ channelIds, options }) => { checkScopes(request, channelName).then(() => channelNameToIds(request, channelName, params)).then(({
channelIds,
options,
}) => {
if (subscriptions[channelIds.join(';')]) { if (subscriptions[channelIds.join(';')]) {
return; return;
} }
const onSend = streamToWs(request, socket, streamNameFromChannelName(channelName, params)); const onSend = streamToWs(request, socket, streamNameFromChannelName(channelName, params));
const stopHeartbeat = subscriptionHeartbeat(channelIds); const stopHeartbeat = subscriptionHeartbeat(channelIds);
const listener = streamFrom(channelIds, request, onSend, undefined, options.needsFiltering, options.allowLocalOnly);
const listener = streamFrom(channelIds, request, onSend, undefined, options.needsFiltering, options.allowLocalOnly);
subscriptions[channelIds.join(';')] = { subscriptions[channelIds.join(';')] = {
listener, listener,
@ -1005,7 +991,7 @@ const startWorker = (workerId) => {
const listener = createSystemMessageListener(request, { const listener = createSystemMessageListener(request, {
onKill () { onKill() {
socket.close(); socket.close();
}, },
@ -1015,7 +1001,8 @@ const startWorker = (workerId) => {
subscriptions[systemChannelId] = { subscriptions[systemChannelId] = {
listener, listener,
stopHeartbeat: () => {}, stopHeartbeat: () => {
},
}; };
}; };
@ -1034,7 +1021,7 @@ const startWorker = (workerId) => {
wss.on('connection', (ws, req) => { wss.on('connection', (ws, req) => {
const location = url.parse(req.url, true); const location = url.parse(req.url, true);
req.requestId = uuid.v4(); req.requestId = uuid.v4();
req.remoteAddress = ws._socket.remoteAddress; req.remoteAddress = ws._socket.remoteAddress;
ws.isAlive = true; ws.isAlive = true;

1629
yarn.lock
View File

File diff suppressed because it is too large Load Diff