2020-08-11 16:24:59 +00:00
// @ts-check
2024-02-27 14:59:20 +00:00
import fs from 'node:fs' ;
import http from 'node:http' ;
import path from 'node:path' ;
import url from 'node:url' ;
import cors from 'cors' ;
import dotenv from 'dotenv' ;
import express from 'express' ;
import { Redis } from 'ioredis' ;
import { JSDOM } from 'jsdom' ;
import pg from 'pg' ;
import pgConnectionString from 'pg-connection-string' ;
2024-06-24 10:29:00 +00:00
import { WebSocketServer } from 'ws' ;
2024-02-27 14:59:20 +00:00
import { AuthenticationError , RequestError , extractStatusAndMessage as extractErrorStatusAndMessage } from './errors.js' ;
import { logger , httpLogger , initializeLogLevel , attachWebsocketHttpLogger , createWebsocketLogger } from './logging.js' ;
import { setupMetrics } from './metrics.js' ;
import { isTruthy , normalizeHashtag , firstParam } from './utils.js' ;
2023-11-28 14:24:41 +00:00
2023-04-26 09:37:51 +00:00
const environment = process . env . NODE _ENV || 'development' ;
2017-02-02 15:11:36 +00:00
2023-11-14 17:43:20 +00:00
// Correctly detect and load .env or .env.production file based on environment:
const dotenvFile = environment === 'production' ? '.env.production' : '.env' ;
2024-02-27 14:59:20 +00:00
const dotenvFilePath = path . resolve (
url . fileURLToPath (
new URL ( path . join ( '..' , dotenvFile ) , import . meta . url )
)
) ;
2023-11-14 17:43:20 +00:00
2017-02-02 15:11:36 +00:00
dotenv . config ( {
2024-02-27 14:59:20 +00:00
path : dotenvFilePath
2017-05-20 15:31:47 +00:00
} ) ;
2017-02-02 00:31:09 +00:00
2024-01-18 18:40:25 +00:00
initializeLogLevel ( process . env , environment ) ;
/ * *
* Declares the result type for accountFromToken / accountFromRequest .
*
* Note : This is here because jsdoc doesn ' t like importing types that
* are nested in functions
* @ typedef ResolvedAccount
* @ property { string } accessTokenId
* @ property { string [ ] } scopes
* @ property { string } accountId
* @ property { string [ ] } chosenLanguages
* @ property { string } deviceId
* /
2017-05-28 14:25:26 +00:00
2020-08-11 16:24:59 +00:00
/ * *
2024-01-25 16:46:02 +00:00
* @ param { RedisConfiguration } config
* @ returns { Promise < Redis > }
2020-08-11 16:24:59 +00:00
* /
2024-01-25 16:46:02 +00:00
const createRedisClient = async ( { redisParams , redisUrl } ) => {
let client ;
if ( typeof redisUrl === 'string' ) {
client = new Redis ( redisUrl , redisParams ) ;
} else {
client = new Redis ( redisParams ) ;
}
2024-01-18 18:40:25 +00:00
client . on ( 'error' , ( err ) => logger . error ( { err } , 'Redis Client Error!' ) ) ;
2017-05-20 19:06:09 +00:00
2021-12-25 21:55:06 +00:00
return client ;
2017-05-20 19:06:09 +00:00
} ;
2020-09-22 13:30:41 +00:00
/ * *
2023-06-09 17:29:16 +00:00
* Attempts to safely parse a string as JSON , used when both receiving a message
* from redis and when receiving a message from a client over a websocket
* connection , this is why it accepts a ` req ` argument .
2020-09-22 13:30:41 +00:00
* @ param { string } json
2023-06-09 17:29:16 +00:00
* @ param { any ? } req
2023-04-30 00:29:54 +00:00
* @ returns { Object . < string , any > | null }
2020-09-22 13:30:41 +00:00
* /
2022-02-16 13:37:26 +00:00
const parseJSON = ( json , req ) => {
2020-09-22 13:30:41 +00:00
try {
return JSON . parse ( json ) ;
} catch ( err ) {
2023-06-09 17:29:16 +00:00
/ * F I X M E : T h i s l o g g i n g i s n ' t g r e a t , a n d s h o u l d p r o b a b l y b e d o n e a t t h e
* call - site of parseJSON , not in the method , but this would require changing
* the signature of parseJSON to return something akin to a Result type :
* [ Error | null , null | Object < string , any } ] , and then handling the error
* scenarios .
* /
if ( req ) {
if ( req . accountId ) {
2024-01-18 18:40:25 +00:00
req . log . error ( { err } , ` Error parsing message from user ${ req . accountId } ` ) ;
2023-06-09 17:29:16 +00:00
} else {
2024-01-18 18:40:25 +00:00
req . log . error ( { err } , ` Error parsing message from ${ req . remoteAddress } ` ) ;
2023-06-09 17:29:16 +00:00
}
2022-02-16 13:37:26 +00:00
} else {
2024-01-18 18:40:25 +00:00
logger . error ( { err } , ` Error parsing message from redis ` ) ;
2022-02-16 13:37:26 +00:00
}
2020-09-22 13:30:41 +00:00
return null ;
}
} ;
2023-03-05 00:52:12 +00:00
/ * *
2024-01-25 16:46:02 +00:00
* Takes an environment variable that should be an integer , attempts to parse
* it falling back to a default if not set , and handles errors parsing .
* @ param { string | undefined } value
* @ param { number } defaultValue
* @ param { string } variableName
* @ returns { number }
* /
const parseIntFromEnv = ( value , defaultValue , variableName ) => {
if ( typeof value === 'string' && value . length > 0 ) {
const parsedValue = parseInt ( value , 10 ) ;
if ( isNaN ( parsedValue ) ) {
throw new Error ( ` Invalid ${ variableName } environment variable: ${ value } ` ) ;
}
return parsedValue ;
} else {
return defaultValue ;
}
} ;
/ * *
* @ param { NodeJS . ProcessEnv } env the ` process.env ` value to read configuration from
* @ returns { pg . PoolConfig } the configuration for the PostgreSQL connection
2023-03-05 00:52:12 +00:00
* /
2023-04-26 09:37:51 +00:00
const pgConfigFromEnv = ( env ) => {
2024-01-25 16:46:02 +00:00
/** @type {Record<string, pg.PoolConfig>} */
2017-04-17 02:32:30 +00:00
const pgConfigs = {
development : {
2024-01-25 16:46:02 +00:00
user : env . DB _USER || pg . defaults . user ,
2023-04-26 09:37:51 +00:00
password : env . DB _PASS || pg . defaults . password ,
database : env . DB _NAME || 'mastodon_development' ,
2024-01-25 16:46:02 +00:00
host : env . DB _HOST || pg . defaults . host ,
port : parseIntFromEnv ( env . DB _PORT , pg . defaults . port ? ? 5432 , 'DB_PORT' )
2017-04-17 02:32:30 +00:00
} ,
production : {
2024-01-25 16:46:02 +00:00
user : env . DB _USER || 'mastodon' ,
2023-04-26 09:37:51 +00:00
password : env . DB _PASS || '' ,
database : env . DB _NAME || 'mastodon_production' ,
2024-01-25 16:46:02 +00:00
host : env . DB _HOST || 'localhost' ,
port : parseIntFromEnv ( env . DB _PORT , 5432 , 'DB_PORT' )
2017-05-20 15:31:47 +00:00
} ,
} ;
2017-02-02 00:31:09 +00:00
2024-01-25 16:46:02 +00:00
/ * *
* @ type { pg . PoolConfig }
* /
let baseConfig = { } ;
2020-08-11 16:24:59 +00:00
2023-04-26 09:37:51 +00:00
if ( env . DATABASE _URL ) {
2024-02-27 14:59:20 +00:00
const parsedUrl = pgConnectionString . parse ( env . DATABASE _URL ) ;
2024-01-25 16:46:02 +00:00
// The result of dbUrlToConfig from pg-connection-string is not type
// compatible with pg.PoolConfig, since parts of the connection URL may be
// `null` when pg.PoolConfig expects `undefined`, as such we have to
// manually create the baseConfig object from the properties of the
// parsedUrl.
//
// For more information see:
// https://github.com/brianc/node-postgres/issues/2280
//
// FIXME: clean up once brianc/node-postgres#3128 lands
if ( typeof parsedUrl . password === 'string' ) baseConfig . password = parsedUrl . password ;
if ( typeof parsedUrl . host === 'string' ) baseConfig . host = parsedUrl . host ;
if ( typeof parsedUrl . user === 'string' ) baseConfig . user = parsedUrl . user ;
if ( typeof parsedUrl . port === 'string' ) {
const parsedPort = parseInt ( parsedUrl . port , 10 ) ;
if ( isNaN ( parsedPort ) ) {
throw new Error ( 'Invalid port specified in DATABASE_URL environment variable' ) ;
}
baseConfig . port = parsedPort ;
}
if ( typeof parsedUrl . database === 'string' ) baseConfig . database = parsedUrl . database ;
if ( typeof parsedUrl . options === 'string' ) baseConfig . options = parsedUrl . options ;
// The pg-connection-string type definition isn't correct, as parsedUrl.ssl
// can absolutely be an Object, this is to work around these incorrect
// types, including the casting of parsedUrl.ssl to Record<string, any>
if ( typeof parsedUrl . ssl === 'boolean' ) {
baseConfig . ssl = parsedUrl . ssl ;
} else if ( typeof parsedUrl . ssl === 'object' && ! Array . isArray ( parsedUrl . ssl ) && parsedUrl . ssl !== null ) {
/** @type {Record<string, any>} */
const sslOptions = parsedUrl . ssl ;
baseConfig . ssl = { } ;
baseConfig . ssl . cert = sslOptions . cert ;
baseConfig . ssl . key = sslOptions . key ;
baseConfig . ssl . ca = sslOptions . ca ;
baseConfig . ssl . rejectUnauthorized = sslOptions . rejectUnauthorized ;
}
2023-08-18 13:05:35 +00:00
// Support overriding the database password in the connection URL
if ( ! baseConfig . password && env . DB _PASS ) {
baseConfig . password = env . DB _PASS ;
}
2024-02-27 15:42:05 +00:00
} else if ( Object . hasOwn ( pgConfigs , environment ) ) {
2023-04-26 09:37:51 +00:00
baseConfig = pgConfigs [ environment ] ;
2023-03-05 00:52:12 +00:00
2023-04-26 09:37:51 +00:00
if ( env . DB _SSLMODE ) {
switch ( env . DB _SSLMODE ) {
2023-03-05 00:52:12 +00:00
case 'disable' :
case '' :
baseConfig . ssl = false ;
break ;
case 'no-verify' :
baseConfig . ssl = { rejectUnauthorized : false } ;
break ;
default :
baseConfig . ssl = { } ;
break ;
}
}
2024-01-25 16:46:02 +00:00
} else {
throw new Error ( 'Unable to resolve postgresql database configuration.' ) ;
2023-03-05 00:52:12 +00:00
}
2017-12-12 14:13:24 +00:00
2023-03-05 00:52:12 +00:00
return {
... baseConfig ,
2024-01-25 16:46:02 +00:00
max : parseIntFromEnv ( env . DB _POOL , 10 , 'DB_POOL' ) ,
2023-02-09 10:20:59 +00:00
connectionTimeoutMillis : 15000 ,
2024-01-25 16:46:02 +00:00
// Deliberately set application_name to an empty string to prevent excessive
// CPU usage with PG Bouncer. See:
// - https://github.com/mastodon/mastodon/pull/23958
// - https://github.com/pgbouncer/pgbouncer/issues/349
2023-03-05 00:52:12 +00:00
application _name : '' ,
} ;
} ;
2023-04-26 09:37:51 +00:00
/ * *
2024-01-25 16:46:02 +00:00
* @ typedef RedisConfiguration
* @ property { import ( 'ioredis' ) . RedisOptions } redisParams
* @ property { string } redisPrefix
* @ property { string | undefined } redisUrl
* /
/ * *
* @ param { NodeJS . ProcessEnv } env the ` process.env ` value to read configuration from
* @ returns { RedisConfiguration } configuration for the Redis connection
2023-04-26 09:37:51 +00:00
* /
const redisConfigFromEnv = ( env ) => {
2023-09-01 15:44:28 +00:00
// ioredis *can* transparently add prefixes for us, but it doesn't *in some cases*,
// which means we can't use it. But this is something that should be looked into.
const redisPrefix = env . REDIS _NAMESPACE ? ` ${ env . REDIS _NAMESPACE } : ` : '' ;
2017-02-07 13:37:12 +00:00
2024-01-25 16:46:02 +00:00
let redisPort = parseIntFromEnv ( env . REDIS _PORT , 6379 , 'REDIS_PORT' ) ;
let redisDatabase = parseIntFromEnv ( env . REDIS _DB , 0 , 'REDIS_DB' ) ;
/** @type {import('ioredis').RedisOptions} */
2017-05-07 17:42:32 +00:00
const redisParams = {
2023-09-01 15:44:28 +00:00
host : env . REDIS _HOST || '127.0.0.1' ,
2024-01-25 16:46:02 +00:00
port : redisPort ,
db : redisDatabase ,
2023-04-26 09:37:51 +00:00
password : env . REDIS _PASSWORD || undefined ,
2017-05-20 15:31:47 +00:00
} ;
2017-05-07 17:42:32 +00:00
2023-09-01 15:44:28 +00:00
// redisParams.path takes precedence over host and port.
if ( env . REDIS _URL && env . REDIS _URL . startsWith ( 'unix://' ) ) {
redisParams . path = env . REDIS _URL . slice ( 7 ) ;
2017-05-07 17:42:32 +00:00
}
2017-05-20 19:06:09 +00:00
2023-04-26 09:37:51 +00:00
return {
redisParams ,
redisPrefix ,
2024-01-25 16:46:02 +00:00
redisUrl : typeof env . REDIS _URL === 'string' ? env . REDIS _URL : undefined ,
2023-04-26 09:37:51 +00:00
} ;
} ;
2023-09-19 10:25:30 +00:00
const PUBLIC _CHANNELS = [
'public' ,
'public:media' ,
'public:local' ,
'public:local:media' ,
'public:remote' ,
'public:remote:media' ,
'hashtag' ,
'hashtag:local' ,
] ;
// Used for priming the counters/gauges for the various metrics that are
// per-channel
const CHANNEL _NAMES = [
'system' ,
'user' ,
'user:notification' ,
'list' ,
'direct' ,
... PUBLIC _CHANNELS
] ;
2023-04-26 09:37:51 +00:00
const startServer = async ( ) => {
2024-01-15 10:36:30 +00:00
const pgPool = new pg . Pool ( pgConfigFromEnv ( process . env ) ) ;
const server = http . createServer ( ) ;
2024-06-24 10:29:00 +00:00
const wss = new WebSocketServer ( { noServer : true } ) ;
2024-01-15 10:36:30 +00:00
// Set the X-Request-Id header on WebSockets:
wss . on ( "headers" , function onHeaders ( headers , req ) {
headers . push ( ` X-Request-Id: ${ req . id } ` ) ;
} ) ;
2023-04-26 09:37:51 +00:00
const app = express ( ) ;
app . set ( 'trust proxy' , process . env . TRUSTED _PROXY _IP ? process . env . TRUSTED _PROXY _IP . split ( /(?:\s*,\s*|\s+)/ ) : 'loopback,uniquelocal' ) ;
2024-01-18 18:40:25 +00:00
app . use ( httpLogger ) ;
2024-01-04 09:18:03 +00:00
app . use ( cors ( ) ) ;
2023-04-26 09:37:51 +00:00
2024-01-15 10:36:30 +00:00
// Handle eventsource & other http requests:
server . on ( 'request' , app ) ;
// Handle upgrade requests:
server . on ( 'upgrade' , async function handleUpgrade ( request , socket , head ) {
2024-01-18 18:40:25 +00:00
// Setup the HTTP logger, since websocket upgrades don't get the usual http
// logger. This decorates the `request` object.
attachWebsocketHttpLogger ( request ) ;
request . log . info ( "HTTP Upgrade Requested" ) ;
2024-01-15 10:36:30 +00:00
/** @param {Error} err */
const onSocketError = ( err ) => {
2024-01-18 18:40:25 +00:00
request . log . error ( { error : err } , err . message ) ;
2024-01-15 10:36:30 +00:00
} ;
socket . on ( 'error' , onSocketError ) ;
2024-01-18 18:40:25 +00:00
/** @type {ResolvedAccount} */
let resolvedAccount ;
2024-01-15 10:36:30 +00:00
try {
2024-01-18 18:40:25 +00:00
resolvedAccount = await accountFromRequest ( request ) ;
2024-01-15 10:36:30 +00:00
} catch ( err ) {
// Unfortunately for using the on('upgrade') setup, we need to manually
// write a HTTP Response to the Socket to close the connection upgrade
// attempt, so the following code is to handle all of that.
2024-02-27 14:59:20 +00:00
const { statusCode , errorMessage } = extractErrorStatusAndMessage ( err ) ;
2024-01-15 10:36:30 +00:00
2024-01-18 18:40:25 +00:00
/** @type {Record<string, string | number | import('pino-http').ReqId>} */
2024-01-15 10:36:30 +00:00
const headers = {
'Connection' : 'close' ,
'Content-Type' : 'text/plain' ,
'Content-Length' : 0 ,
'X-Request-Id' : request . id ,
2024-02-22 13:20:20 +00:00
'X-Error-Message' : errorMessage
2024-01-15 10:36:30 +00:00
} ;
// Ensure the socket is closed once we've finished writing to it:
socket . once ( 'finish' , ( ) => {
socket . destroy ( ) ;
} ) ;
// Write the HTTP response manually:
socket . end ( ` HTTP/1.1 ${ statusCode } ${ http . STATUS _CODES [ statusCode ] } \r \n ${ Object . keys ( headers ) . map ( ( key ) => ` ${ key } : ${ headers [ key ] } ` ) . join ( '\r\n' ) } \r \n \r \n ` ) ;
2024-01-18 18:40:25 +00:00
// Finally, log the error:
request . log . error ( {
err ,
res : {
statusCode ,
headers
}
2024-02-22 13:20:20 +00:00
} , errorMessage ) ;
2024-01-18 18:40:25 +00:00
2024-01-15 10:36:30 +00:00
return ;
}
2024-01-18 18:40:25 +00:00
// Remove the error handler, wss.handleUpgrade has its own:
socket . removeListener ( 'error' , onSocketError ) ;
2024-01-15 10:36:30 +00:00
wss . handleUpgrade ( request , socket , head , function done ( ws ) {
2024-01-18 18:40:25 +00:00
request . log . info ( "Authenticated request & upgraded to WebSocket connection" ) ;
const wsLogger = createWebsocketLogger ( request , resolvedAccount ) ;
2024-01-15 10:36:30 +00:00
// Start the connection:
2024-01-18 18:40:25 +00:00
wss . emit ( 'connection' , ws , request , wsLogger ) ;
2024-01-15 10:36:30 +00:00
} ) ;
} ) ;
2022-03-21 18:08:29 +00:00
/ * *
2023-06-09 17:29:16 +00:00
* @ type { Object . < string , Array . < function ( Object < string , any > ) : void >> }
2022-03-21 18:08:29 +00:00
* /
const subs = { } ;
2023-09-01 15:44:28 +00:00
const redisConfig = redisConfigFromEnv ( process . env ) ;
const redisSubscribeClient = await createRedisClient ( redisConfig ) ;
const redisClient = await createRedisClient ( redisConfig ) ;
const { redisPrefix } = redisConfig ;
2017-02-07 13:37:12 +00:00
2023-11-28 14:24:41 +00:00
const metrics = setupMetrics ( CHANNEL _NAMES , pgPool ) ;
// TODO: migrate all metrics to metrics.X.method() instead of just X.method()
const {
connectedClients ,
connectedChannels ,
redisSubscriptions ,
redisMessagesReceived ,
messagesSent ,
} = metrics ;
2023-09-19 10:25:30 +00:00
2023-08-04 14:11:30 +00:00
// When checking metrics in the browser, the favicon is requested this
// prevents the request from falling through to the API Router, which would
// error for this endpoint:
2024-01-18 18:40:25 +00:00
app . get ( '/favicon.ico' , ( _req , res ) => res . status ( 404 ) . end ( ) ) ;
2023-08-04 14:11:30 +00:00
2024-01-18 18:40:25 +00:00
app . get ( '/api/v1/streaming/health' , ( _req , res ) => {
2023-08-04 14:11:30 +00:00
res . writeHead ( 200 , { 'Content-Type' : 'text/plain' } ) ;
res . end ( 'OK' ) ;
} ) ;
app . get ( '/metrics' , async ( req , res ) => {
try {
res . set ( 'Content-Type' , metrics . register . contentType ) ;
res . end ( await metrics . register . metrics ( ) ) ;
} catch ( ex ) {
2024-01-18 18:40:25 +00:00
req . log . error ( ex ) ;
2023-08-04 14:11:30 +00:00
res . status ( 500 ) . end ( ) ;
}
} ) ;
2020-08-11 16:24:59 +00:00
/ * *
* @ param { string [ ] } channels
2023-04-30 00:29:54 +00:00
* @ returns { function ( ) : void }
2020-08-11 16:24:59 +00:00
* /
2020-06-02 17:24:53 +00:00
const subscriptionHeartbeat = channels => {
const interval = 6 * 60 ;
2017-06-03 18:50:53 +00:00
const tellSubscribed = ( ) => {
2020-06-02 17:24:53 +00:00
channels . forEach ( channel => redisClient . set ( ` ${ redisPrefix } subscribed: ${ channel } ` , '1' , 'EX' , interval * 3 ) ) ;
2017-06-03 18:50:53 +00:00
} ;
2020-06-02 17:24:53 +00:00
2017-06-03 18:50:53 +00:00
tellSubscribed ( ) ;
2020-06-02 17:24:53 +00:00
const heartbeat = setInterval ( tellSubscribed , interval * 1000 ) ;
2017-06-03 18:50:53 +00:00
return ( ) => {
clearInterval ( heartbeat ) ;
} ;
} ;
2017-02-07 13:37:12 +00:00
2022-03-21 18:08:29 +00:00
/ * *
* @ param { string } channel
2023-09-01 15:44:28 +00:00
* @ param { string } message
2022-03-21 18:08:29 +00:00
* /
2023-09-01 15:44:28 +00:00
const onRedisMessage = ( channel , message ) => {
2023-09-19 10:25:30 +00:00
redisMessagesReceived . inc ( ) ;
2022-03-21 18:08:29 +00:00
const callbacks = subs [ channel ] ;
2024-01-18 18:40:25 +00:00
logger . debug ( ` New message on channel ${ redisPrefix } ${ channel } ` ) ;
2022-03-21 18:08:29 +00:00
if ( ! callbacks ) {
return ;
}
2023-06-09 17:29:16 +00:00
const json = parseJSON ( message , null ) ;
if ( ! json ) return ;
callbacks . forEach ( callback => callback ( json ) ) ;
2022-03-21 18:08:29 +00:00
} ;
2023-09-01 15:44:28 +00:00
redisSubscribeClient . on ( "message" , onRedisMessage ) ;
2022-03-21 18:08:29 +00:00
2023-07-28 10:06:29 +00:00
/ * *
* @ callback SubscriptionListener
* @ param { ReturnType < parseJSON > } json of the message
* @ returns void
* /
2020-08-11 16:24:59 +00:00
/ * *
* @ param { string } channel
2023-07-28 10:06:29 +00:00
* @ param { SubscriptionListener } callback
2020-08-11 16:24:59 +00:00
* /
2017-04-17 02:32:30 +00:00
const subscribe = ( channel , callback ) => {
2024-01-18 18:40:25 +00:00
logger . debug ( ` Adding listener for ${ channel } ` ) ;
2020-08-11 16:24:59 +00:00
2022-03-21 18:08:29 +00:00
subs [ channel ] = subs [ channel ] || [ ] ;
if ( subs [ channel ] . length === 0 ) {
2024-01-18 18:40:25 +00:00
logger . debug ( ` Subscribe ${ channel } ` ) ;
2023-09-01 15:44:28 +00:00
redisSubscribeClient . subscribe ( channel , ( err , count ) => {
if ( err ) {
2024-01-18 18:40:25 +00:00
logger . error ( ` Error subscribing to ${ channel } ` ) ;
} else if ( typeof count === 'number' ) {
2023-09-01 15:44:28 +00:00
redisSubscriptions . set ( count ) ;
}
} ) ;
2022-03-21 18:08:29 +00:00
}
subs [ channel ] . push ( callback ) ;
2017-05-20 15:31:47 +00:00
} ;
2017-02-03 17:27:42 +00:00
2020-08-11 16:24:59 +00:00
/ * *
* @ param { string } channel
2023-07-28 10:06:29 +00:00
* @ param { SubscriptionListener } callback
2020-08-11 16:24:59 +00:00
* /
2022-01-07 18:50:12 +00:00
const unsubscribe = ( channel , callback ) => {
2024-01-18 18:40:25 +00:00
logger . debug ( ` Removing listener for ${ channel } ` ) ;
2020-08-11 16:24:59 +00:00
2022-03-21 18:08:29 +00:00
if ( ! subs [ channel ] ) {
return ;
}
subs [ channel ] = subs [ channel ] . filter ( item => item !== callback ) ;
if ( subs [ channel ] . length === 0 ) {
2024-01-18 18:40:25 +00:00
logger . debug ( ` Unsubscribe ${ channel } ` ) ;
2023-09-01 15:44:28 +00:00
redisSubscribeClient . unsubscribe ( channel , ( err , count ) => {
if ( err ) {
2024-01-18 18:40:25 +00:00
logger . error ( ` Error unsubscribing to ${ channel } ` ) ;
} else if ( typeof count === 'number' ) {
2023-09-01 15:44:28 +00:00
redisSubscriptions . set ( count ) ;
}
} ) ;
2022-03-21 18:08:29 +00:00
delete subs [ channel ] ;
}
2017-05-20 15:31:47 +00:00
} ;
2017-02-03 17:27:42 +00:00
2020-08-11 16:24:59 +00:00
/ * *
2024-01-18 18:40:25 +00:00
* @ param { http . IncomingMessage & ResolvedAccount } req
2021-09-26 11:23:28 +00:00
* @ param { string [ ] } necessaryScopes
2023-04-30 00:29:54 +00:00
* @ returns { boolean }
2021-09-26 11:23:28 +00:00
* /
const isInScope = ( req , necessaryScopes ) =>
req . scopes . some ( scope => necessaryScopes . includes ( scope ) ) ;
2020-08-11 16:24:59 +00:00
/ * *
* @ param { string } token
* @ param { any } req
2024-01-15 10:36:30 +00:00
* @ returns { Promise < ResolvedAccount > }
2020-08-11 16:24:59 +00:00
* /
2024-07-12 12:23:09 +00:00
const accountFromToken = async ( token , req ) => {
const result = await pgPool . query ( 'SELECT oauth_access_tokens.id, oauth_access_tokens.resource_owner_id, users.account_id, users.chosen_languages, oauth_access_tokens.scopes, devices.device_id FROM oauth_access_tokens INNER JOIN users ON oauth_access_tokens.resource_owner_id = users.id LEFT OUTER JOIN devices ON oauth_access_tokens.id = devices.access_token_id WHERE oauth_access_tokens.token = $1 AND oauth_access_tokens.revoked_at IS NULL LIMIT 1' , [ token ] ) ;
2017-02-02 00:31:09 +00:00
2024-07-12 12:23:09 +00:00
if ( result . rows . length === 0 ) {
throw new AuthenticationError ( 'Invalid access token' ) ;
}
2019-05-24 13:21:42 +00:00
2024-07-12 12:23:09 +00:00
req . accessTokenId = result . rows [ 0 ] . id ;
req . scopes = result . rows [ 0 ] . scopes . split ( ' ' ) ;
req . accountId = result . rows [ 0 ] . account _id ;
req . chosenLanguages = result . rows [ 0 ] . chosen _languages ;
req . deviceId = result . rows [ 0 ] . device _id ;
return {
accessTokenId : result . rows [ 0 ] . id ,
scopes : result . rows [ 0 ] . scopes . split ( ' ' ) ,
accountId : result . rows [ 0 ] . account _id ,
chosenLanguages : result . rows [ 0 ] . chosen _languages ,
deviceId : result . rows [ 0 ] . device _id
} ;
} ;
2017-02-03 23:34:31 +00:00
2020-08-11 16:24:59 +00:00
/ * *
* @ param { any } req
2024-01-15 10:36:30 +00:00
* @ returns { Promise < ResolvedAccount > }
2020-08-11 16:24:59 +00:00
* /
2023-03-06 20:00:10 +00:00
const accountFromRequest = ( req ) => new Promise ( ( resolve , reject ) => {
2017-05-29 16:20:53 +00:00
const authorization = req . headers . authorization ;
2020-08-11 16:24:59 +00:00
const location = url . parse ( req . url , true ) ;
const accessToken = location . query . access _token || req . headers [ 'sec-websocket-protocol' ] ;
2017-02-03 23:34:31 +00:00
2017-05-21 19:13:11 +00:00
if ( ! authorization && ! accessToken ) {
2024-02-22 13:20:20 +00:00
reject ( new AuthenticationError ( 'Missing access token' ) ) ;
2023-03-06 20:00:10 +00:00
return ;
2017-04-17 02:32:30 +00:00
}
2017-02-02 16:10:59 +00:00
2017-05-21 19:13:11 +00:00
const token = authorization ? authorization . replace ( /^Bearer / , '' ) : accessToken ;
2017-02-02 12:56:14 +00:00
2020-08-11 16:24:59 +00:00
resolve ( accountFromToken ( token , req ) ) ;
} ) ;
/ * *
* @ param { any } req
2023-06-09 17:29:16 +00:00
* @ returns { string | undefined }
2020-08-11 16:24:59 +00:00
* /
const channelNameFromPath = req => {
const { path , query } = req ;
const onlyMedia = isTruthy ( query . only _media ) ;
2021-12-25 21:55:06 +00:00
switch ( path ) {
2020-08-11 16:24:59 +00:00
case '/api/v1/streaming/user' :
return 'user' ;
case '/api/v1/streaming/user/notification' :
return 'user:notification' ;
case '/api/v1/streaming/public' :
return onlyMedia ? 'public:media' : 'public' ;
case '/api/v1/streaming/public/local' :
return onlyMedia ? 'public:local:media' : 'public:local' ;
case '/api/v1/streaming/public/remote' :
return onlyMedia ? 'public:remote:media' : 'public:remote' ;
case '/api/v1/streaming/hashtag' :
return 'hashtag' ;
case '/api/v1/streaming/hashtag/local' :
return 'hashtag:local' ;
case '/api/v1/streaming/direct' :
return 'direct' ;
case '/api/v1/streaming/list' :
return 'list' ;
2020-11-23 16:35:14 +00:00
default :
return undefined ;
2020-08-11 16:24:59 +00:00
}
2017-05-20 15:31:47 +00:00
} ;
2017-02-05 02:19:04 +00:00
2020-08-11 16:24:59 +00:00
/ * *
2024-01-18 18:40:25 +00:00
* @ param { http . IncomingMessage & ResolvedAccount } req
* @ param { import ( 'pino' ) . Logger } logger
2023-08-04 14:11:30 +00:00
* @ param { string | undefined } channelName
2023-04-30 00:29:54 +00:00
* @ returns { Promise . < void > }
2020-08-11 16:24:59 +00:00
* /
2024-01-18 18:40:25 +00:00
const checkScopes = ( req , logger , channelName ) => new Promise ( ( resolve , reject ) => {
logger . debug ( ` Checking OAuth scopes for ${ channelName } ` ) ;
2020-08-11 16:24:59 +00:00
// When accessing public channels, no scopes are needed
2024-01-18 18:40:25 +00:00
if ( channelName && PUBLIC _CHANNELS . includes ( channelName ) ) {
2020-08-11 16:24:59 +00:00
resolve ( ) ;
return ;
}
2019-05-24 13:21:42 +00:00
2020-08-11 16:24:59 +00:00
// The `read` scope has the highest priority, if the token has it
// then it can access all streams
const requiredScopes = [ 'read' ] ;
// When accessing specifically the notifications stream,
// we need a read:notifications, while in all other cases,
// we can allow access with read:statuses. Mind that the
// user stream will not contain notifications unless
// the token has either read or read:notifications scope
// as well, this is handled separately.
if ( channelName === 'user:notification' ) {
requiredScopes . push ( 'read:notifications' ) ;
} else {
requiredScopes . push ( 'read:statuses' ) ;
2019-05-24 13:21:42 +00:00
}
2017-12-12 14:13:24 +00:00
2021-10-13 03:02:55 +00:00
if ( req . scopes && requiredScopes . some ( requiredScope => req . scopes . includes ( requiredScope ) ) ) {
2020-08-11 16:24:59 +00:00
resolve ( ) ;
return ;
}
2017-05-29 16:20:53 +00:00
2024-02-22 13:20:20 +00:00
reject ( new AuthenticationError ( 'Access token does not have the required scopes' ) ) ;
2020-08-11 16:24:59 +00:00
} ) ;
2017-12-12 14:13:24 +00:00
2020-11-12 22:05:24 +00:00
/ * *
* @ typedef SystemMessageHandlers
* @ property { function ( ) : void } onKill
* /
/ * *
* @ param { any } req
* @ param { SystemMessageHandlers } eventHandlers
2024-01-18 18:40:25 +00:00
* @ returns { SubscriptionListener }
2020-11-12 22:05:24 +00:00
* /
const createSystemMessageListener = ( req , eventHandlers ) => {
return message => {
2024-01-18 18:40:25 +00:00
if ( ! message ? . event ) {
return ;
}
2023-06-09 17:29:16 +00:00
const { event } = message ;
2020-11-12 22:05:24 +00:00
2024-01-18 18:40:25 +00:00
req . log . debug ( ` System message for ${ req . accountId } : ${ event } ` ) ;
2020-11-12 22:05:24 +00:00
if ( event === 'kill' ) {
2024-01-18 18:40:25 +00:00
req . log . debug ( ` Closing connection for ${ req . accountId } due to expired access token ` ) ;
2020-11-12 22:05:24 +00:00
eventHandlers . onKill ( ) ;
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 07:42:13 +00:00
} else if ( event === 'filters_changed' ) {
2024-01-18 18:40:25 +00:00
req . log . debug ( ` Invalidating filters cache for ${ req . accountId } ` ) ;
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 07:42:13 +00:00
req . cachedFilters = null ;
2020-11-12 22:05:24 +00:00
}
2020-11-23 16:35:14 +00:00
} ;
2020-11-12 22:05:24 +00:00
} ;
/ * *
2024-01-18 18:40:25 +00:00
* @ param { http . IncomingMessage & ResolvedAccount } req
* @ param { http . OutgoingMessage } res
2020-11-12 22:05:24 +00:00
* /
const subscribeHttpToSystemChannel = ( req , res ) => {
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 07:42:13 +00:00
const accessTokenChannelId = ` timeline:access_token: ${ req . accessTokenId } ` ;
const systemChannelId = ` timeline:system: ${ req . accountId } ` ;
2020-11-12 22:05:24 +00:00
const listener = createSystemMessageListener ( req , {
2021-12-25 21:55:06 +00:00
onKill ( ) {
2020-11-12 22:05:24 +00:00
res . end ( ) ;
} ,
} ) ;
res . on ( 'close' , ( ) => {
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 07:42:13 +00:00
unsubscribe ( ` ${ redisPrefix } ${ accessTokenChannelId } ` , listener ) ;
2020-11-12 22:05:24 +00:00
unsubscribe ( ` ${ redisPrefix } ${ systemChannelId } ` , listener ) ;
2023-08-04 14:11:30 +00:00
connectedChannels . labels ( { type : 'eventsource' , channel : 'system' } ) . dec ( 2 ) ;
2020-11-12 22:05:24 +00:00
} ) ;
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 07:42:13 +00:00
subscribe ( ` ${ redisPrefix } ${ accessTokenChannelId } ` , listener ) ;
2020-11-12 22:05:24 +00:00
subscribe ( ` ${ redisPrefix } ${ systemChannelId } ` , listener ) ;
2023-08-04 14:11:30 +00:00
connectedChannels . labels ( { type : 'eventsource' , channel : 'system' } ) . inc ( 2 ) ;
2020-11-12 22:05:24 +00:00
} ;
2020-08-11 16:24:59 +00:00
/ * *
* @ param { any } req
* @ param { any } res
* @ param { function ( Error = ) : void } next
* /
2017-05-29 16:20:53 +00:00
const authenticationMiddleware = ( req , res , next ) => {
if ( req . method === 'OPTIONS' ) {
next ( ) ;
return ;
}
2023-08-04 14:11:30 +00:00
const channelName = channelNameFromPath ( req ) ;
// If no channelName can be found for the request, then we should terminate
// the connection, as there's nothing to stream back
if ( ! channelName ) {
2024-02-22 13:20:20 +00:00
next ( new RequestError ( 'Unknown channel requested' ) ) ;
2023-08-04 14:11:30 +00:00
return ;
}
2024-01-18 18:40:25 +00:00
accountFromRequest ( req ) . then ( ( ) => checkScopes ( req , req . log , channelName ) ) . then ( ( ) => {
2020-11-12 22:05:24 +00:00
subscribeHttpToSystemChannel ( req , res ) ;
} ) . then ( ( ) => {
2020-08-11 16:24:59 +00:00
next ( ) ;
} ) . catch ( err => {
next ( err ) ;
} ) ;
2017-05-29 16:20:53 +00:00
} ;
2020-08-11 16:24:59 +00:00
/ * *
* @ param { Error } err
* @ param { any } req
* @ param { any } res
* @ param { function ( Error = ) : void } next
* /
const errorMiddleware = ( err , req , res , next ) => {
2024-01-18 18:40:25 +00:00
req . log . error ( { err } , err . toString ( ) ) ;
2020-08-11 16:24:59 +00:00
if ( res . headersSent ) {
2020-11-23 16:35:14 +00:00
next ( err ) ;
return ;
2020-08-11 16:24:59 +00:00
}
2024-02-27 14:59:20 +00:00
const { statusCode , errorMessage } = extractErrorStatusAndMessage ( err ) ;
2024-01-18 18:40:25 +00:00
res . writeHead ( statusCode , { 'Content-Type' : 'application/json' } ) ;
res . end ( JSON . stringify ( { error : errorMessage } ) ) ;
2017-05-20 15:31:47 +00:00
} ;
2017-02-05 02:19:04 +00:00
2020-08-11 16:24:59 +00:00
/ * *
2024-01-18 18:40:25 +00:00
* @ param { any [ ] } arr
2020-08-11 16:24:59 +00:00
* @ param { number = } shift
2023-04-30 00:29:54 +00:00
* @ returns { string }
2020-08-11 16:24:59 +00:00
* /
2024-01-18 18:40:25 +00:00
// @ts-ignore
2017-04-17 02:32:30 +00:00
const placeholders = ( arr , shift = 0 ) => arr . map ( ( _ , i ) => ` $ ${ i + 1 + shift } ` ) . join ( ', ' ) ;
2017-02-02 00:31:09 +00:00
2020-08-11 16:24:59 +00:00
/ * *
* @ param { string } listId
* @ param { any } req
2023-04-30 00:29:54 +00:00
* @ returns { Promise . < void > }
2020-08-11 16:24:59 +00:00
* /
2024-07-12 12:23:09 +00:00
const authorizeListAccess = async ( listId , req ) => {
2020-08-11 16:24:59 +00:00
const { accountId } = req ;
2024-07-12 12:23:09 +00:00
const result = await pgPool . query ( 'SELECT id, account_id FROM lists WHERE id = $1 AND account_id = $2 LIMIT 1' , [ listId , accountId ] ) ;
2017-11-17 23:16:48 +00:00
2024-07-12 12:23:09 +00:00
if ( result . rows . length === 0 ) {
throw new AuthenticationError ( 'List not found' ) ;
}
} ;
2017-11-17 23:16:48 +00:00
2020-08-11 16:24:59 +00:00
/ * *
2024-01-18 18:40:25 +00:00
* @ param { string [ ] } channelIds
* @ param { http . IncomingMessage & ResolvedAccount } req
* @ param { import ( 'pino' ) . Logger } log
2020-08-11 16:24:59 +00:00
* @ param { function ( string , string ) : void } output
2023-07-28 10:06:29 +00:00
* @ param { undefined | function ( string [ ] , SubscriptionListener ) : void } attachCloseHandler
2023-09-19 10:25:30 +00:00
* @ param { 'websocket' | 'eventsource' } destinationType
2020-08-11 16:24:59 +00:00
* @ param { boolean = } needsFiltering
2020-08-11 17:19:27 +00:00
* @ param { boolean = } allowLocalOnly
2023-07-28 10:06:29 +00:00
* @ returns { SubscriptionListener }
2020-08-11 16:24:59 +00:00
* /
2024-01-22 18:01:35 +00:00
const streamFrom = ( channelIds , req , log , output , attachCloseHandler , destinationType , needsFiltering = false , allowLocalOnly = false ) => {
2024-01-18 18:40:25 +00:00
log . info ( { channelIds } , ` Starting stream ` ) ;
2017-04-17 02:32:30 +00:00
2024-01-18 18:40:25 +00:00
/ * *
* @ param { string } event
* @ param { object | string } payload
* /
2023-07-27 13:38:18 +00:00
const transmit = ( event , payload ) => {
// TODO: Replace "string"-based delete payloads with object payloads:
const encodedPayload = typeof payload === 'object' ? JSON . stringify ( payload ) : payload ;
2017-02-02 12:56:14 +00:00
2023-09-19 10:25:30 +00:00
messagesSent . labels ( { type : destinationType } ) . inc ( 1 ) ;
2024-01-18 18:40:25 +00:00
log . debug ( { event , payload } , ` Transmitting ${ event } to ${ req . accountId } ` ) ;
2023-07-27 13:38:18 +00:00
output ( event , encodedPayload ) ;
} ;
2017-02-02 12:56:14 +00:00
2023-07-27 13:38:18 +00:00
// The listener used to process each message off the redis subscription,
// message here is an object with an `event` and `payload` property. Some
// events also include a queued_at value, but this is being removed shortly.
2024-01-18 18:40:25 +00:00
2023-07-28 10:06:29 +00:00
/** @type {SubscriptionListener} */
2023-07-27 13:38:18 +00:00
const listener = message => {
2024-01-18 18:40:25 +00:00
if ( ! message ? . event || ! message ? . payload ) {
return ;
}
2023-07-27 13:38:18 +00:00
const { event , payload } = message ;
2017-02-02 12:56:14 +00:00
2018-11-13 16:30:15 +00:00
// Only send local-only statuses to logged-in users
2023-07-30 14:11:55 +00:00
if ( ( event === 'update' || event === 'status.update' ) && payload . local _only && ! ( req . accountId && allowLocalOnly ) ) {
2024-02-20 19:19:58 +00:00
log . debug ( ` Message ${ payload . id } filtered because it was local-only ` ) ;
2018-11-13 16:30:15 +00:00
return ;
}
2023-07-27 13:38:18 +00:00
// Streaming only needs to apply filtering to some channels and only to
// some events. This is because majority of the filtering happens on the
// Ruby on Rails side when producing the event for streaming.
//
// The only events that require filtering from the streaming server are
// `update` and `status.update`, all other events are transmitted to the
// client as soon as they're received (pass-through).
//
// The channels that need filtering are determined in the function
// `channelNameToIds` defined below:
if ( ! needsFiltering || ( event !== 'update' && event !== 'status.update' ) ) {
transmit ( event , payload ) ;
2018-04-17 11:49:09 +00:00
return ;
}
2017-02-02 12:56:14 +00:00
2023-07-27 13:38:18 +00:00
// The rest of the logic from here on in this function is to handle
// filtering of statuses:
2017-04-17 02:32:30 +00:00
2023-07-27 13:38:18 +00:00
// Filter based on language:
2023-07-27 13:12:10 +00:00
if ( Array . isArray ( req . chosenLanguages ) && payload . language !== null && req . chosenLanguages . indexOf ( payload . language ) === - 1 ) {
2024-01-18 18:40:25 +00:00
log . debug ( ` Message ${ payload . id } filtered by language ( ${ payload . language } ) ` ) ;
2018-04-17 11:49:09 +00:00
return ;
}
// When the account is not logged in, it is not necessary to confirm the block or mute
if ( ! req . accountId ) {
2023-07-27 13:38:18 +00:00
transmit ( event , payload ) ;
2018-04-17 11:49:09 +00:00
return ;
}
2023-07-27 13:38:18 +00:00
// Filter based on domain blocks, blocks, mutes, or custom filters:
2024-01-18 18:40:25 +00:00
// @ts-ignore
2023-07-27 13:38:18 +00:00
const targetAccountIds = [ payload . account . id ] . concat ( payload . mentions . map ( item => item . id ) ) ;
const accountDomain = payload . account . acct . split ( '@' ) [ 1 ] ;
// TODO: Move this logic out of the message handling loop
pgPool . connect ( ( err , client , releasePgConnection ) => {
2018-04-17 11:49:09 +00:00
if ( err ) {
log . error ( err ) ;
return ;
}
const queries = [
2024-01-18 18:40:25 +00:00
// @ts-ignore
2021-12-25 21:55:06 +00:00
client . query ( ` SELECT 1
FROM blocks
WHERE ( account _id = $1 AND target _account _id IN ( $ { placeholders ( targetAccountIds , 2 ) } ) )
OR ( account _id = $2 AND target _account _id = $1 )
UNION
SELECT 1
FROM mutes
WHERE account _id = $1
2023-07-27 13:12:10 +00:00
AND target _account _id IN ( $ { placeholders ( targetAccountIds , 2 ) } ) ` , [req.accountId, payload.account.id].concat(targetAccountIds)),
2018-04-17 11:49:09 +00:00
] ;
if ( accountDomain ) {
2024-01-18 18:40:25 +00:00
// @ts-ignore
2018-04-17 11:49:09 +00:00
queries . push ( client . query ( 'SELECT 1 FROM account_domain_blocks WHERE account_id = $1 AND domain = $2' , [ req . accountId , accountDomain ] ) ) ;
}
2024-01-18 18:40:25 +00:00
// @ts-ignore
2023-07-27 13:12:10 +00:00
if ( ! payload . filtered && ! req . cachedFilters ) {
2024-01-18 18:40:25 +00:00
// @ts-ignore
2022-11-15 01:09:58 +00:00
queries . push ( client . query ( 'SELECT filter.id AS id, filter.phrase AS title, filter.context AS context, filter.expires_at AS expires_at, filter.action AS filter_action, keyword.keyword AS keyword, keyword.whole_word AS whole_word FROM custom_filter_keywords keyword JOIN custom_filters filter ON keyword.custom_filter_id = filter.id WHERE filter.account_id = $1 AND (filter.expires_at IS NULL OR filter.expires_at > NOW())' , [ req . accountId ] ) ) ;
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 07:42:13 +00:00
}
2018-04-17 11:49:09 +00:00
Promise . all ( queries ) . then ( values => {
2023-07-27 13:38:18 +00:00
releasePgConnection ( ) ;
2018-04-17 11:49:09 +00:00
2023-07-27 13:38:18 +00:00
// Handling blocks & mutes and domain blocks: If one of those applies,
// then we don't transmit the payload of the event to the client
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 07:42:13 +00:00
if ( values [ 0 ] . rows . length > 0 || ( accountDomain && values [ 1 ] . rows . length > 0 ) ) {
2017-05-26 22:53:48 +00:00
return ;
}
2023-07-27 13:38:18 +00:00
// If the payload already contains the `filtered` property, it means
2023-07-28 17:11:58 +00:00
// that filtering has been applied on the ruby on rails side, as
2023-07-27 13:38:18 +00:00
// such, we don't need to construct or apply the filters in streaming:
2024-02-27 15:42:05 +00:00
if ( Object . hasOwn ( payload , "filtered" ) ) {
2023-07-27 13:38:18 +00:00
transmit ( event , payload ) ;
return ;
}
// Handling for constructing the custom filters and caching them on the request
// TODO: Move this logic out of the message handling lifecycle
2024-01-18 18:40:25 +00:00
// @ts-ignore
2023-07-27 13:38:18 +00:00
if ( ! req . cachedFilters ) {
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 07:42:13 +00:00
const filterRows = values [ accountDomain ? 2 : 1 ] . rows ;
2024-01-18 18:40:25 +00:00
// @ts-ignore
2023-07-27 13:38:18 +00:00
req . cachedFilters = filterRows . reduce ( ( cache , filter ) => {
if ( cache [ filter . id ] ) {
cache [ filter . id ] . keywords . push ( [ filter . keyword , filter . whole _word ] ) ;
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 07:42:13 +00:00
} else {
2023-07-27 13:38:18 +00:00
cache [ filter . id ] = {
keywords : [ [ filter . keyword , filter . whole _word ] ] ,
expires _at : filter . expires _at ,
filter : {
id : filter . id ,
title : filter . title ,
context : filter . context ,
expires _at : filter . expires _at ,
// filter.filter_action is the value from the
// custom_filters.action database column, it is an integer
// representing a value in an enum defined by Ruby on Rails:
//
// enum { warn: 0, hide: 1 }
filter _action : [ 'warn' , 'hide' ] [ filter . filter _action ] ,
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 07:42:13 +00:00
} ,
} ;
}
return cache ;
} , { } ) ;
2023-07-27 13:38:18 +00:00
// Construct the regular expressions for the custom filters: This
// needs to be done in a separate loop as the database returns one
// filterRow per keyword, so we need all the keywords before
// constructing the regular expression
2024-01-18 18:40:25 +00:00
// @ts-ignore
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 07:42:13 +00:00
Object . keys ( req . cachedFilters ) . forEach ( ( key ) => {
2024-01-18 18:40:25 +00:00
// @ts-ignore
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 07:42:13 +00:00
req . cachedFilters [ key ] . regexp = new RegExp ( req . cachedFilters [ key ] . keywords . map ( ( [ keyword , whole _word ] ) => {
2022-12-18 15:51:37 +00:00
let expr = keyword . replace ( /[.*+?^${}()|[\]\\]/g , '\\$&' ) ;
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 07:42:13 +00:00
if ( whole _word ) {
if ( /^[\w]/ . test ( expr ) ) {
expr = ` \\ b ${ expr } ` ;
}
if ( /[\w]$/ . test ( expr ) ) {
expr = ` ${ expr } \\ b ` ;
}
}
return expr ;
} ) . join ( '|' ) , 'i' ) ;
} ) ;
}
2023-07-27 13:38:18 +00:00
// Apply cachedFilters against the payload, constructing a
// `filter_results` array of FilterResult entities
2024-01-18 18:40:25 +00:00
// @ts-ignore
2023-07-27 13:38:18 +00:00
if ( req . cachedFilters ) {
2023-07-27 13:12:10 +00:00
const status = payload ;
2023-07-27 13:38:18 +00:00
// TODO: Calculate searchableContent in Ruby on Rails:
2024-01-18 18:40:25 +00:00
// @ts-ignore
2023-07-27 13:38:18 +00:00
const searchableContent = ( [ status . spoiler _text || '' , status . content ] . concat ( ( status . poll && status . poll . options ) ? status . poll . options . map ( option => option . title ) : [ ] ) ) . concat ( status . media _attachments . map ( att => att . description ) ) . join ( '\n\n' ) . replace ( /<br\s*\/?>/g , '\n' ) . replace ( /<\/p><p>/g , '\n\n' ) ;
const searchableTextContent = JSDOM . fragment ( searchableContent ) . textContent ;
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 07:42:13 +00:00
const now = new Date ( ) ;
2024-01-18 18:40:25 +00:00
// @ts-ignore
2023-07-27 13:38:18 +00:00
const filter _results = Object . values ( req . cachedFilters ) . reduce ( ( results , cachedFilter ) => {
// Check the filter hasn't expired before applying:
if ( cachedFilter . expires _at !== null && cachedFilter . expires _at < now ) {
2023-07-28 17:11:58 +00:00
return results ;
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 07:42:13 +00:00
}
2023-07-27 13:38:18 +00:00
// Just in-case JSDOM fails to find textContent in searchableContent
if ( ! searchableTextContent ) {
2023-07-28 17:11:58 +00:00
return results ;
2023-07-27 13:38:18 +00:00
}
const keyword _matches = searchableTextContent . match ( cachedFilter . regexp ) ;
if ( keyword _matches ) {
// results is an Array of FilterResult; status_matches is always
// null as we only are only applying the keyword-based custom
// filters, not the status-based custom filters.
// https://docs.joinmastodon.org/entities/FilterResult/
results . push ( {
filter : cachedFilter . filter ,
keyword _matches ,
status _matches : null
} ) ;
}
2023-07-28 17:11:58 +00:00
return results ;
2023-07-27 13:38:18 +00:00
} , [ ] ) ;
// Send the payload + the FilterResults as the `filtered` property
// to the streaming connection. To reach this code, the `event` must
// have been either `update` or `status.update`, meaning the
// `payload` is a Status entity, which has a `filtered` property:
//
// filtered: https://docs.joinmastodon.org/entities/Status/#filtered
transmit ( event , {
... payload ,
filtered : filter _results
} ) ;
2023-07-27 13:12:10 +00:00
} else {
2023-07-27 13:38:18 +00:00
transmit ( event , payload ) ;
2023-07-27 13:12:10 +00:00
}
2018-04-17 11:49:09 +00:00
} ) . catch ( err => {
log . error ( err ) ;
2023-07-28 15:59:31 +00:00
releasePgConnection ( ) ;
2017-05-20 15:31:47 +00:00
} ) ;
2018-04-17 11:49:09 +00:00
} ) ;
2017-05-20 15:31:47 +00:00
} ;
2017-04-17 02:32:30 +00:00
2024-01-18 18:40:25 +00:00
channelIds . forEach ( id => {
2020-06-02 17:24:53 +00:00
subscribe ( ` ${ redisPrefix } ${ id } ` , listener ) ;
} ) ;
2023-07-28 10:06:29 +00:00
if ( typeof attachCloseHandler === 'function' ) {
2024-01-18 18:40:25 +00:00
attachCloseHandler ( channelIds . map ( id => ` ${ redisPrefix } ${ id } ` ) , listener ) ;
2020-08-11 16:24:59 +00:00
}
return listener ;
2017-05-20 15:31:47 +00:00
} ;
2017-02-02 00:31:09 +00:00
2020-08-11 16:24:59 +00:00
/ * *
* @ param { any } req
* @ param { any } res
2023-04-30 00:29:54 +00:00
* @ returns { function ( string , string ) : void }
2020-08-11 16:24:59 +00:00
* /
2017-04-17 02:32:30 +00:00
const streamToHttp = ( req , res ) => {
2023-08-04 14:11:30 +00:00
const channelName = channelNameFromPath ( req ) ;
connectedClients . labels ( { type : 'eventsource' } ) . inc ( ) ;
// In theory we'll always have a channel name, but channelNameFromPath can return undefined:
if ( typeof channelName === 'string' ) {
connectedChannels . labels ( { type : 'eventsource' , channel : channelName } ) . inc ( ) ;
}
2017-05-20 15:31:47 +00:00
res . setHeader ( 'Content-Type' , 'text/event-stream' ) ;
2020-01-24 19:51:33 +00:00
res . setHeader ( 'Cache-Control' , 'no-store' ) ;
2017-05-20 15:31:47 +00:00
res . setHeader ( 'Transfer-Encoding' , 'chunked' ) ;
2017-02-03 23:34:31 +00:00
2020-01-24 19:51:33 +00:00
res . write ( ':)\n' ) ;
2017-05-20 15:31:47 +00:00
const heartbeat = setInterval ( ( ) => res . write ( ':thump\n' ) , 15000 ) ;
2017-02-03 23:34:31 +00:00
2017-04-17 02:32:30 +00:00
req . on ( 'close' , ( ) => {
2024-01-18 18:40:25 +00:00
req . log . info ( { accountId : req . accountId } , ` Ending stream ` ) ;
2023-08-04 14:11:30 +00:00
// We decrement these counters here instead of in streamHttpEnd as in that
// method we don't have knowledge of the channel names
connectedClients . labels ( { type : 'eventsource' } ) . dec ( ) ;
// In theory we'll always have a channel name, but channelNameFromPath can return undefined:
if ( typeof channelName === 'string' ) {
connectedChannels . labels ( { type : 'eventsource' , channel : channelName } ) . dec ( ) ;
}
2017-05-20 15:31:47 +00:00
clearInterval ( heartbeat ) ;
} ) ;
2017-02-02 14:20:31 +00:00
2017-04-17 02:32:30 +00:00
return ( event , payload ) => {
2017-05-20 15:31:47 +00:00
res . write ( ` event: ${ event } \n ` ) ;
res . write ( ` data: ${ payload } \n \n ` ) ;
} ;
} ;
2017-02-02 00:31:09 +00:00
2020-08-11 16:24:59 +00:00
/ * *
* @ param { any } req
* @ param { function ( ) : void } [ closeHandler ]
2023-07-28 10:06:29 +00:00
* @ returns { function ( string [ ] , SubscriptionListener ) : void }
2020-08-11 16:24:59 +00:00
* /
2023-07-28 10:06:29 +00:00
const streamHttpEnd = ( req , closeHandler = undefined ) => ( ids , listener ) => {
2017-04-17 02:32:30 +00:00
req . on ( 'close' , ( ) => {
2020-06-02 17:24:53 +00:00
ids . forEach ( id => {
2023-07-28 10:06:29 +00:00
unsubscribe ( id , listener ) ;
2020-06-02 17:24:53 +00:00
} ) ;
2017-06-03 18:50:53 +00:00
if ( closeHandler ) {
closeHandler ( ) ;
}
2017-05-20 15:31:47 +00:00
} ) ;
} ;
2017-02-03 23:34:31 +00:00
2020-08-11 16:24:59 +00:00
/ * *
2024-01-15 10:36:30 +00:00
* @ param { http . IncomingMessage } req
2024-07-10 15:59:18 +00:00
* @ param { import ( 'ws' ) . WebSocket } ws
2020-08-11 16:24:59 +00:00
* @ param { string [ ] } streamName
2023-04-30 00:29:54 +00:00
* @ returns { function ( string , string ) : void }
2020-08-11 16:24:59 +00:00
* /
const streamToWs = ( req , ws , streamName ) => ( event , payload ) => {
2017-05-28 14:25:26 +00:00
if ( ws . readyState !== ws . OPEN ) {
2024-01-18 18:40:25 +00:00
req . log . error ( 'Tried writing to closed socket' ) ;
2017-05-28 14:25:26 +00:00
return ;
}
2017-02-03 23:34:31 +00:00
2024-01-15 10:36:30 +00:00
const message = JSON . stringify ( { stream : streamName , event , payload } ) ;
2024-01-18 18:40:25 +00:00
ws . send ( message , ( /** @type {Error|undefined} */ err ) => {
2023-06-10 16:35:57 +00:00
if ( err ) {
2024-01-18 18:40:25 +00:00
req . log . error ( { err } , ` Failed to send to websocket ` ) ;
2023-06-10 16:35:57 +00:00
}
} ) ;
2017-05-20 15:31:47 +00:00
} ;
2017-02-03 23:34:31 +00:00
2020-08-11 16:24:59 +00:00
/ * *
2024-02-22 13:20:20 +00:00
* @ param { http . ServerResponse } res
2020-08-11 16:24:59 +00:00
* /
2018-10-11 17:24:43 +00:00
const httpNotFound = res => {
res . writeHead ( 404 , { 'Content-Type' : 'application/json' } ) ;
res . end ( JSON . stringify ( { error : 'Not found' } ) ) ;
} ;
2023-08-04 14:11:30 +00:00
const api = express . Router ( ) ;
2018-08-26 09:54:25 +00:00
2023-08-04 14:11:30 +00:00
app . use ( api ) ;
api . use ( authenticationMiddleware ) ;
api . use ( errorMiddleware ) ;
api . get ( '/api/v1/streaming/*' , ( req , res ) => {
2024-02-22 13:20:20 +00:00
const channelName = channelNameFromPath ( req ) ;
// FIXME: In theory we'd never actually reach here due to
// authenticationMiddleware catching this case, however, we need to refactor
// how those middlewares work, so I'm adding the extra check in here.
if ( ! channelName ) {
httpNotFound ( res ) ;
return ;
}
channelNameToIds ( req , channelName , req . query ) . then ( ( { channelIds , options } ) => {
2020-08-11 16:24:59 +00:00
const onSend = streamToHttp ( req , res ) ;
2021-12-25 21:55:06 +00:00
const onEnd = streamHttpEnd ( req , subscriptionHeartbeat ( channelIds ) ) ;
2018-10-11 17:24:43 +00:00
2024-01-18 18:40:25 +00:00
// @ts-ignore
2024-01-22 18:01:35 +00:00
streamFrom ( channelIds , req , req . log , onSend , onEnd , 'eventsource' , options . needsFiltering , options . allowLocalOnly ) ;
2020-08-11 16:24:59 +00:00
} ) . catch ( err => {
2024-02-27 14:59:20 +00:00
const { statusCode , errorMessage } = extractErrorStatusAndMessage ( err ) ;
2024-02-22 13:20:20 +00:00
res . log . info ( { err } , 'Eventsource subscription error' ) ;
res . writeHead ( statusCode , { 'Content-Type' : 'application/json' } ) ;
res . end ( JSON . stringify ( { error : errorMessage } ) ) ;
2017-11-17 23:16:48 +00:00
} ) ;
} ) ;
2020-08-11 16:24:59 +00:00
/ * *
* @ typedef StreamParams
* @ property { string } [ tag ]
* @ property { string } [ list ]
* @ property { string } [ only _media ]
* /
2021-09-26 11:23:28 +00:00
/ * *
* @ param { any } req
2023-04-30 00:29:54 +00:00
* @ returns { string [ ] }
2021-09-26 11:23:28 +00:00
* /
const channelsForUserStream = req => {
const arr = [ ` timeline: ${ req . accountId } ` ] ;
if ( isInScope ( req , [ 'crypto' ] ) && req . deviceId ) {
arr . push ( ` timeline: ${ req . accountId } : ${ req . deviceId } ` ) ;
}
if ( isInScope ( req , [ 'read' , 'read:notifications' ] ) ) {
arr . push ( ` timeline: ${ req . accountId } :notifications ` ) ;
}
return arr ;
} ;
2020-08-11 16:24:59 +00:00
/ * *
* @ param { any } req
* @ param { string } name
* @ param { StreamParams } params
2023-04-30 00:29:54 +00:00
* @ returns { Promise . < { channelIds : string [ ] , options : { needsFiltering : boolean } } > }
2020-08-11 16:24:59 +00:00
* /
const channelNameToIds = ( req , name , params ) => new Promise ( ( resolve , reject ) => {
2021-12-25 21:55:06 +00:00
switch ( name ) {
2017-05-29 16:20:53 +00:00
case 'user' :
2020-08-11 16:24:59 +00:00
resolve ( {
2021-09-26 11:23:28 +00:00
channelIds : channelsForUserStream ( req ) ,
2021-09-26 16:28:59 +00:00
options : { needsFiltering : false , allowLocalOnly : true } ,
2020-08-11 16:24:59 +00:00
} ) ;
2020-06-02 17:24:53 +00:00
2017-06-03 18:50:53 +00:00
break ;
case 'user:notification' :
2020-08-11 16:24:59 +00:00
resolve ( {
2021-09-26 11:23:28 +00:00
channelIds : [ ` timeline: ${ req . accountId } :notifications ` ] ,
2021-09-26 16:28:59 +00:00
options : { needsFiltering : false , allowLocalOnly : true } ,
2020-08-11 16:24:59 +00:00
} ) ;
2017-05-29 16:20:53 +00:00
break ;
case 'public' :
2020-08-11 16:24:59 +00:00
resolve ( {
channelIds : [ 'timeline:public' ] ,
2021-09-26 16:28:59 +00:00
options : { needsFiltering : true , allowLocalOnly : isTruthy ( params . allow _local _only ) } ,
2020-08-11 17:19:27 +00:00
} ) ;
2017-05-29 16:20:53 +00:00
break ;
2019-01-08 17:33:43 +00:00
case 'public:allow_local_only' :
2020-08-11 17:19:27 +00:00
resolve ( {
channelIds : [ 'timeline:public' ] ,
2021-09-26 16:28:59 +00:00
options : { needsFiltering : true , allowLocalOnly : true } ,
2020-08-11 16:24:59 +00:00
} ) ;
2019-01-08 17:33:43 +00:00
break ;
2017-05-29 16:20:53 +00:00
case 'public:local' :
2020-08-11 16:24:59 +00:00
resolve ( {
channelIds : [ 'timeline:public:local' ] ,
2021-09-26 16:28:59 +00:00
options : { needsFiltering : true , allowLocalOnly : true } ,
2020-08-11 16:24:59 +00:00
} ) ;
2017-05-29 16:20:53 +00:00
break ;
2020-05-10 08:36:18 +00:00
case 'public:remote' :
2020-08-11 16:24:59 +00:00
resolve ( {
channelIds : [ 'timeline:public:remote' ] ,
2021-09-26 16:28:59 +00:00
options : { needsFiltering : true , allowLocalOnly : false } ,
2020-08-11 16:24:59 +00:00
} ) ;
2020-05-10 08:36:18 +00:00
break ;
2018-05-21 10:43:38 +00:00
case 'public:media' :
2020-08-11 16:24:59 +00:00
resolve ( {
channelIds : [ 'timeline:public:media' ] ,
2023-05-07 16:22:25 +00:00
options : { needsFiltering : true , allowLocalOnly : isTruthy ( params . allow _local _only ) } ,
2020-08-11 17:19:27 +00:00
} ) ;
2018-05-21 10:43:38 +00:00
break ;
2019-01-08 17:33:43 +00:00
case 'public:allow_local_only:media' :
2020-08-11 17:19:27 +00:00
resolve ( {
channelIds : [ 'timeline:public:media' ] ,
2021-09-26 16:28:59 +00:00
options : { needsFiltering : true , allowLocalOnly : true } ,
2020-08-11 16:24:59 +00:00
} ) ;
2019-01-08 17:33:43 +00:00
break ;
2018-05-21 10:43:38 +00:00
case 'public:local:media' :
2020-08-11 16:24:59 +00:00
resolve ( {
channelIds : [ 'timeline:public:local:media' ] ,
2021-09-26 16:28:59 +00:00
options : { needsFiltering : true , allowLocalOnly : true } ,
2020-08-11 16:24:59 +00:00
} ) ;
2018-05-21 10:43:38 +00:00
break ;
2020-05-10 08:36:18 +00:00
case 'public:remote:media' :
2020-08-11 16:24:59 +00:00
resolve ( {
channelIds : [ 'timeline:public:remote:media' ] ,
2021-09-26 16:28:59 +00:00
options : { needsFiltering : true , allowLocalOnly : false } ,
2020-08-11 16:24:59 +00:00
} ) ;
2020-05-10 08:36:18 +00:00
break ;
2018-04-18 11:09:06 +00:00
case 'direct' :
2020-08-11 16:24:59 +00:00
resolve ( {
channelIds : [ ` timeline:direct: ${ req . accountId } ` ] ,
2021-09-26 16:28:59 +00:00
options : { needsFiltering : false , allowLocalOnly : true } ,
2020-08-11 16:24:59 +00:00
} ) ;
2018-04-18 11:09:06 +00:00
break ;
2017-05-29 16:20:53 +00:00
case 'hashtag' :
2024-02-22 13:20:20 +00:00
if ( ! params . tag ) {
reject ( new RequestError ( 'Missing tag name parameter' ) ) ;
2020-08-11 16:24:59 +00:00
} else {
resolve ( {
2022-07-13 13:03:28 +00:00
channelIds : [ ` timeline:hashtag: ${ normalizeHashtag ( params . tag ) } ` ] ,
2021-09-26 16:28:59 +00:00
options : { needsFiltering : true , allowLocalOnly : true } ,
2020-08-11 16:24:59 +00:00
} ) ;
2018-10-11 17:24:43 +00:00
}
2017-05-29 16:20:53 +00:00
break ;
case 'hashtag:local' :
2024-02-22 13:20:20 +00:00
if ( ! params . tag ) {
reject ( new RequestError ( 'Missing tag name parameter' ) ) ;
2020-08-11 16:24:59 +00:00
} else {
resolve ( {
2022-07-13 13:03:28 +00:00
channelIds : [ ` timeline:hashtag: ${ normalizeHashtag ( params . tag ) } :local ` ] ,
2021-09-26 16:28:59 +00:00
options : { needsFiltering : true , allowLocalOnly : true } ,
2020-08-11 16:24:59 +00:00
} ) ;
2018-10-11 17:24:43 +00:00
}
2017-05-29 16:20:53 +00:00
break ;
2017-11-17 23:16:48 +00:00
case 'list' :
2024-02-22 13:20:20 +00:00
if ( ! params . list ) {
reject ( new RequestError ( 'Missing list name parameter' ) ) ;
return ;
}
2020-08-11 16:24:59 +00:00
authorizeListAccess ( params . list , req ) . then ( ( ) => {
resolve ( {
channelIds : [ ` timeline:list: ${ params . list } ` ] ,
2021-09-26 16:28:59 +00:00
options : { needsFiltering : false , allowLocalOnly : true } ,
2020-08-11 16:24:59 +00:00
} ) ;
} ) . catch ( ( ) => {
2024-02-22 13:20:20 +00:00
reject ( new AuthenticationError ( 'Not authorized to stream this list' ) ) ;
2017-11-17 23:16:48 +00:00
} ) ;
2020-08-11 16:24:59 +00:00
2017-11-17 23:16:48 +00:00
break ;
2017-05-29 16:20:53 +00:00
default :
2024-02-22 13:20:20 +00:00
reject ( new RequestError ( 'Unknown stream type' ) ) ;
2020-08-11 16:24:59 +00:00
}
} ) ;
/ * *
* @ param { string } channelName
* @ param { StreamParams } params
2023-04-30 00:29:54 +00:00
* @ returns { string [ ] }
2020-08-11 16:24:59 +00:00
* /
const streamNameFromChannelName = ( channelName , params ) => {
2024-01-18 18:40:25 +00:00
if ( channelName === 'list' && params . list ) {
2020-08-11 16:24:59 +00:00
return [ channelName , params . list ] ;
2024-01-18 18:40:25 +00:00
} else if ( [ 'hashtag' , 'hashtag:local' ] . includes ( channelName ) && params . tag ) {
2020-08-11 16:24:59 +00:00
return [ channelName , params . tag ] ;
} else {
return [ channelName ] ;
}
} ;
/ * *
* @ typedef WebSocketSession
2024-07-10 15:59:18 +00:00
* @ property { import ( 'ws' ) . WebSocket & { isAlive : boolean } } websocket
2024-01-18 18:40:25 +00:00
* @ property { http . IncomingMessage & ResolvedAccount } request
* @ property { import ( 'pino' ) . Logger } logger
2023-08-04 14:11:30 +00:00
* @ property { Object . < string , { channelName : string , listener : SubscriptionListener , stopHeartbeat : function ( ) : void } > } subscriptions
2020-08-11 16:24:59 +00:00
* /
/ * *
* @ param { WebSocketSession } session
* @ param { string } channelName
* @ param { StreamParams } params
2023-08-04 14:11:30 +00:00
* @ returns { void }
2020-08-11 16:24:59 +00:00
* /
2024-01-18 18:40:25 +00:00
const subscribeWebsocketToChannel = ( { websocket , request , logger , subscriptions } , channelName , params ) => {
checkScopes ( request , logger , channelName ) . then ( ( ) => channelNameToIds ( request , channelName , params ) ) . then ( ( {
2021-12-25 21:55:06 +00:00
channelIds ,
options ,
} ) => {
2020-08-11 16:24:59 +00:00
if ( subscriptions [ channelIds . join ( ';' ) ] ) {
return ;
}
2024-01-18 18:40:25 +00:00
const onSend = streamToWs ( request , websocket , streamNameFromChannelName ( channelName , params ) ) ;
2020-08-11 16:24:59 +00:00
const stopHeartbeat = subscriptionHeartbeat ( channelIds ) ;
2024-01-22 18:01:35 +00:00
const listener = streamFrom ( channelIds , request , logger , onSend , undefined , 'websocket' , options . needsFiltering , options . allowLocalOnly ) ;
2020-08-11 16:24:59 +00:00
2023-08-04 14:11:30 +00:00
connectedChannels . labels ( { type : 'websocket' , channel : channelName } ) . inc ( ) ;
2020-08-11 16:24:59 +00:00
subscriptions [ channelIds . join ( ';' ) ] = {
2023-08-04 14:11:30 +00:00
channelName ,
2020-08-11 16:24:59 +00:00
listener ,
stopHeartbeat ,
} ;
} ) . catch ( err => {
2024-02-27 14:59:20 +00:00
const { statusCode , errorMessage } = extractErrorStatusAndMessage ( err ) ;
2024-02-22 13:20:20 +00:00
logger . error ( { err } , 'Websocket subscription error' ) ;
// If we have a socket that is alive and open still, send the error back to the client:
if ( websocket . isAlive && websocket . readyState === websocket . OPEN ) {
websocket . send ( JSON . stringify ( {
error : errorMessage ,
status : statusCode
} ) ) ;
}
2020-08-11 16:24:59 +00:00
} ) ;
2023-10-09 11:38:29 +00:00
} ;
2023-08-04 14:11:30 +00:00
2024-01-18 18:40:25 +00:00
/ * *
* @ param { WebSocketSession } session
* @ param { string [ ] } channelIds
* /
const removeSubscription = ( { request , logger , subscriptions } , channelIds ) => {
logger . info ( { channelIds , accountId : request . accountId } , ` Ending stream ` ) ;
2023-08-04 14:11:30 +00:00
const subscription = subscriptions [ channelIds . join ( ';' ) ] ;
if ( ! subscription ) {
return ;
}
channelIds . forEach ( channelId => {
unsubscribe ( ` ${ redisPrefix } ${ channelId } ` , subscription . listener ) ;
} ) ;
connectedChannels . labels ( { type : 'websocket' , channel : subscription . channelName } ) . dec ( ) ;
subscription . stopHeartbeat ( ) ;
delete subscriptions [ channelIds . join ( ';' ) ] ;
2023-10-09 11:38:29 +00:00
} ;
2020-08-11 16:24:59 +00:00
/ * *
* @ param { WebSocketSession } session
* @ param { string } channelName
* @ param { StreamParams } params
2023-08-04 14:11:30 +00:00
* @ returns { void }
2020-08-11 16:24:59 +00:00
* /
2024-01-18 18:40:25 +00:00
const unsubscribeWebsocketFromChannel = ( session , channelName , params ) => {
const { websocket , request , logger } = session ;
2020-08-11 16:24:59 +00:00
channelNameToIds ( request , channelName , params ) . then ( ( { channelIds } ) => {
2024-01-18 18:40:25 +00:00
removeSubscription ( session , channelIds ) ;
2023-08-04 14:11:30 +00:00
} ) . catch ( err => {
2024-02-22 13:20:20 +00:00
logger . error ( { err } , 'Websocket unsubscribe error' ) ;
2020-08-11 16:24:59 +00:00
2023-08-04 14:11:30 +00:00
// If we have a socket that is alive and open still, send the error back to the client:
2024-01-18 18:40:25 +00:00
if ( websocket . isAlive && websocket . readyState === websocket . OPEN ) {
2024-02-22 13:20:20 +00:00
// TODO: Use a better error response here
2024-01-18 18:40:25 +00:00
websocket . send ( JSON . stringify ( { error : "Error unsubscribing from channel" } ) ) ;
2020-08-11 16:24:59 +00:00
}
} ) ;
2023-10-09 11:38:29 +00:00
} ;
2020-08-11 16:24:59 +00:00
2020-11-12 22:05:24 +00:00
/ * *
* @ param { WebSocketSession } session
* /
2024-01-18 18:40:25 +00:00
const subscribeWebsocketToSystemChannel = ( { websocket , request , subscriptions } ) => {
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 07:42:13 +00:00
const accessTokenChannelId = ` timeline:access_token: ${ request . accessTokenId } ` ;
const systemChannelId = ` timeline:system: ${ request . accountId } ` ;
2020-11-12 22:05:24 +00:00
const listener = createSystemMessageListener ( request , {
2021-12-25 21:55:06 +00:00
onKill ( ) {
2024-01-18 18:40:25 +00:00
websocket . close ( ) ;
2020-11-12 22:05:24 +00:00
} ,
} ) ;
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 07:42:13 +00:00
subscribe ( ` ${ redisPrefix } ${ accessTokenChannelId } ` , listener ) ;
2020-11-12 22:05:24 +00:00
subscribe ( ` ${ redisPrefix } ${ systemChannelId } ` , listener ) ;
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 07:42:13 +00:00
subscriptions [ accessTokenChannelId ] = {
2023-08-04 14:11:30 +00:00
channelName : 'system' ,
Revamp post filtering system (#18058)
* Add model for custom filter keywords
* Use CustomFilterKeyword internally
Does not change the API
* Fix /filters/edit and /filters/new
* Add migration tests
* Remove whole_word column from custom_filters (covered by custom_filter_keywords)
* Redesign /filters
Instead of a list, present a card that displays more information and handles
multiple keywords per filter.
* Redesign /filters/new and /filters/edit to add and remove keywords
This adds a new gem dependency: cocoon, as well as a npm dependency:
cocoon-js-vanilla. Those are used to easily populate and remove form fields
from the user interface when manipulating multiple keyword filters at once.
* Add /api/v2/filters to edit filter with multiple keywords
Entities:
- `Filter`: `id`, `title`, `filter_action` (either `hide` or `warn`), `context`
`keywords`
- `FilterKeyword`: `id`, `keyword`, `whole_word`
API endpoits:
- `GET /api/v2/filters` to list filters (including keywords)
- `POST /api/v2/filters` to create a new filter
`keywords_attributes` can also be passed to create keywords in one request
- `GET /api/v2/filters/:id` to read a particular filter
- `PUT /api/v2/filters/:id` to update a new filter
`keywords_attributes` can also be passed to edit, delete or add keywords in
one request
- `DELETE /api/v2/filters/:id` to delete a particular filter
- `GET /api/v2/filters/:id/keywords` to list keywords for a filter
- `POST /api/v2/filters/:filter_id/keywords/:id` to add a new keyword to a
filter
- `GET /api/v2/filter_keywords/:id` to read a particular keyword
- `PUT /api/v2/filter_keywords/:id` to edit a particular keyword
- `DELETE /api/v2/filter_keywords/:id` to delete a particular keyword
* Change from `irreversible` boolean to `action` enum
* Remove irrelevent `irreversible_must_be_within_context` check
* Fix /filters/new and /filters/edit with update for filter_action
* Fix Rubocop/Codeclimate complaining about task names
* Refactor FeedManager#phrase_filtered?
This moves regexp building and filter caching to the `CustomFilter` class.
This does not change the functional behavior yet, but this changes how the
cache is built, doing per-custom_filter regexps so that filters can be matched
independently, while still offering caching.
* Perform server-side filtering and output result in REST API
* Fix numerous filters_changed events being sent when editing multiple keywords at once
* Add some tests
* Use the new API in the WebUI
- use client-side logic for filters we have fetched rules for.
This is so that filter changes can be retroactively applied without
reloading the UI.
- use server-side logic for filters we haven't fetched rules for yet
(e.g. network error, or initial timeline loading)
* Minor optimizations and refactoring
* Perform server-side filtering on the streaming server
* Change the wording of filter action labels
* Fix issues pointed out by linter
* Change design of “Show anyway” link in accordence to review comments
* Drop “irreversible” filtering behavior
* Move /api/v2/filter_keywords to /api/v1/filters/keywords
* Rename `filter_results` attribute to `filtered`
* Rename REST::LegacyFilterSerializer to REST::V1::FilterSerializer
* Fix systemChannelId value in streaming server
* Simplify code by removing client-side filtering code
The simplifcation comes at a cost though: filters aren't retroactively
applied anymore.
2022-06-28 07:42:13 +00:00
listener ,
stopHeartbeat : ( ) => {
} ,
} ;
2020-11-12 22:05:24 +00:00
subscriptions [ systemChannelId ] = {
2023-08-04 14:11:30 +00:00
channelName : 'system' ,
2020-11-12 22:05:24 +00:00
listener ,
2021-12-25 21:55:06 +00:00
stopHeartbeat : ( ) => {
} ,
2020-11-12 22:05:24 +00:00
} ;
2023-08-04 14:11:30 +00:00
connectedChannels . labels ( { type : 'websocket' , channel : 'system' } ) . inc ( 2 ) ;
2020-11-12 22:05:24 +00:00
} ;
2024-01-15 10:36:30 +00:00
/ * *
2024-07-10 15:59:18 +00:00
* @ param { import ( 'ws' ) . WebSocket & { isAlive : boolean } } ws
2024-01-18 18:40:25 +00:00
* @ param { http . IncomingMessage & ResolvedAccount } req
* @ param { import ( 'pino' ) . Logger } log
2024-01-15 10:36:30 +00:00
* /
2024-01-18 18:40:25 +00:00
function onConnection ( ws , req , log ) {
2023-10-02 11:21:43 +00:00
// Note: url.parse could throw, which would terminate the connection, so we
// increment the connected clients metric straight away when we establish
// the connection, without waiting:
connectedClients . labels ( { type : 'websocket' } ) . inc ( ) ;
2020-08-11 16:24:59 +00:00
2023-10-02 11:21:43 +00:00
// Setup connection keep-alive state:
2021-03-24 08:37:41 +00:00
ws . isAlive = true ;
ws . on ( 'pong' , ( ) => {
ws . isAlive = true ;
} ) ;
2020-08-11 16:24:59 +00:00
/ * *
* @ type { WebSocketSession }
* /
const session = {
2024-01-18 18:40:25 +00:00
websocket : ws ,
2020-08-11 16:24:59 +00:00
request : req ,
2024-01-18 18:40:25 +00:00
logger : log ,
2020-08-11 16:24:59 +00:00
subscriptions : { } ,
} ;
2023-10-02 11:21:43 +00:00
ws . on ( 'close' , function onWebsocketClose ( ) {
2023-08-04 14:11:30 +00:00
const subscriptions = Object . keys ( session . subscriptions ) ;
2020-08-11 16:24:59 +00:00
2023-08-04 14:11:30 +00:00
subscriptions . forEach ( channelIds => {
2024-01-18 18:40:25 +00:00
removeSubscription ( session , channelIds . split ( ';' ) ) ;
2023-08-04 14:11:30 +00:00
} ) ;
2020-08-11 16:24:59 +00:00
2023-10-02 11:21:43 +00:00
// Decrement the metrics for connected clients:
connectedClients . labels ( { type : 'websocket' } ) . dec ( ) ;
2024-02-27 14:59:20 +00:00
// We need to unassign the session object as to ensure it correctly gets
2024-01-18 18:40:25 +00:00
// garbage collected, without doing this we could accidentally hold on to
// references to the websocket, the request, and the logger, causing
// memory leaks.
2024-02-27 14:59:20 +00:00
// This is commented out because `delete` only operated on object properties
// It needs to be replaced by `session = undefined`, but it requires every calls to
// `session` to check for it, thus a significant refactor
// delete session;
2023-10-02 11:21:43 +00:00
} ) ;
2020-08-11 16:24:59 +00:00
2023-10-02 11:21:43 +00:00
// Note: immediately after the `error` event is emitted, the `close` event
// is emitted. As such, all we need to do is log the error here.
2024-01-18 18:40:25 +00:00
ws . on ( 'error' , ( /** @type {Error} */ err ) => {
log . error ( err ) ;
2023-10-02 11:21:43 +00:00
} ) ;
2020-08-11 16:24:59 +00:00
2023-06-09 17:29:16 +00:00
ws . on ( 'message' , ( data , isBinary ) => {
if ( isBinary ) {
2024-01-18 18:40:25 +00:00
log . warn ( 'Received binary data, closing connection' ) ;
2023-06-09 17:29:16 +00:00
ws . close ( 1003 , 'The mastodon streaming server does not support binary messages' ) ;
return ;
}
const message = data . toString ( 'utf8' ) ;
const json = parseJSON ( message , session . request ) ;
2020-11-12 22:05:24 +00:00
2020-09-22 13:30:41 +00:00
if ( ! json ) return ;
2020-11-12 22:05:24 +00:00
2020-09-22 13:30:41 +00:00
const { type , stream , ... params } = json ;
2020-08-11 16:24:59 +00:00
if ( type === 'subscribe' ) {
subscribeWebsocketToChannel ( session , firstParam ( stream ) , params ) ;
} else if ( type === 'unsubscribe' ) {
2020-11-23 16:35:14 +00:00
unsubscribeWebsocketFromChannel ( session , firstParam ( stream ) , params ) ;
2020-08-11 16:24:59 +00:00
} else {
// Unknown action type
}
} ) ;
2020-11-12 22:05:24 +00:00
subscribeWebsocketToSystemChannel ( session ) ;
2023-10-02 11:21:43 +00:00
// Parse the URL for the connection arguments (if supplied), url.parse can throw:
const location = req . url && url . parse ( req . url , true ) ;
if ( location && location . query . stream ) {
2020-08-11 16:24:59 +00:00
subscribeWebsocketToChannel ( session , firstParam ( location . query . stream ) , location . query ) ;
2017-05-29 16:20:53 +00:00
}
2024-01-15 10:36:30 +00:00
}
wss . on ( 'connection' , onConnection ) ;
2017-02-03 23:34:31 +00:00
2021-03-24 08:37:41 +00:00
setInterval ( ( ) => {
wss . clients . forEach ( ws => {
2024-01-18 18:40:25 +00:00
// @ts-ignore
2021-03-24 08:37:41 +00:00
if ( ws . isAlive === false ) {
ws . terminate ( ) ;
return ;
}
2024-01-18 18:40:25 +00:00
// @ts-ignore
2021-03-24 08:37:41 +00:00
ws . isAlive = false ;
2021-05-02 12:30:26 +00:00
ws . ping ( '' , false ) ;
2021-03-24 08:37:41 +00:00
} ) ;
} , 30000 ) ;
2017-05-28 14:25:26 +00:00
2018-10-20 00:25:25 +00:00
attachServerWithConfig ( server , address => {
2024-01-18 18:40:25 +00:00
logger . info ( ` Streaming API now listening on ${ address } ` ) ;
2018-10-20 00:25:25 +00:00
} ) ;
2017-04-21 17:24:31 +00:00
2017-05-28 14:25:26 +00:00
const onExit = ( ) => {
2017-05-20 15:31:47 +00:00
server . close ( ) ;
2017-07-07 18:01:00 +00:00
process . exit ( 0 ) ;
2017-05-28 14:25:26 +00:00
} ;
2024-01-18 18:40:25 +00:00
/** @param {Error} err */
2017-05-28 14:25:26 +00:00
const onError = ( err ) => {
2024-01-18 18:40:25 +00:00
logger . error ( err ) ;
2017-12-12 19:19:33 +00:00
server . close ( ) ;
process . exit ( 0 ) ;
2017-05-28 14:25:26 +00:00
} ;
process . on ( 'SIGINT' , onExit ) ;
process . on ( 'SIGTERM' , onExit ) ;
process . on ( 'exit' , onExit ) ;
2017-12-12 19:19:33 +00:00
process . on ( 'uncaughtException' , onError ) ;
2017-05-28 14:25:26 +00:00
} ;
2020-08-11 16:24:59 +00:00
/ * *
* @ param { any } server
* @ param { function ( string ) : void } [ onSuccess ]
* /
2018-10-20 00:25:25 +00:00
const attachServerWithConfig = ( server , onSuccess ) => {
if ( process . env . SOCKET || process . env . PORT && isNaN ( + process . env . PORT ) ) {
server . listen ( process . env . SOCKET || process . env . PORT , ( ) => {
if ( onSuccess ) {
2018-10-21 14:41:33 +00:00
fs . chmodSync ( server . address ( ) , 0o666 ) ;
2018-10-20 00:25:25 +00:00
onSuccess ( server . address ( ) ) ;
}
} ) ;
} else {
2024-01-18 18:40:25 +00:00
server . listen ( + ( process . env . PORT || 4000 ) , process . env . BIND || '127.0.0.1' , ( ) => {
2018-10-20 00:25:25 +00:00
if ( onSuccess ) {
onSuccess ( ` ${ server . address ( ) . address } : ${ server . address ( ) . port } ` ) ;
}
} ) ;
}
} ;
2023-04-26 09:37:51 +00:00
startServer ( ) ;