From 7c28daec848a0277ad060480a31714566c0aa647 Mon Sep 17 00:00:00 2001 From: kouhai Date: Mon, 15 Apr 2024 00:33:47 -0700 Subject: [PATCH] th: fork docs and configuration Co-authored-by: Rin Co-authored-by: fox --- .env.development | 14 ++++ .env.production.sample | 7 +- .env.test | 5 ++ .gitignore | 15 ++++ CONTRIBUTING.md | 35 +++++++-- DIVERGENCES.md | 14 ++++ Procfile.dev | 2 +- Rakefile | 9 +++ SETUP.md | 143 +++++++++++++++++++++++++++++++++++ lib/mastodon/redis_config.rb | 21 ++++- lib/tasks/deps.rake | 109 ++++++++++++++++++++++++++ redis-dev.conf | 12 +++ streaming/index.js | 61 ++++++++++----- 13 files changed, 418 insertions(+), 29 deletions(-) create mode 100644 DIVERGENCES.md create mode 100644 SETUP.md create mode 100644 lib/tasks/deps.rake create mode 100644 redis-dev.conf diff --git a/.env.development b/.env.development index 0330da8377..fb60fbf2f7 100644 --- a/.env.development +++ b/.env.development @@ -1,3 +1,17 @@ +LOCAL_DOMAIN=localhost +ALTERNATE_DOMAINS=mastodon.internal +STREAMING_API_BASE_URL=https://streaming.mastodon.internal + +DB_HOST=$PWD/data/postgres +DB_USER=mastodon +DB_NAME=mastodon_dev +REDIS_URL=unix://$PWD/data/redis/redis-dev.sock +TH_MENTION_SPAM_HEURISTIC_AUTO_LIMIT_ACTIVE=can-spam +TH_MENTION_SPAM_THRESHOLD=2 +TH_STAFF_ACCOUNT=staff + +TH_USE_INVITE_QUOTA=1 + # Required by ActiveRecord encryption feature ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY=fkSxKD2bF396kdQbrP1EJ7WbU7ZgNokR ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT=r0hvVmzBVsjxC7AMlwhOzmtc36ZCOS1E diff --git a/.env.production.sample b/.env.production.sample index 1b3c511042..90b6217c28 100644 --- a/.env.production.sample +++ b/.env.production.sample @@ -14,7 +14,7 @@ # ---------- # This identifies your server and cannot be changed safely later # ---------- -LOCAL_DOMAIN=example.com +LOCAL_DOMAIN=localhost # Use this only if you need to run mastodon on a different domain than the one used for federation. # You can read more about this option on https://docs.joinmastodon.org/admin/config/#web-domain @@ -25,6 +25,7 @@ LOCAL_DOMAIN=example.com # handler@example2.com etc. for the same user. LOCAL_DOMAIN should not # be added. Comma separated values # ALTERNATE_DOMAINS=example1.com,example2.com +ALTERNATE_DOMAINS=mastodon.internal # Use HTTP proxy for outgoing request (optional) # http_proxy=http://gateway.local:8118 @@ -43,13 +44,13 @@ LOCAL_DOMAIN=example.com # Redis # ----- -REDIS_HOST=localhost +REDIS_HOST=redis REDIS_PORT=6379 # PostgreSQL # ---------- -DB_HOST=/var/run/postgresql +DB_HOST=db DB_USER=mastodon DB_NAME=mastodon_production DB_PASS= diff --git a/.env.test b/.env.test index d2763e582a..f238060084 100644 --- a/.env.test +++ b/.env.test @@ -4,6 +4,11 @@ NODE_ENV=production LOCAL_DOMAIN=cb6e6126.ngrok.io LOCAL_HTTPS=true +DB_HOST=$(pwd)/data/postgres +DB_USER=mastodon +DB_NAME=mastodon_dev +REDIS_URL=unix://./data/redis/redis-dev.sock + # Secret values required by ActiveRecord encryption feature # Use `bin/rails db:encryption:init` to generate fresh secrets ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY=test_determinist_key_DO_NOT_USE_IN_PRODUCTION diff --git a/.gitignore b/.gitignore index a70f30f952..f3bb50fbc1 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ # or operating system, you probably want to add a global ignore instead: # git config --global core.excludesfile '~/.gitignore_global' +# Ignore local dotenv overrides +.env.*.local + # Ignore bundler config and downloaded libraries. /.bundle /vendor/bundle @@ -12,6 +15,9 @@ /db/*.sqlite3 /db/*.sqlite3-journal +# Ignore local data directory +/data + # Ignore all logfiles and tempfiles. .eslintcache /log/* @@ -69,5 +75,14 @@ yarn-debug.log # Ignore Docker option files docker-compose.override.yml +# Yarn Berry +.pnp.* +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/sdks +!.yarn/versions + # Ignore dotenv .local files .env*.local diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2271802ca9..348ac9ca8a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,11 +1,29 @@ -# Contributing to Mastodon Glitch Edition +# Contributing to Mastodon Glitch+Treehouse Edition -Thank you for your interest in contributing to the `glitch-soc` project! +Thank you for your interest in contributing to the **Treehouse Mastodon** project! Here are some guidelines, and ways you can help. > (This document is a bit of a work-in-progress, so please bear with us. > If you don't see what you're looking for here, please don't hesitate to reach out!) +## Merging + +If your username is kouhai, or you're otherwise merging from upstream glitch-soc +for some reason, the following snippets may be useful: + +```sh +git fetch glitch && git merge glitch/main && git checkout glitch/main -- yarn.lock +``` + +```sh +export RAILS_ENV=production NODE_ENV=production +export OTP_SECRET=precompile_placeholder SECRET_KEY_BASE=precompile_placeholder +bundle install \ +&& yarn install \ +&& bundle exec rake assets:clobber \ +&& bundle exec rake webpacker:compile | tee /tmp/out.log +``` + ## Translations You can submit glitch-soc-specific translations via [Crowdin](https://crowdin.com/project/glitch-soc). They are periodically merged into the codebase. @@ -14,14 +32,19 @@ You can submit glitch-soc-specific translations via [Crowdin](https://crowdin.co ## Planning -Right now a lot of the planning for this project takes place in our development Discord, or through GitHub Issues and Projects. +Right now a lot of the planning for this project takes place in the `#fediverse` +channel of the Treehouse Discord, or through Gitea Issues. + We're working on ways to improve the planning structure and better solicit feedback, and if you feel like you can help in this respect, feel free to give us a holler. ## Documentation -The documentation for this repository is available at [`glitch-soc/docs`](https://github.com/glitch-soc/docs) (online at [glitch-soc.github.io/docs/](https://glitch-soc.github.io/docs/)). -Right now, we've mostly focused on the features that make this fork different from upstream in some manner. -Adding screenshots, improving descriptions, and so forth are all ways to help contribute to the project even if you don't know any code. +The upstream Glitch documentation for this repository is available at [`glitch-soc/docs`](https://github.com/glitch-soc/docs) (online at [glitch-soc.github.io/docs/](https://glitch-soc.github.io/docs/)). + +## Setup + +For a some-batteries-required guide to setting up a development environment for this repository, read Rin's excellent +[SETUP.md](https://gitea.treehouse.systems/treehouse/mastodon/src/branch/main/SETUP.md). ## Frontend Development diff --git a/DIVERGENCES.md b/DIVERGENCES.md new file mode 100644 index 0000000000..036e25dd53 --- /dev/null +++ b/DIVERGENCES.md @@ -0,0 +1,14 @@ +# Divergences + +## Major Features + +- quote posting +- Treehouse::Automod (experimental feature flagged) + +## Other Changes + +- various build system changes + - a better dockerfile + - yarn v2 (a mistake, tbh) + - various dev env changes +- various css/style changes diff --git a/Procfile.dev b/Procfile.dev index f81333b04c..d8070bc610 100644 --- a/Procfile.dev +++ b/Procfile.dev @@ -1,4 +1,4 @@ web: env PORT=3000 RAILS_ENV=development bundle exec puma -C config/puma.rb sidekiq: env PORT=3000 RAILS_ENV=development bundle exec sidekiq -stream: env PORT=4000 yarn workspace @mastodon/streaming start +stream: env PORT=4000 NODE_ENV=development yarn workspace @mastodon/streaming start | npx pino-pretty webpack: bin/webpack-dev-server diff --git a/Rakefile b/Rakefile index e51cf0e17e..5ad80bcfcc 100644 --- a/Rakefile +++ b/Rakefile @@ -6,3 +6,12 @@ require File.expand_path('config/application', __dir__) Rails.application.load_tasks + +# please don't do this +if Rake::Task.task_defined?('assets:precompile') && ENV.include?('RAKE_NO_YARN_INSTALL_HACK') + task = Rake::Task['assets:precompile'] + puts task.prerequisites + task.prerequisites.delete('webpacker:yarn_install') + task.prerequisites.delete('yarn:install') + puts task.prerequisites +end diff --git a/SETUP.md b/SETUP.md new file mode 100644 index 0000000000..716048a2b1 --- /dev/null +++ b/SETUP.md @@ -0,0 +1,143 @@ +# Setting up a dev environment + +## Prerequisites + +Mastodon development requires the following: + +- Ruby 3.0 +- Ruby gems: + - `bundler` + - `irb` + - `foreman` +- NodeJS v18 (LTS) +- NPM packages: + - `yarn` +- Postgres +- Redis + +### macOS + +First, make sure you have Homebrew installed. Follow the instructions at [brew.sh](https://brew.sh). + +Run the following to install all necessary packages: +``` +brew install ruby@3.0 foreman node yarn postgresql redis +``` + +Ruby 3.0 is **keg-only** by default. Follow the instructions in the **Caveat** to add it to your path. + +### Linux + +We will assume that you know how to locate the correct packages for your distro. That said, some distros package `bundler` and `irb` separately. Make sure that you also install these. + +On Arch, you will need: +- `ruby` +- `ruby-bundler` +- `ruby-irb` +- `ruby-foreman` +- `redis` +- `postgresql` +- `yarn` +- `gmp` +- `libidn` + +### Windows + +Unfortunately, none of the authors use Windows. Contributions welcome! + +## Database + +In the root of this repository, go through the following script: +```sh +# Create a folder for local data +mkdir -p data + +# Set up a local database +pg_ctl -D data/postgres initdb -o '-U mastodon --auth-host=trust' + +# Use the data/postgres folder for the DB connection unix socket +# +# If you don't know what that means, it's just a way for Mastodon to communicate +# with a database on the same machine efficiently. +# +# See: https://manpages.ubuntu.com/manpages/jammy/man7/unix.7.html +echo 'unix_socket_directories = .' >> data/postgres/postgresql.conf + +# Start the database +pg_ctl -D data/postgres start --silent +``` + +## Redis + +In the root of this repository, run the following: +```sh +# Create a folder for redis data +mkdir -p data/redis + +# Start Redis +redis-server ./redis-dev.conf + +# [Optional] Stop Redis +# kill "$(cat ./data/redis/redis-dev.pid)" +``` + +## Ruby + +```sh +export RAILS_ENV=development + +# Bundle installs all Ruby gems globally by default, which might cause problems. +bundle config set --local path 'vendor/bundle' + +# [Apple Silicon] If using macOS on Apple Silicon, run the following: +# bundle config build.idn-ruby -- --with-idn-dir="$(brew --prefix libidn)" + +# Install dependencies using bundle (Ruby) and yarn (JS) +bundle install +yarn install + +# Setup the database using the pre-defined Rake task +# +# Rake is a command runner for Ruby projects. The `bundle exec` ensures that +# we use the version of Rake that this project requires. +bundle exec rake db:setup + +# [Optional] If that fails, run the following and try again: +# bundle exec rake db:reset +``` + +## Running Mastodon + +1. Run `export RAILS_ENV=development NODE_ENV=development`. + - Put these in your shell's .rc, or a script you can source if you want to skip this step in the future. +2. Run `bundle exec rake assets:precompile`. + - If this explodes, complaining about `Hash`, you'll need to `export NODE_OPTIONS=--openssl-legacy-provider`. + - After doing this, you will need to run `bundle exec rake assets:clobber` and then re-run `bundle exec rake assets:precompile`. +3. Run `foreman start` + +# Updates/Troubleshooting + +## RubyVM/DebugInspector Issues + +Still unable to fix. Circumvent by removing `better_errors` and `binding_of_caller` from Gemfile. + +Happy to troubleshoot with someone better with Ruby than us >_<'/. + +## Webpack Issues + +If Webpack compalins about being unable to find some assets or locales: + +Try: +1. `rm -rf node_modules` +2. `yarn install` + +If this doesn't help, try: +1. `yarn add webpack` +2. `git restore package.json yarn.lock` +3. `yarn install` + +Then re-run `foreman start`. No. We have no idea why this worked. + +# Need Help? + +If the above instructions don't work, please contact @Rin here, or @tammy@social.treehouse.systems. diff --git a/lib/mastodon/redis_config.rb b/lib/mastodon/redis_config.rb index c858b61a05..1097016749 100644 --- a/lib/mastodon/redis_config.rb +++ b/lib/mastodon/redis_config.rb @@ -1,10 +1,29 @@ # frozen_string_literal: true +require 'pathname' + def setup_redis_env_url(prefix = nil, defaults = true) prefix = "#{prefix.to_s.upcase}_" unless prefix.nil? prefix = '' if prefix.nil? + redis_url_key = "#{prefix}REDIS_URL" - return if ENV["#{prefix}REDIS_URL"].present? + if ENV[redis_url_key].present? + conn = +ENV["#{prefix}REDIS_URL"].sub(/redis:\/\//i, '') + + # Strip any prefixing `unix://` + unix = !conn.sub!(/^unix:\/\//i, '').nil? + # Strip any prefixing `./` + unix |= conn.sub!(/^(\.\/)+/, '') + unix |= conn.start_with?('/') + + if unix + pn = Pathname.new(conn) + pn = Pathname.getwd / pn if pn.relative? + ENV[redis_url_key] = "unix://#{pn}" + end + + return + end password = ENV.fetch("#{prefix}REDIS_PASSWORD") { '' if defaults } host = ENV.fetch("#{prefix}REDIS_HOST") { 'localhost' if defaults } diff --git a/lib/tasks/deps.rake b/lib/tasks/deps.rake new file mode 100644 index 0000000000..2ebd8d5084 --- /dev/null +++ b/lib/tasks/deps.rake @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +require 'pathname' + +DATA_DIR = Pathname.new('data') +POSTGRES_DIR = DATA_DIR / 'postgres' +POSTGRES_CONF_FILE = POSTGRES_DIR / 'postgresql.conf' +POSTGRES_SOCKET_FILE = POSTGRES_DIR / '.s.PGSQL.5432' +POSTGRES_PID_FILE = POSTGRES_DIR / 'postmaster.pid' +REDIS_DIR = DATA_DIR / 'redis' +REDIS_PID_FILE = REDIS_DIR / 'redis-dev.pid' + +def divider + puts '==========' +end + +def get_pid(pid_file) + return false unless File.file?(pid_file) + pid = File.read(pid_file).to_i + + Process.kill(0, pid) + pid +rescue Errno::ESRCH + nil +end + +def postgres_running? + get_pid POSTGRES_PID_FILE +end + +directory REDIS_DIR.to_s + +namespace :deps do + task start: ['postgres:start', 'redis:start'] + task stop: ['postgres:stop', 'redis:stop'] + + namespace :postgres do + namespace :setup do + task all: [POSTGRES_DIR.to_s] + + file POSTGRES_DIR.to_s do + if POSTGRES_CONF_FILE.exist? + puts 'Postgres conf exists, skipping initdb' + next + end + sh %(printf '%s\\n' pg_ctl -D data/postgres initdb -o '-U mastodon --auth-host=trust') + end + + task configure: [POSTGRES_DIR.to_s] do + next if File.foreach(POSTGRES_CONF_FILE).detect? { |line| line == /^unix_socket_directories = \.\s*$/ } + + POSTGRES_CONF_FILE.open('at') do |f| + f.write("\n", PG_SOCKET_DIRECTORIES_LINE, "\n") + end + end + end + + task start: ['setup:all'] do + if (pid = get_pid POSTGRES_PID_FILE) + puts "Postgres is running (pid #{pid})!" + next + end + + puts 'Starting postgres...' + divider + sh %(pg_ctl -D ./data/postgres start) + divider + end + + task :stop do + unless (pid = get_pid POSTGRES_PID_FILE) + puts "Postgres isn't running!" + next + end + + puts "Stopping Postgres (pid #{pid})..." + sh %(pg_ctl -D ./data/postgres stop) + divider + end + end + + namespace :redis do + task init: [REDIS_DIR.to_s] do + end + + task start: [:init] do + if (pid = get_pid REDIS_PID_FILE) + puts "Redis is running (pid #{pid})!" + next + end + + puts 'Starting redis...' + divider + sh %(redis-server redis-dev.conf) + divider + end + + task :stop do + unless (pid = get_pid REDIS_PID_FILE) + puts "Redis isn't running!" + next + end + + puts "Stopping Redis (pid #{pid})..." + divider + Process.kill(:TERM, pid) + end + end +end diff --git a/redis-dev.conf b/redis-dev.conf new file mode 100644 index 0000000000..ed197151d0 --- /dev/null +++ b/redis-dev.conf @@ -0,0 +1,12 @@ +# This redis configuration is for development only + +daemonize yes + +dir ./data/redis +pidfile ./redis-dev.pid + +unixsocket ./redis-dev.sock +unixsocketperm 700 + +# Disable TCP +port 0 diff --git a/streaming/index.js b/streaming/index.js index 74121774cc..09063f9315 100644 --- a/streaming/index.js +++ b/streaming/index.js @@ -19,20 +19,30 @@ import { logger, httpLogger, initializeLogLevel, attachWebsocketHttpLogger, crea import { setupMetrics } from './metrics.js'; import { isTruthy, normalizeHashtag, firstParam } from './utils.js'; -const environment = process.env.NODE_ENV || 'development'; +const environment = process.env.NODE_ENV !== 'development' ? 'production' : 'development'; // Correctly detect and load .env or .env.production file based on environment: -const dotenvFile = environment === 'production' ? '.env.production' : '.env'; -const dotenvFilePath = path.resolve( - url.fileURLToPath( - new URL(path.join('..', dotenvFile), import.meta.url) - ) -); +const dotenvFile = environment === 'production' ? '.env.production' : '.env.development'; +const dotenvFileLocal = `${dotenvFile}.local` -dotenv.config({ - path: dotenvFilePath +// Replicate dotenv-rails's behavior +const projectDir = path.resolve(url.fileURLToPath(new URL('..', import.meta.url))) +const dotenvFiles = ['.env', dotenvFile, '.env.local', dotenvFileLocal] + .map(s => path.join(projectDir, s)); +dotenvFiles.forEach(path => dotenv.config({path})); + +const subEnv = (s) => s.replaceAll(/\$\w+|\$\{\w+\}/g, (match) => { + const name = match.startsWith('${') ? match.slice(2, -1) : match.slice(1); + if (name === 'PWD') { + return projectDir; + } + return process.env[name]; }); +if (process.env.REDIS_URL && process.env.PWD) { + process.env.REDIS_URL = process.env.REDIS_URL.replace(/\$PWD\b|$\{PWD\}/, projectDir); +} + initializeLogLevel(process.env, environment); /** @@ -52,16 +62,29 @@ initializeLogLevel(process.env, environment); * @param {RedisConfiguration} config * @returns {Promise} */ -const createRedisClient = async ({ redisParams, redisUrl }) => { +const createRedisClient = async (config) => { + const { redisParams, redisUrl } = config; + const parsed = url.parse(redisUrl + ''); + + // so apparently ioredis doesn't handle relative paths let client; - - if (typeof redisUrl === 'string') { - client = new Redis(redisUrl, redisParams); - } else { + if (!redisUrl) { + // @ts-ignore client = new Redis(redisParams); + } else if (parsed.host === null && parsed?.path?.[0] === '.') { + redisParams.path = parsed.path; + // @ts-ignore + client = new Redis(redisParams); + } else if (parsed.host === '.' || parsed.protocol === 'unix:' && parsed.host !== '') { + redisParams.path = redisUrl.host + redisUrl.path; + // @ts-ignore + client = new Redis(redisParams); + } else { + // @ts-ignore + client = new Redis(redisUrl, redisParams);; } - - client.on('error', (err) => logger.error({ err }, 'Redis Client Error!')); + // @ts-ignore + client.on('error', (err) => logger.error(err, 'Redis Client Error!')); return client; }; @@ -122,7 +145,9 @@ const parseIntFromEnv = (value, defaultValue, variableName) => { * @returns {pg.PoolConfig} the configuration for the PostgreSQL connection */ const pgConfigFromEnv = (env) => { - /** @type {Record} */ + if (env.DB_HOST) { + env.DB_HOST = subEnv(env.DB_HOST); + } const pgConfigs = { development: { user: env.DB_USER || pg.defaults.user, @@ -260,7 +285,7 @@ const redisConfigFromEnv = (env) => { return { redisParams, redisPrefix, - redisUrl: typeof env.REDIS_URL === 'string' ? env.REDIS_URL : undefined, + redisUrl: env.REDIS_URL, }; };