th: fork docs and configuration

Co-authored-by: Rin <rin@rin.systems>
Co-authored-by: fox <fox@neko.business>

.env.development

@@ -1,3 +1,17 @@
+LOCAL_DOMAIN=localhost
+ALTERNATE_DOMAINS=mastodon.internal
+STREAMING_API_BASE_URL=https://streaming.mastodon.internal
+
+DB_HOST=$PWD/data/postgres
+DB_USER=mastodon
+DB_NAME=mastodon_dev
+REDIS_URL=unix://$PWD/data/redis/redis-dev.sock
+TH_MENTION_SPAM_HEURISTIC_AUTO_LIMIT_ACTIVE=can-spam
+TH_MENTION_SPAM_THRESHOLD=2
+TH_STAFF_ACCOUNT=staff
+
+TH_USE_INVITE_QUOTA=1
+
 # Required by ActiveRecord encryption feature
 ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY=fkSxKD2bF396kdQbrP1EJ7WbU7ZgNokR
 ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT=r0hvVmzBVsjxC7AMlwhOzmtc36ZCOS1E

.env.production.sample

@@ -14,7 +14,7 @@
 # ----------
 # This identifies your server and cannot be changed safely later
 # ----------
-LOCAL_DOMAIN=example.com
+LOCAL_DOMAIN=localhost
 
 # Use this only if you need to run mastodon on a different domain than the one used for federation.
 # You can read more about this option on https://docs.joinmastodon.org/admin/config/#web-domain
@@ -25,6 +25,7 @@ LOCAL_DOMAIN=example.com
 # handler@example2.com etc. for the same user. LOCAL_DOMAIN should not
 # be added. Comma separated values
 # ALTERNATE_DOMAINS=example1.com,example2.com
+ALTERNATE_DOMAINS=mastodon.internal
 
 # Use HTTP proxy for outgoing request (optional)
 # http_proxy=http://gateway.local:8118
@@ -43,13 +44,13 @@ LOCAL_DOMAIN=example.com
 
 # Redis
 # -----
-REDIS_HOST=localhost
+REDIS_HOST=redis
 REDIS_PORT=6379
 
 
 # PostgreSQL
 # ----------
-DB_HOST=/var/run/postgresql
+DB_HOST=db
 DB_USER=mastodon
 DB_NAME=mastodon_production
 DB_PASS=

.env.test

@@ -4,6 +4,11 @@ NODE_ENV=production
 LOCAL_DOMAIN=cb6e6126.ngrok.io
 LOCAL_HTTPS=true
 
+DB_HOST=$(pwd)/data/postgres
+DB_USER=mastodon
+DB_NAME=mastodon_dev
+REDIS_URL=unix://./data/redis/redis-dev.sock
+
 # Secret values required by ActiveRecord encryption feature
 # Use `bin/rails db:encryption:init` to generate fresh secrets
 ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY=test_determinist_key_DO_NOT_USE_IN_PRODUCTION

.gitignore

@@ -4,6 +4,9 @@
 # or operating system, you probably want to add a global ignore instead:
 #   git config --global core.excludesfile '~/.gitignore_global'
 
+# Ignore local dotenv overrides
+.env.*.local
+
 # Ignore bundler config and downloaded libraries.
 /.bundle
 /vendor/bundle
@@ -12,6 +15,9 @@
 /db/*.sqlite3
 /db/*.sqlite3-journal
 
+# Ignore local data directory
+/data
+
 # Ignore all logfiles and tempfiles.
 .eslintcache
 /log/*
@@ -69,5 +75,14 @@ yarn-debug.log
 # Ignore Docker option files
 docker-compose.override.yml
 
+# Yarn Berry
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions
+
 # Ignore dotenv .local files
 .env*.local

CONTRIBUTING.md

@@ -1,11 +1,29 @@
-# Contributing to Mastodon Glitch Edition
+# Contributing to Mastodon Glitch+Treehouse Edition
 
-Thank you for your interest in contributing to the `glitch-soc` project!
+Thank you for your interest in contributing to the **Treehouse Mastodon** project!
 Here are some guidelines, and ways you can help.
 
 > (This document is a bit of a work-in-progress, so please bear with us.
 > If you don't see what you're looking for here, please don't hesitate to reach out!)
 
+## Merging
+
+If your username is kouhai, or you're otherwise merging from upstream glitch-soc
+for some reason, the following snippets may be useful:
+
+```sh
+git fetch glitch && git merge glitch/main && git checkout glitch/main -- yarn.lock
+```
+
+```sh
+export RAILS_ENV=production NODE_ENV=production
+export OTP_SECRET=precompile_placeholder SECRET_KEY_BASE=precompile_placeholder
+bundle install \
+&& yarn install \
+&& bundle exec rake assets:clobber \
+&& bundle exec rake webpacker:compile | tee /tmp/out.log
+```
+
 ## Translations
 
 You can submit glitch-soc-specific translations via [Crowdin](https://crowdin.com/project/glitch-soc). They are periodically merged into the codebase.
@@ -14,14 +32,19 @@ You can submit glitch-soc-specific translations via [Crowdin](https://crowdin.co
 
 ## Planning
 
-Right now a lot of the planning for this project takes place in our development Discord, or through GitHub Issues and Projects.
+Right now a lot of the planning for this project takes place in the `#fediverse`
+channel of the Treehouse Discord, or through Gitea Issues.
+
 We're working on ways to improve the planning structure and better solicit feedback, and if you feel like you can help in this respect, feel free to give us a holler.
 
 ## Documentation
 
-The documentation for this repository is available at [`glitch-soc/docs`](https://github.com/glitch-soc/docs) (online at [glitch-soc.github.io/docs/](https://glitch-soc.github.io/docs/)).
-Right now, we've mostly focused on the features that make this fork different from upstream in some manner.
-Adding screenshots, improving descriptions, and so forth are all ways to help contribute to the project even if you don't know any code.
+The upstream Glitch documentation for this repository is available at [`glitch-soc/docs`](https://github.com/glitch-soc/docs) (online at [glitch-soc.github.io/docs/](https://glitch-soc.github.io/docs/)).
+
+## Setup
+
+For a some-batteries-required guide to setting up a development environment for this repository, read Rin's excellent
+[SETUP.md](https://gitea.treehouse.systems/treehouse/mastodon/src/branch/main/SETUP.md).
 
 ## Frontend Development
 

DIVERGENCES.md

@@ -0,0 +1,14 @@
+# Divergences
+
+## Major Features
+
+- quote posting
+- Treehouse::Automod (experimental feature flagged)
+
+## Other Changes
+
+- various build system changes
+  - a better dockerfile
+  - yarn v2 (a mistake, tbh)
+  - various dev env changes
+- various css/style changes

Procfile.dev

@@ -1,4 +1,4 @@
 web: env PORT=3000 RAILS_ENV=development bundle exec puma -C config/puma.rb
 sidekiq: env PORT=3000 RAILS_ENV=development bundle exec sidekiq
-stream: env PORT=4000 yarn workspace @mastodon/streaming start
+stream: env PORT=4000 NODE_ENV=development yarn workspace @mastodon/streaming start | npx pino-pretty
 webpack: bin/webpack-dev-server

Rakefile

@@ -6,3 +6,12 @@
 require File.expand_path('config/application', __dir__)
 
 Rails.application.load_tasks
+
+# please don't do this
+if Rake::Task.task_defined?('assets:precompile') && ENV.include?('RAKE_NO_YARN_INSTALL_HACK')
+  task = Rake::Task['assets:precompile']
+  puts task.prerequisites
+  task.prerequisites.delete('webpacker:yarn_install')
+  task.prerequisites.delete('yarn:install')
+  puts task.prerequisites
+end

SETUP.md

@@ -0,0 +1,143 @@
+# Setting up a dev environment
+
+## Prerequisites
+
+Mastodon development requires the following:
+
+- Ruby 3.0
+- Ruby gems:
+  - `bundler`
+  - `irb`
+  - `foreman`
+- NodeJS v18 (LTS)
+- NPM packages:
+  - `yarn`
+- Postgres
+- Redis
+
+### macOS
+
+First, make sure you have Homebrew installed. Follow the instructions at [brew.sh](https://brew.sh).
+
+Run the following to install all necessary packages:
+```
+brew install ruby@3.0 foreman node yarn postgresql redis
+```
+
+Ruby 3.0 is **keg-only** by default. Follow the instructions in the **Caveat** to add it to your path.
+
+### Linux
+
+We will assume that you know how to locate the correct packages for your distro. That said, some distros package `bundler` and `irb` separately. Make sure that you also install these.
+
+On Arch, you will need:
+- `ruby`
+- `ruby-bundler`
+- `ruby-irb`
+- `ruby-foreman`
+- `redis`
+- `postgresql`
+- `yarn`
+- `gmp`
+- `libidn`
+
+### Windows
+
+Unfortunately, none of the authors use Windows. Contributions welcome!
+
+## Database
+
+In the root of this repository, go through the following script:
+```sh
+# Create a folder for local data
+mkdir -p data
+
+# Set up a local database
+pg_ctl -D data/postgres initdb -o '-U mastodon --auth-host=trust'
+
+# Use the data/postgres folder for the DB connection unix socket
+#
+# If you don't know what that means, it's just a way for Mastodon to communicate
+# with a database on the same machine efficiently.
+#
+# See: https://manpages.ubuntu.com/manpages/jammy/man7/unix.7.html
+echo 'unix_socket_directories = .' >> data/postgres/postgresql.conf
+
+# Start the database
+pg_ctl -D data/postgres start --silent
+```
+
+## Redis
+
+In the root of this repository, run the following:
+```sh
+# Create a folder for redis data
+mkdir -p data/redis
+
+# Start Redis
+redis-server ./redis-dev.conf
+
+# [Optional] Stop Redis
+# kill "$(cat ./data/redis/redis-dev.pid)"
+```
+
+## Ruby
+
+```sh
+export RAILS_ENV=development
+
+# Bundle installs all Ruby gems globally by default, which might cause problems.
+bundle config set --local path 'vendor/bundle'
+
+# [Apple Silicon] If using macOS on Apple Silicon, run the following:
+# bundle config build.idn-ruby -- --with-idn-dir="$(brew --prefix libidn)"
+
+# Install dependencies using bundle (Ruby) and yarn (JS)
+bundle install
+yarn install
+
+# Setup the database using the pre-defined Rake task
+#
+# Rake is a command runner for Ruby projects. The `bundle exec` ensures that
+# we use the version of Rake that this project requires.
+bundle exec rake db:setup
+
+# [Optional] If that fails, run the following and try again:
+# bundle exec rake db:reset
+```
+
+## Running Mastodon
+
+1. Run `export RAILS_ENV=development NODE_ENV=development`.
+    - Put these in your shell's .rc, or a script you can source if you want to skip this step in the future.
+2. Run `bundle exec rake assets:precompile`.
+    - If this explodes, complaining about `Hash`, you'll need to `export NODE_OPTIONS=--openssl-legacy-provider`.
+    - After doing this, you will need to run `bundle exec rake assets:clobber` and then re-run `bundle exec rake assets:precompile`.
+3. Run `foreman start`
+
+# Updates/Troubleshooting
+
+## RubyVM/DebugInspector Issues
+
+Still unable to fix. Circumvent by removing `better_errors` and `binding_of_caller` from Gemfile.
+
+Happy to troubleshoot with someone better with Ruby than us >_<'/.
+
+## Webpack Issues
+
+If Webpack compalins about being unable to find some assets or locales:
+
+Try:
+1. `rm -rf node_modules`
+2. `yarn install`
+
+If this doesn't help, try:
+1. `yarn add webpack`
+2. `git restore package.json yarn.lock`
+3. `yarn install`
+
+Then re-run `foreman start`. No. We have no idea why this worked.
+
+# Need Help?
+
+If the above instructions don't work, please contact @Rin here, or @tammy@social.treehouse.systems.

lib/mastodon/redis_config.rb

@@ -1,10 +1,29 @@
 # frozen_string_literal: true
 
+require 'pathname'
+
 def setup_redis_env_url(prefix = nil, defaults = true)
   prefix = "#{prefix.to_s.upcase}_" unless prefix.nil?
   prefix = '' if prefix.nil?
+  redis_url_key = "#{prefix}REDIS_URL"
 
-  return if ENV["#{prefix}REDIS_URL"].present?
+  if ENV[redis_url_key].present?
+    conn = +ENV["#{prefix}REDIS_URL"].sub(/redis:\/\//i, '')
+
+    # Strip any prefixing `unix://`
+    unix = !conn.sub!(/^unix:\/\//i, '').nil?
+    # Strip any prefixing `./`
+    unix |= conn.sub!(/^(\.\/)+/, '')
+    unix |= conn.start_with?('/')
+
+    if unix
+      pn = Pathname.new(conn)
+      pn = Pathname.getwd / pn if pn.relative?
+      ENV[redis_url_key] = "unix://#{pn}"
+    end
+
+    return
+  end
 
   password = ENV.fetch("#{prefix}REDIS_PASSWORD") { '' if defaults }
   host     = ENV.fetch("#{prefix}REDIS_HOST") { 'localhost' if defaults }

lib/tasks/deps.rake

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require 'pathname'
+
+DATA_DIR = Pathname.new('data')
+POSTGRES_DIR = DATA_DIR / 'postgres'
+POSTGRES_CONF_FILE = POSTGRES_DIR / 'postgresql.conf'
+POSTGRES_SOCKET_FILE = POSTGRES_DIR / '.s.PGSQL.5432'
+POSTGRES_PID_FILE = POSTGRES_DIR / 'postmaster.pid'
+REDIS_DIR = DATA_DIR / 'redis'
+REDIS_PID_FILE = REDIS_DIR / 'redis-dev.pid'
+
+def divider
+  puts '=========='
+end
+
+def get_pid(pid_file)
+  return false unless File.file?(pid_file)
+  pid = File.read(pid_file).to_i
+
+  Process.kill(0, pid)
+  pid
+rescue Errno::ESRCH
+  nil
+end
+
+def postgres_running?
+  get_pid POSTGRES_PID_FILE
+end
+
+directory REDIS_DIR.to_s
+
+namespace :deps do
+  task start: ['postgres:start', 'redis:start']
+  task stop: ['postgres:stop', 'redis:stop']
+
+  namespace :postgres do
+    namespace :setup do
+      task all: [POSTGRES_DIR.to_s]
+
+      file POSTGRES_DIR.to_s do
+        if POSTGRES_CONF_FILE.exist?
+          puts 'Postgres conf exists, skipping initdb'
+          next
+        end
+        sh %(printf '%s\\n' pg_ctl -D data/postgres initdb -o '-U mastodon --auth-host=trust')
+      end
+
+      task configure: [POSTGRES_DIR.to_s] do
+        next if File.foreach(POSTGRES_CONF_FILE).detect? { |line| line == /^unix_socket_directories = \.\s*$/ }
+
+        POSTGRES_CONF_FILE.open('at') do |f|
+          f.write("\n", PG_SOCKET_DIRECTORIES_LINE, "\n")
+        end
+      end
+    end
+
+    task start: ['setup:all'] do
+      if (pid = get_pid POSTGRES_PID_FILE)
+        puts "Postgres is running (pid #{pid})!"
+        next
+      end
+
+      puts 'Starting postgres...'
+      divider
+      sh %(pg_ctl -D ./data/postgres start)
+      divider
+    end
+
+    task :stop do
+      unless (pid = get_pid POSTGRES_PID_FILE)
+        puts "Postgres isn't running!"
+        next
+      end
+
+      puts "Stopping Postgres (pid #{pid})..."
+      sh %(pg_ctl -D ./data/postgres stop)
+      divider
+    end
+  end
+
+  namespace :redis do
+    task init: [REDIS_DIR.to_s] do
+    end
+
+    task start: [:init] do
+      if (pid = get_pid REDIS_PID_FILE)
+        puts "Redis is running (pid #{pid})!"
+        next
+      end
+
+      puts 'Starting redis...'
+      divider
+      sh %(redis-server redis-dev.conf)
+      divider
+    end
+
+    task :stop do
+      unless (pid = get_pid REDIS_PID_FILE)
+        puts "Redis isn't running!"
+        next
+      end
+
+      puts "Stopping Redis (pid #{pid})..."
+      divider
+      Process.kill(:TERM, pid)
+    end
+  end
+end

redis-dev.conf

@@ -0,0 +1,12 @@
+# This redis configuration is for development only
+
+daemonize yes
+
+dir ./data/redis
+pidfile ./redis-dev.pid
+
+unixsocket ./redis-dev.sock
+unixsocketperm 700
+
+# Disable TCP
+port 0

streaming/index.js

@@ -19,20 +19,30 @@ import { logger, httpLogger, initializeLogLevel, attachWebsocketHttpLogger, crea
 import { setupMetrics } from './metrics.js';
 import { isTruthy, normalizeHashtag, firstParam } from './utils.js';
 
-const environment = process.env.NODE_ENV || 'development';
+const environment = process.env.NODE_ENV !== 'development' ? 'production' : 'development';
 
 // Correctly detect and load .env or .env.production file based on environment:
-const dotenvFile = environment === 'production' ? '.env.production' : '.env';
-const dotenvFilePath = path.resolve(
-  url.fileURLToPath(
-    new URL(path.join('..', dotenvFile), import.meta.url)
-  )
-);
+const dotenvFile = environment === 'production' ? '.env.production' : '.env.development';
+const dotenvFileLocal = `${dotenvFile}.local`
 
-dotenv.config({
-  path: dotenvFilePath
+// Replicate dotenv-rails's behavior
+const projectDir = path.resolve(__dirname, '..')
+const dotenvFiles = ['.env', dotenvFile, '.env.local', dotenvFileLocal]
+  .map(s => path.join(projectDir, s));
+dotenvFiles.forEach(path => dotenv.config({path}));
+
+const subEnv = (s) => s.replaceAll(/\$\w+|\$\{\w+\}/g, (match) => {
+  const name = match.startsWith('${') ? match.slice(2, -1) : match.slice(1);
+  if (name === 'PWD') {
+    return projectDir;
+  }
+  return process.env[name];
 });
 
+if (process.env.REDIS_URL && process.env.PWD) {
+  process.env.REDIS_URL = process.env.REDIS_URL.replace(/\$PWD\b|$\{PWD\}/, projectDir);
+}
+
 initializeLogLevel(process.env, environment);
 
 /**
@@ -52,16 +62,29 @@ initializeLogLevel(process.env, environment);
  * @param {RedisConfiguration} config
  * @returns {Promise<Redis>}
  */
-const createRedisClient = async ({ redisParams, redisUrl }) => {
+const createRedisClient = async (config) => {
+  const { redisParams, redisUrl } = config;
+  const parsed = url.parse(redisUrl);
+
+  // so apparently ioredis doesn't handle relative paths
   let client;
-
-  if (typeof redisUrl === 'string') {
-    client = new Redis(redisUrl, redisParams);
-  } else {
+  if (!redisUrl) {
+    // @ts-ignore
     client = new Redis(redisParams);
+  } else if (parsed.host === null && parsed?.path?.[0] === '.') {
+    redisParams.path = parsed.path;
+    // @ts-ignore
+    client = new Redis(redisParams);
+  } else if (parsed.host === '.' || parsed.protocol === 'unix:' && parsed.host !== '') {
+    redisParams.path = redisUrl.host + redisUrl.path;
+    // @ts-ignore
+    client = new Redis(redisParams);
+  } else {
+    // @ts-ignore
+    client = new Redis(redisUrl, redisParams);;
   }
-
-  client.on('error', (err) => logger.error({ err }, 'Redis Client Error!'));
+  // @ts-ignore
+  client.on('error', (err) => logger.error(err, 'Redis Client Error!'));
 
   return client;
 };
@@ -122,7 +145,9 @@ const parseIntFromEnv = (value, defaultValue, variableName) => {
  * @returns {pg.PoolConfig} the configuration for the PostgreSQL connection
  */
 const pgConfigFromEnv = (env) => {
-  /** @type {Record<string, pg.PoolConfig>} */
+  if (env.DB_HOST) {
+    env.DB_HOST = subEnv(env.DB_HOST);
+  }
   const pgConfigs = {
     development: {
       user: env.DB_USER || pg.defaults.user,
@@ -257,10 +282,14 @@ const redisConfigFromEnv = (env) => {
     redisParams.path = env.REDIS_URL.slice(7);
   }
 
+  const redisUrlParams = env.REDIS_URL ? {
+    redisUrl: env.REDIS_URL,
+  } : {};
+
   return {
     redisParams,
     redisPrefix,
-    redisUrl: typeof env.REDIS_URL === 'string' ? env.REDIS_URL : undefined,
+    ...redisUrlParams,
   };
 };