Merge pull request #236 from glitch-soc/plaintext-mutes

Make keyword mutes operate on plain text input

.travis.yml

@@ -18,7 +18,6 @@ env:
     - LOCAL_DOMAIN=cb6e6126.ngrok.io
     - LOCAL_HTTPS=true
     - RAILS_ENV=test
-    - NOKOGIRI_USE_SYSTEM_LIBRARIES=true
     - PARALLEL_TEST_PROCESSORS=2
     - "PATH=$HOME:$PATH"
 

Gemfile

@@ -42,6 +42,7 @@ gem 'fast_blank', '~> 1.0'
 gem 'goldfinger', '~> 2.1'
 gem 'hiredis', '~> 0.6'
 gem 'redis-namespace', '~> 1.5'
+gem 'html2text'
 gem 'htmlentities', '~> 4.3'
 gem 'http', '~> 3.0'
 gem 'http_accept_language', '~> 2.1'

Gemfile.lock

@@ -205,6 +205,8 @@ GEM
     highline (1.7.10)
     hiredis (0.6.1)
     hkdf (0.3.0)
+    html2text (0.2.1)
+      nokogiri (~> 1.6)
     htmlentities (4.3.4)
     http (3.0.0)
       addressable (~> 2.3)
@@ -601,6 +603,7 @@ DEPENDENCIES
   goldfinger (~> 2.1)
   hamlit-rails (~> 0.2)
   hiredis (~> 0.6)
+  html2text
   htmlentities (~> 4.3)
   http (~> 3.0)
   http_accept_language (~> 2.1)

app/lib/feed_manager.rb

@@ -178,22 +178,7 @@ class FeedManager
   end
 
   def keyword_filter?(status, receiver_id)
-    text_matcher = Glitch::KeywordMute.text_matcher_for(receiver_id)
+    Glitch::KeywordMuteHelper.new(receiver_id).matches?(status)
-    tag_matcher  = Glitch::KeywordMute.tag_matcher_for(receiver_id)
-
-    should_filter   = text_matcher.matches?(status.text)
-    should_filter ||= text_matcher.matches?(status.spoiler_text)
-    should_filter ||= tag_matcher.matches?(status.tags)
-
-    if status.reblog?
-      reblog = status.reblog
-
-      should_filter ||= text_matcher.matches?(reblog.text)
-      should_filter ||= text_matcher.matches?(reblog.spoiler_text)
-      should_filter ||= tag_matcher.matches?(status.tags)
-    end
-
-    should_filter
   end
 
   def filter_from_mentions?(status, receiver_id)

app/models/glitch/keyword_mute_helper.rb

@@ -0,0 +1,27 @@
+require 'html2text'
+
+class Glitch::KeywordMuteHelper
+  attr_reader :text_matcher
+  attr_reader :tag_matcher
+
+  def initialize(receiver_id)
+    @text_matcher   = Glitch::KeywordMute.text_matcher_for(receiver_id)
+    @tag_matcher    = Glitch::KeywordMute.tag_matcher_for(receiver_id)
+  end
+
+  def matches?(status)
+    matchers_match?(status) || (status.reblog? && matchers_match?(status.reblog))
+  end
+
+  private
+
+  def matchers_match?(status)
+    text_matcher.matches?(prepare_text(status.text)) ||
+      text_matcher.matches?(prepare_text(status.spoiler_text)) ||
+      tag_matcher.matches?(status.tags)
+  end
+
+  def prepare_text(text)
+    Html2Text.convert(text)
+  end
+end

spec/models/glitch/keyword_mute_helper_spec.rb

@@ -0,0 +1,50 @@
+require 'rails_helper'
+
+RSpec.describe Glitch::KeywordMuteHelper do
+  describe '#matches?' do
+    let(:alice) { Fabricate(:account, username: 'alice').tap(&:save!) }
+    let(:helper) { Glitch::KeywordMuteHelper.new(alice) }
+
+    it 'ignores names of HTML tags in status text' do
+      status = Fabricate(:status, text: '<addr>uh example</addr>')
+      Glitch::KeywordMute.create!(account: alice, keyword: 'addr')
+
+      expect(helper.matches?(status)).to be false
+    end
+
+    it 'ignores properties of HTML tags in status text' do
+      status = Fabricate(:status, text: '<a href="https://www.example.org">uh example</a>')
+      Glitch::KeywordMute.create!(account: alice, keyword: 'href')
+
+      expect(helper.matches?(status)).to be false
+    end
+
+    it 'matches text inside HTML tags' do
+      status = Fabricate(:status, text: '<p>HEY THIS IS SOMETHING ANNOYING</p>')
+      Glitch::KeywordMute.create!(account: alice, keyword: 'annoying')
+
+      expect(helper.matches?(status)).to be true
+    end
+
+    it 'matches < in HTML-stripped text' do
+      status = Fabricate(:status, text: '<p>I <3 oats</p>')
+      Glitch::KeywordMute.create!(account: alice, keyword: '<3')
+
+      expect(helper.matches?(status)).to be true
+    end
+
+    it 'matches &lt; in HTML text' do
+      status = Fabricate(:status, text: '<p>I &lt;3 oats</p>')
+      Glitch::KeywordMute.create!(account: alice, keyword: '<3')
+
+      expect(helper.matches?(status)).to be true
+    end
+
+    it 'matches link hrefs in HTML text' do
+      status = Fabricate(:status, text: '<p><a href="https://example.com/it-was-milk">yep</a></p>')
+      Glitch::KeywordMute.create!(account: alice, keyword: 'milk')
+
+      expect(helper.matches?(status)).to be true
+    end
+  end
+end