From 65c42e5398304273b54916cc90441ade23317d85 Mon Sep 17 00:00:00 2001 From: multiple creatures Date: Fri, 2 Aug 2019 02:30:35 -0500 Subject: [PATCH] filters now have options to separately match post text, content warnings/titles, & hashtags + option to filter threads containing a matching post --- app/controllers/api/v1/filters_controller.rb | 2 +- app/controllers/filters_controller.rb | 2 +- app/helpers/filter_helper.rb | 51 +++++++++++++++++++ app/lib/feed_manager.rb | 44 +++------------- app/lib/status_filter.rb | 22 +++++--- app/models/custom_filter.rb | 5 ++ app/views/filters/_fields.html.haml | 6 ++- config/locales/simple_form.en.yml | 8 ++- ...0801212756_add_thread_to_custom_filters.rb | 5 ++ ...801213117_add_spoiler_to_custom_filters.rb | 5 ++ ...190801213606_add_tags_to_custom_filters.rb | 5 ++ ...22645_add_status_text_to_custom_filters.rb | 5 ++ ...t_status_text_bool_for_existing_filters.rb | 6 +++ db/schema.rb | 6 ++- 14 files changed, 122 insertions(+), 50 deletions(-) create mode 100644 app/helpers/filter_helper.rb create mode 100644 db/migrate/20190801212756_add_thread_to_custom_filters.rb create mode 100644 db/migrate/20190801213117_add_spoiler_to_custom_filters.rb create mode 100644 db/migrate/20190801213606_add_tags_to_custom_filters.rb create mode 100644 db/migrate/20190801222645_add_status_text_to_custom_filters.rb create mode 100644 db/migrate/20190801222823_set_status_text_bool_for_existing_filters.rb diff --git a/app/controllers/api/v1/filters_controller.rb b/app/controllers/api/v1/filters_controller.rb index 496964394..48177f41a 100644 --- a/app/controllers/api/v1/filters_controller.rb +++ b/app/controllers/api/v1/filters_controller.rb @@ -43,6 +43,6 @@ class Api::V1::FiltersController < Api::BaseController end def resource_params - params.permit(:phrase, :expires_in, :irreversible, :whole_word, :exclude_media, :media_only, context: []) + params.permit(:phrase, :expires_in, :irreversible, :whole_word, :exclude_media, :media_only, :status_text, :spoiler, :tags, context: []) end end diff --git a/app/controllers/filters_controller.rb b/app/controllers/filters_controller.rb index 2f6f98272..55ee05833 100644 --- a/app/controllers/filters_controller.rb +++ b/app/controllers/filters_controller.rb @@ -58,7 +58,7 @@ class FiltersController < ApplicationController end def resource_params - params.require(:custom_filter).permit(:phrase, :expires_in, :irreversible, :whole_word, :exclude_media, :media_only, context: []) + params.require(:custom_filter).permit(:phrase, :expires_in, :irreversible, :whole_word, :exclude_media, :spoiler, :tags, :thread, :media_only, context: []) end def set_body_classes diff --git a/app/helpers/filter_helper.rb b/app/helpers/filter_helper.rb new file mode 100644 index 000000000..861f37551 --- /dev/null +++ b/app/helpers/filter_helper.rb @@ -0,0 +1,51 @@ +module FilterHelper + def phrase_filtered?(status, receiver_id, context) + filters = Rails.cache.fetch("filters:#{receiver_id}") { CustomFilter.where(account_id: receiver_id).active_irreversible.to_a }.to_a + + filters.select! { |filter| filter.context.include?(context.to_s) && !filter.expired? } + + if status.media_attachments.any? + filters.delete_if { |filter| filter.exclude_media } + else + filters.delete_if { |filter| filter.media_only } + end + + return false if filters.empty? + + status = status.reblog if status.reblog? + status_text = Formatter.instance.plaintext(status) + spoiler_text = status.spoiler_text + tags = status.tags.pluck(:name).join("\n") + + filters.each do |filter| + if filter.whole_word + sb = filter.phrase =~ /\A[[:word:]]/ ? '\b' : '' + eb = filter.phrase =~ /[[:word:]]\z/ ? '\b' : '' + + regex = /(?mix:#{sb}#{Regexp.escape(filter.phrase)}#{eb})/ + else + regex = /#{Regexp.escape(filter.phrase)}/i + end + + matched = false + matched = true unless regex.match(status_text).nil? + matched = true unless spoiler_text.blank? || regex.match(spoiler_text).nil? + matched = true unless tags.empty? || tags_regex.match(tags).nil? + + if matched + filter_thread(receiver_id, status.conversation_id) if filter.thread + return true + end + end + + false + end + + def filter_thread(account_id, conversation_id) + Redis.cache.sadd("filtered_threads:#{account_id}", conversation_id) + end + + def filtering_thread?(account_id, conversation_id) + Redis.cache.sismember("filtered_threads:#{account_id}", conversation_id) + end +end diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index 9996291c2..6d8f46e6f 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -5,6 +5,7 @@ require 'singleton' class FeedManager include Singleton include Redisable + include FilterHelper MAX_ITEMS = 1666 @@ -156,6 +157,7 @@ class FeedManager def filter_from_home?(status, receiver_id) return false if receiver_id == status.account_id return true if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?) + return true if filtering_thread?(receiver_id, status.conversation_id) return true if phrase_filtered?(status, receiver_id, :home) check_for_blocks = status.active_mentions.pluck(:account_id) @@ -168,12 +170,12 @@ class FeedManager return true if blocks_or_mutes?(receiver_id, check_for_blocks, :home) - if status.reply? && !status.in_reply_to_account_id.nil? # Filter out if it's a reply - should_filter = !Follow.where(account_id: receiver_id, target_account_id: status.in_reply_to_account_id).exists? # and I'm not following the person it's a reply to - should_filter &&= receiver_id != status.in_reply_to_account_id # and it's not a reply to me - should_filter &&= status.account_id != status.in_reply_to_account_id # and it's not a self-reply + if status.reply? && !status.in_reply_to_account_id.nil? # Filter out if it's a reply + should_filter = !Follow.where(account_id: receiver_id, target_account_id: status.in_reply_to_account_id).exists? # and I'm not following the person it's a reply to + should_filter &&= receiver_id != status.in_reply_to_account_id # and it's not a reply to me + should_filter &&= status.account_id != status.in_reply_to_account_id # and it's not a self-reply return should_filter - elsif status.reblog? # Filter out a reblog + elsif status.reblog? # Filter out a reblog should_filter = Follow.where(account_id: receiver_id, target_account_id: status.account_id, show_reblogs: false).exists? # if the reblogger's reblogs are suppressed should_filter ||= (status.reblog.account.silenced? && !Follow.where(account_id: receiver_id, target_account_id: status.reblog.account_id).exists?) # or if the account is silenced and I'm not following them should_filter ||= Block.where(account_id: status.reblog.account_id, target_account_id: receiver_id).exists? # or if the author of the reblogged status is blocking me @@ -186,6 +188,7 @@ class FeedManager def filter_from_mentions?(status, receiver_id) return true if receiver_id == status.account_id + return true if filtering_thread?(receiver_id, status.conversation_id) return true if phrase_filtered?(status, receiver_id, :notifications) # This filter is called from NotifyService, but already after the sender of @@ -200,37 +203,6 @@ class FeedManager should_filter end - def phrase_filtered?(status, receiver_id, context) - active_filters = Rails.cache.fetch("filters:#{receiver_id}") { CustomFilter.where(account_id: receiver_id).active_irreversible.to_a }.to_a - - active_filters.select! { |filter| filter.context.include?(context.to_s) && !filter.expired? } - - if status.media_attachments.any? - active_filters.delete_if { |filter| filter.exclude_media } - else - active_filters.delete_if { |filter| filter.media_only } - end - - active_filters.map! do |filter| - if filter.whole_word - sb = filter.phrase =~ /\A[[:word:]]/ ? '\b' : '' - eb = filter.phrase =~ /[[:word:]]\z/ ? '\b' : '' - - /(?mix:#{sb}#{Regexp.escape(filter.phrase)}#{eb})/ - else - /#{Regexp.escape(filter.phrase)}/i - end - end - - return false if active_filters.empty? - - combined_regex = active_filters.reduce { |memo, obj| Regexp.union(memo, obj) } - status = status.reblog if status.reblog? - - !combined_regex.match(Formatter.instance.plaintext(status)).nil? || - (status.spoiler_text.present? && !combined_regex.match(status.spoiler_text).nil?) - end - # Adds a status to an account's feed, returning true if a status was # added, and false if it was not added to the feed. Note that this is # an internal helper: callers must call trim or push updates if diff --git a/app/lib/status_filter.rb b/app/lib/status_filter.rb index f12986c93..a28c8282a 100644 --- a/app/lib/status_filter.rb +++ b/app/lib/status_filter.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class StatusFilter + include FilterHelper + attr_reader :status, :account def initialize(status, account, preloaded_relations = {}) @@ -21,6 +23,7 @@ class StatusFilter end def filtered_status? + return true if filtering_thread?(account.id, status.conversation_id) blocking_account? || blocking_domain? || muting_account? || filtered_reference? end @@ -30,6 +33,9 @@ class StatusFilter return true if account.user_hides_replies_of_muted? && reply_to_muted? return true if account.user_hides_replies_of_blocker? && reply_to_blocker? + # filtered by user? + return true if phrase_filtered?(status, account.id, 'thread') + # kajiht has no filters if status has no mentions return false if status&.mentions.blank? @@ -43,23 +49,23 @@ class StatusFilter return true if account.user_hides_mentions_of_blocked? && mentioned_accounts.where.not(suspended_at: nil).exists? return true if mentioned_account_ids.any? do |mentioned_account_id| - return true if @preloaded_relations[:muting] && account.user_hides_mentions_of_muted? && @preloaded_relations[:muting][mentioned_account_id] - return true if @preloaded_relations[:blocking] && account.user_hides_mentions_of_blocked? && @preloaded_relations[:blocking][mentioned_account_id] + break true if @preloaded_relations[:muting] && account.user_hides_mentions_of_muted? && @preloaded_relations[:muting][mentioned_account_id] + break true if @preloaded_relations[:blocking] && account.user_hides_mentions_of_blocked? && @preloaded_relations[:blocking][mentioned_account_id] if @preloaded_relations[:blocked_by] - return true if account.user_hides_mentions_of_blocker? && @preloaded_relations[:blocked_by][mentioned_account_id] + break true if account.user_hides_mentions_of_blocker? && @preloaded_relations[:blocked_by][mentioned_account_id] else - return true if account.user_hides_mentions_of_blocker? && Block.where(account_id: mentioned_account_id, target_account_id: account.id).exists? + break true if account.user_hides_mentions_of_blocker? && Block.where(account_id: mentioned_account_id, target_account_id: account.id).exists? end - return false unless status.reply? - @preloaded_relations[:following] && account.user_hides_mentions_outside_scope? && status.private_visibility? && !@preloaded_relations[:following][mentioned_account_id] + break false unless status.reply? && status.private_visibility? && account.user_hides_mentions_outside_scope? + @preloaded_relations[:following] && !@preloaded_relations[:following][mentioned_account_id] end return true if !@preloaded_relations[:muting] && account.user_hides_mentions_of_muted? && account.muting?(mentioned_account_ids) return true if !@preloaded_relations[:blocking] && account.user_hides_mentions_of_blocked? && account.blocking?(mentioned_account_ids) - return false unless status.reply? - !@preloaded_relations[:following] && account.user_hides_mentions_outside_scope? && status.private_visibility? && (mentioned_account_ids - account.following_ids).any? + return false unless status.reply? && status.private_visibility? && account.user_hides_mentions_outside_scope? + !@preloaded_relations[:following] && (mentioned_account_ids - account.following_ids).any? end def reply_to_blocked? diff --git a/app/models/custom_filter.rb b/app/models/custom_filter.rb index d4cb2206e..a1db3940c 100644 --- a/app/models/custom_filter.rb +++ b/app/models/custom_filter.rb @@ -14,6 +14,10 @@ # whole_word :boolean default(TRUE), not null # exclude_media :boolean default(FALSE), not null # media_only :boolean default(FALSE), not null +# thread :boolean default(FALSE), not null +# spoiler :boolean default(FALSE), not null +# tags :boolean default(FALSE), not null +# status_text :boolean default(FALSE), not null # class CustomFilter < ApplicationRecord @@ -45,6 +49,7 @@ class CustomFilter < ApplicationRecord def remove_cache Rails.cache.delete("filters:#{account_id}") + Rails.cache.delete("filtered_threads:#{account_id}") Redis.current.publish("timeline:#{account_id}", Oj.dump(event: :filters_changed)) end diff --git a/app/views/filters/_fields.html.haml b/app/views/filters/_fields.html.haml index 84b380f13..8229cb728 100644 --- a/app/views/filters/_fields.html.haml +++ b/app/views/filters/_fields.html.haml @@ -16,5 +16,9 @@ = f.input :whole_word, wrapper: :with_label .fields-group - = f.input :exclude_media, wrapper: :with_label + = f.input :status_text, wrapper: :with_label + = f.input :spoiler, wrapper: :with_label + = f.input :tags, wrapper: :with_label + = f.input :thread, wrapper: :with_label = f.input :media_only, wrapper: :with_label + = f.input :exclude_media, wrapper: :with_label diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml index 82e4feabb..75ff9d69d 100644 --- a/config/locales/simple_form.en.yml +++ b/config/locales/simple_form.en.yml @@ -97,8 +97,12 @@ en: header: Header inbox_url: URL of the relay inbox irreversible: Drop instead of hide - exclude_media: Don't filter roars with attachments - media_only: Only filter roars with attachments + exclude_media: Filter roars WITHOUT attachments + status_text: Filter roars with matching body text + media_only: Filter roars with attachments + thread: Filter the entire thread this match is contained in + spoiler: Filter roars with matching content warnings or topics + tags: Filter roars with matching tag(s) locale: Interface language locked: Lock account max_uses: Max number of uses diff --git a/db/migrate/20190801212756_add_thread_to_custom_filters.rb b/db/migrate/20190801212756_add_thread_to_custom_filters.rb new file mode 100644 index 000000000..de2507451 --- /dev/null +++ b/db/migrate/20190801212756_add_thread_to_custom_filters.rb @@ -0,0 +1,5 @@ +class AddThreadToCustomFilters < ActiveRecord::Migration[5.2] + def change + safety_assured { add_column :custom_filters, :thread, :boolean, null: false, default: false } + end +end diff --git a/db/migrate/20190801213117_add_spoiler_to_custom_filters.rb b/db/migrate/20190801213117_add_spoiler_to_custom_filters.rb new file mode 100644 index 000000000..63aba46ac --- /dev/null +++ b/db/migrate/20190801213117_add_spoiler_to_custom_filters.rb @@ -0,0 +1,5 @@ +class AddSpoilerToCustomFilters < ActiveRecord::Migration[5.2] + def change + safety_assured { add_column :custom_filters, :spoiler, :boolean, null: false, default: false } + end +end diff --git a/db/migrate/20190801213606_add_tags_to_custom_filters.rb b/db/migrate/20190801213606_add_tags_to_custom_filters.rb new file mode 100644 index 000000000..4ed1a6e34 --- /dev/null +++ b/db/migrate/20190801213606_add_tags_to_custom_filters.rb @@ -0,0 +1,5 @@ +class AddTagsToCustomFilters < ActiveRecord::Migration[5.2] + def change + safety_assured { add_column :custom_filters, :tags, :boolean, null: false, default: false } + end +end diff --git a/db/migrate/20190801222645_add_status_text_to_custom_filters.rb b/db/migrate/20190801222645_add_status_text_to_custom_filters.rb new file mode 100644 index 000000000..cf4bbd28b --- /dev/null +++ b/db/migrate/20190801222645_add_status_text_to_custom_filters.rb @@ -0,0 +1,5 @@ +class AddStatusTextToCustomFilters < ActiveRecord::Migration[5.2] + def change + safety_assured { add_column :custom_filters, :status_text, :boolean, null: false, default: false } + end +end diff --git a/db/migrate/20190801222823_set_status_text_bool_for_existing_filters.rb b/db/migrate/20190801222823_set_status_text_bool_for_existing_filters.rb new file mode 100644 index 000000000..787c63929 --- /dev/null +++ b/db/migrate/20190801222823_set_status_text_bool_for_existing_filters.rb @@ -0,0 +1,6 @@ +class SetStatusTextBoolForExistingFilters < ActiveRecord::Migration[5.2] + def up + CustomFilter.where(status_text: false).in_batches.update_all(status_text: true) + CustomFilter.where(spoiler_text: false).in_batches.update_all(spoiler_text: true) + end +end diff --git a/db/schema.rb b/db/schema.rb index 4d7085515..9197404aa 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_07_30_213656) do +ActiveRecord::Schema.define(version: 2019_08_01_222823) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -251,6 +251,10 @@ ActiveRecord::Schema.define(version: 2019_07_30_213656) do t.boolean "whole_word", default: true, null: false t.boolean "exclude_media", default: false, null: false t.boolean "media_only", default: false, null: false + t.boolean "thread", default: false, null: false + t.boolean "spoiler", default: false, null: false + t.boolean "tags", default: false, null: false + t.boolean "status_text", default: false, null: false t.index ["account_id"], name: "index_custom_filters_on_account_id" end