From 91c929a42cbb71b47af976ec021f4be235b9f3fe Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 19 Aug 2018 15:48:29 +0200 Subject: [PATCH 01/16] Keep scheduler jobs unique until they're done (#8287) --- app/workers/scheduler/backup_cleanup_scheduler.rb | 2 ++ app/workers/scheduler/doorkeeper_cleanup_scheduler.rb | 2 ++ app/workers/scheduler/email_scheduler.rb | 2 ++ app/workers/scheduler/feed_cleanup_scheduler.rb | 2 ++ app/workers/scheduler/ip_cleanup_scheduler.rb | 2 ++ app/workers/scheduler/media_cleanup_scheduler.rb | 2 ++ app/workers/scheduler/subscriptions_cleanup_scheduler.rb | 2 ++ app/workers/scheduler/subscriptions_scheduler.rb | 2 ++ app/workers/scheduler/user_cleanup_scheduler.rb | 2 ++ 9 files changed, 18 insertions(+) diff --git a/app/workers/scheduler/backup_cleanup_scheduler.rb b/app/workers/scheduler/backup_cleanup_scheduler.rb index 5ab16c057..cfdf6f4af 100644 --- a/app/workers/scheduler/backup_cleanup_scheduler.rb +++ b/app/workers/scheduler/backup_cleanup_scheduler.rb @@ -3,6 +3,8 @@ class Scheduler::BackupCleanupScheduler include Sidekiq::Worker + sidekiq_options unique: :until_executed + def perform old_backups.find_each(&:destroy!) end diff --git a/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb b/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb index bab4ae886..fec08c6bc 100644 --- a/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb +++ b/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb @@ -3,6 +3,8 @@ class Scheduler::DoorkeeperCleanupScheduler include Sidekiq::Worker + sidekiq_options unique: :until_executed + def perform Doorkeeper::AccessToken.where('revoked_at IS NOT NULL').where('revoked_at < NOW()').delete_all Doorkeeper::AccessGrant.where('revoked_at IS NOT NULL').where('revoked_at < NOW()').delete_all diff --git a/app/workers/scheduler/email_scheduler.rb b/app/workers/scheduler/email_scheduler.rb index 36866061b..1a5a1c826 100644 --- a/app/workers/scheduler/email_scheduler.rb +++ b/app/workers/scheduler/email_scheduler.rb @@ -3,6 +3,8 @@ class Scheduler::EmailScheduler include Sidekiq::Worker + sidekiq_options unique: :until_executed + def perform eligible_users.find_each do |user| next unless user.allows_digest_emails? diff --git a/app/workers/scheduler/feed_cleanup_scheduler.rb b/app/workers/scheduler/feed_cleanup_scheduler.rb index 42cf14128..b02bac883 100644 --- a/app/workers/scheduler/feed_cleanup_scheduler.rb +++ b/app/workers/scheduler/feed_cleanup_scheduler.rb @@ -3,6 +3,8 @@ class Scheduler::FeedCleanupScheduler include Sidekiq::Worker + sidekiq_options unique: :until_executed + def perform clean_home_feeds! clean_list_feeds! diff --git a/app/workers/scheduler/ip_cleanup_scheduler.rb b/app/workers/scheduler/ip_cleanup_scheduler.rb index 613a5e336..6bb93df7d 100644 --- a/app/workers/scheduler/ip_cleanup_scheduler.rb +++ b/app/workers/scheduler/ip_cleanup_scheduler.rb @@ -5,6 +5,8 @@ class Scheduler::IpCleanupScheduler RETENTION_PERIOD = 1.year + sidekiq_options unique: :until_executed + def perform time_ago = RETENTION_PERIOD.ago SessionActivation.where('updated_at < ?', time_ago).destroy_all diff --git a/app/workers/scheduler/media_cleanup_scheduler.rb b/app/workers/scheduler/media_cleanup_scheduler.rb index c35686fcb..a27e02953 100644 --- a/app/workers/scheduler/media_cleanup_scheduler.rb +++ b/app/workers/scheduler/media_cleanup_scheduler.rb @@ -3,6 +3,8 @@ class Scheduler::MediaCleanupScheduler include Sidekiq::Worker + sidekiq_options unique: :until_executed + def perform unattached_media.find_each(&:destroy) end diff --git a/app/workers/scheduler/subscriptions_cleanup_scheduler.rb b/app/workers/scheduler/subscriptions_cleanup_scheduler.rb index af2ae3120..06ba66205 100644 --- a/app/workers/scheduler/subscriptions_cleanup_scheduler.rb +++ b/app/workers/scheduler/subscriptions_cleanup_scheduler.rb @@ -3,6 +3,8 @@ class Scheduler::SubscriptionsCleanupScheduler include Sidekiq::Worker + sidekiq_options unique: :until_executed + def perform Subscription.expired.in_batches.delete_all end diff --git a/app/workers/scheduler/subscriptions_scheduler.rb b/app/workers/scheduler/subscriptions_scheduler.rb index dc16e85c2..4b0959af2 100644 --- a/app/workers/scheduler/subscriptions_scheduler.rb +++ b/app/workers/scheduler/subscriptions_scheduler.rb @@ -3,6 +3,8 @@ class Scheduler::SubscriptionsScheduler include Sidekiq::Worker + sidekiq_options unique: :until_executed + def perform Pubsubhubbub::SubscribeWorker.push_bulk(expiring_accounts.pluck(:id)) end diff --git a/app/workers/scheduler/user_cleanup_scheduler.rb b/app/workers/scheduler/user_cleanup_scheduler.rb index 245536cea..dde195bff 100644 --- a/app/workers/scheduler/user_cleanup_scheduler.rb +++ b/app/workers/scheduler/user_cleanup_scheduler.rb @@ -3,6 +3,8 @@ class Scheduler::UserCleanupScheduler include Sidekiq::Worker + sidekiq_options unique: :until_executed + def perform User.where('confirmed_at is NULL AND confirmation_sent_at <= ?', 2.days.ago).find_in_batches do |batch| Account.where(id: batch.map(&:account_id)).delete_all From 0d1d9b9a33149adbcdd063177eae9944462c91ff Mon Sep 17 00:00:00 2001 From: mayaeh Date: Sun, 19 Aug 2018 22:51:57 +0900 Subject: [PATCH 02/16] Fix report text color. (#8288) --- app/javascript/styles/mastodon/components.scss | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss index 7c58828fd..399e3a394 100644 --- a/app/javascript/styles/mastodon/components.scss +++ b/app/javascript/styles/mastodon/components.scss @@ -4176,6 +4176,10 @@ a.status-card { color: $highlight-text-color; } + .status__content p { + color: $inverted-text-color; + } + @media screen and (max-width: 480px) { max-height: 10vh; } From 9e75aa30cd229c45fd95e00a93416221c83a4fd7 Mon Sep 17 00:00:00 2001 From: abcang Date: Sun, 19 Aug 2018 22:52:38 +0900 Subject: [PATCH 03/16] Unuse ActiveRecord::Base#cache_key (#8185) * Unuse ActiveRecord::Base#cache_key * Enable cache_versioning * Call cache_collection --- app/controllers/accounts_controller.rb | 2 +- app/controllers/api/v1/statuses_controller.rb | 3 +-- app/controllers/application_controller.rb | 13 ++++--------- app/controllers/emojis_controller.rb | 2 +- app/controllers/statuses_controller.rb | 4 ++-- app/models/status.rb | 2 -- 6 files changed, 9 insertions(+), 17 deletions(-) diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb index e5a7301ee..f788a9078 100644 --- a/app/controllers/accounts_controller.rb +++ b/app/controllers/accounts_controller.rb @@ -42,7 +42,7 @@ class AccountsController < ApplicationController format.json do skip_session! - render_cached_json(['activitypub', 'actor', @account.cache_key], content_type: 'application/activity+json') do + render_cached_json(['activitypub', 'actor', @account], content_type: 'application/activity+json') do ActiveModelSerializers::SerializableResource.new(@account, serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter) end end diff --git a/app/controllers/api/v1/statuses_controller.rb b/app/controllers/api/v1/statuses_controller.rb index c6925d462..49a52f7a6 100644 --- a/app/controllers/api/v1/statuses_controller.rb +++ b/app/controllers/api/v1/statuses_controller.rb @@ -17,8 +17,7 @@ class Api::V1::StatusesController < Api::BaseController CONTEXT_LIMIT = 4_096 def show - cached = Rails.cache.read(@status.cache_key) - @status = cached unless cached.nil? + @status = cache_collection([@status], Status).first render json: @status, serializer: REST::StatusSerializer end diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 29ba6cad6..eafe27047 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -103,12 +103,8 @@ class ApplicationController < ActionController::Base return raw unless klass.respond_to?(:with_includes) raw = raw.cache_ids.to_a if raw.is_a?(ActiveRecord::Relation) - uncached_ids = [] - cached_keys_with_value = Rails.cache.read_multi(*raw.map(&:cache_key)) - - raw.each do |item| - uncached_ids << item.id unless cached_keys_with_value.key?(item.cache_key) - end + cached_keys_with_value = Rails.cache.read_multi(*raw).transform_keys(&:id) + uncached_ids = raw.map(&:id) - cached_keys_with_value.keys klass.reload_stale_associations!(cached_keys_with_value.values) if klass.respond_to?(:reload_stale_associations!) @@ -116,11 +112,11 @@ class ApplicationController < ActionController::Base uncached = klass.where(id: uncached_ids).with_includes.map { |item| [item.id, item] }.to_h uncached.each_value do |item| - Rails.cache.write(item.cache_key, item) + Rails.cache.write(item, item) end end - raw.map { |item| cached_keys_with_value[item.cache_key] || uncached[item.id] }.compact + raw.map { |item| cached_keys_with_value[item.id] || uncached[item.id] }.compact end def respond_with_error(code) @@ -135,7 +131,6 @@ class ApplicationController < ActionController::Base def render_cached_json(cache_key, **options) options[:expires_in] ||= 3.minutes - cache_key = cache_key.join(':') if cache_key.is_a?(Enumerable) cache_public = options.key?(:public) ? options.delete(:public) : true content_type = options.delete(:content_type) || 'application/json' diff --git a/app/controllers/emojis_controller.rb b/app/controllers/emojis_controller.rb index c9725ccc0..5d306e600 100644 --- a/app/controllers/emojis_controller.rb +++ b/app/controllers/emojis_controller.rb @@ -9,7 +9,7 @@ class EmojisController < ApplicationController format.json do skip_session! - render_cached_json(['activitypub', 'emoji', @emoji.cache_key], content_type: 'application/activity+json') do + render_cached_json(['activitypub', 'emoji', @emoji], content_type: 'application/activity+json') do ActiveModelSerializers::SerializableResource.new(@emoji, serializer: ActivityPub::EmojiSerializer, adapter: ActivityPub::Adapter) end end diff --git a/app/controllers/statuses_controller.rb b/app/controllers/statuses_controller.rb index 819fcfc70..24824aabb 100644 --- a/app/controllers/statuses_controller.rb +++ b/app/controllers/statuses_controller.rb @@ -33,7 +33,7 @@ class StatusesController < ApplicationController format.json do skip_session! unless @stream_entry.hidden? - render_cached_json(['activitypub', 'note', @status.cache_key], content_type: 'application/activity+json', public: !@stream_entry.hidden?) do + render_cached_json(['activitypub', 'note', @status], content_type: 'application/activity+json', public: !@stream_entry.hidden?) do ActiveModelSerializers::SerializableResource.new(@status, serializer: ActivityPub::NoteSerializer, adapter: ActivityPub::Adapter) end end @@ -43,7 +43,7 @@ class StatusesController < ApplicationController def activity skip_session! - render_cached_json(['activitypub', 'activity', @status.cache_key], content_type: 'application/activity+json', public: !@stream_entry.hidden?) do + render_cached_json(['activitypub', 'activity', @status], content_type: 'application/activity+json', public: !@stream_entry.hidden?) do ActiveModelSerializers::SerializableResource.new(@status, serializer: ActivityPub::ActivitySerializer, adapter: ActivityPub::Adapter) end end diff --git a/app/models/status.rb b/app/models/status.rb index 533d12354..6ba7b7a50 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -24,8 +24,6 @@ # class Status < ApplicationRecord - self.cache_versioning = false - include Paginable include Streamable include Cacheable From 62cd0974142c4b1162205859c3bef7dfd3aa7710 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20Miko=C5=82ajczak?= Date: Sun, 19 Aug 2018 17:05:36 +0200 Subject: [PATCH 04/16] i18n: Update Polish translation (#8290) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marcin Mikołajczak --- config/locales/pl.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/config/locales/pl.yml b/config/locales/pl.yml index 09e9e87f7..168975363 100644 --- a/config/locales/pl.yml +++ b/config/locales/pl.yml @@ -6,6 +6,7 @@ pl: about_this: O tej instancji administered_by: 'Administrowana przez:' api: API + apps: Aplikacje closed_registrations: Rejestracja na tej instancji jest obecnie zamknięta. Możesz jednak zarejestrować się na innej instancji, uzyskując dostęp do tej samej sieci. contact: Kontakt contact_missing: Nie ustawiono @@ -281,6 +282,7 @@ pl: search: Szukaj title: Znane instancje invites: + deactivate_all: Unieważnij wszystkie filter: all: Wszystkie available: Dostępne @@ -662,11 +664,14 @@ pl: publishing: Publikowanie web: Sieć remote_follow: - acct: Podaj swój adres (nazwa@domena), z którego chcesz śledzić + acct: Podaj swój adres (nazwa@domena), z którego chcesz wykonać działanie missing_resource: Nie udało się znaleźć adresu przekierowania z Twojej domeny no_account_html: Nie masz konta? Możesz zarejestrować się tutaj proceed: Śledź prompt: 'Zamierzasz śledzić:' + remote_interaction: + proceed: Przejdź do interakcji + prompt: 'Chcesz dokonać interakcji z tym wpisem:' remote_unfollow: error: Błąd title: Tytuł @@ -754,6 +759,7 @@ pl: private: Nie możesz przypiąć niepublicznego wpisu reblog: Nie możesz przypiąć podbicia wpisu show_more: Pokaż więcej + sign_in_to_participate: Zaloguj się, aby udzielić się w tej konwersacji title: '%{name}: "%{quote}"' visibilities: private: Tylko dla śledzących From 8fe1f8d4cecb9f2f749c1e1e77b2439dd640ddc5 Mon Sep 17 00:00:00 2001 From: cpsdqs Date: Sun, 19 Aug 2018 17:11:12 +0200 Subject: [PATCH 05/16] Make dropdown animations respect their placement (#8292) * Make dropdown animations respect their placement Also fix the corner radius on the privacy dropdown button when using top placement * Fix code style issue --- .../mastodon/components/dropdown_menu.js | 2 +- .../compose/components/privacy_dropdown.js | 8 +++-- .../styles/mastodon/components.scss | 30 +++++++++++++++++-- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/app/javascript/mastodon/components/dropdown_menu.js b/app/javascript/mastodon/components/dropdown_menu.js index 0a6e7c627..e83f724e9 100644 --- a/app/javascript/mastodon/components/dropdown_menu.js +++ b/app/javascript/mastodon/components/dropdown_menu.js @@ -137,7 +137,7 @@ class DropdownMenu extends React.PureComponent { // It should not be transformed when mounting because the resulting // size will be used to determine the coordinate of the menu by // react-overlays -
+
    diff --git a/app/javascript/mastodon/features/compose/components/privacy_dropdown.js b/app/javascript/mastodon/features/compose/components/privacy_dropdown.js index a772c1c95..e19778fd2 100644 --- a/app/javascript/mastodon/features/compose/components/privacy_dropdown.js +++ b/app/javascript/mastodon/features/compose/components/privacy_dropdown.js @@ -28,6 +28,7 @@ class PrivacyDropdownMenu extends React.PureComponent { style: PropTypes.object, items: PropTypes.array.isRequired, value: PropTypes.string.isRequired, + placement: PropTypes.string.isRequired, onClose: PropTypes.func.isRequired, onChange: PropTypes.func.isRequired, }; @@ -119,7 +120,7 @@ class PrivacyDropdownMenu extends React.PureComponent { render () { const { mounted } = this.state; - const { style, items, value } = this.props; + const { style, items, placement, value } = this.props; return ( @@ -127,7 +128,7 @@ class PrivacyDropdownMenu extends React.PureComponent { // It should not be transformed when mounting because the resulting // size will be used to determine the coordinate of the menu by // react-overlays -
    +
    {items.map(item => (
    @@ -226,7 +227,7 @@ export default class PrivacyDropdown extends React.PureComponent { const valueOption = this.options.find(item => item.value === value); return ( -
    +
    diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss index 399e3a394..b25a9df31 100644 --- a/app/javascript/styles/mastodon/components.scss +++ b/app/javascript/styles/mastodon/components.scss @@ -230,7 +230,6 @@ .dropdown-menu { position: absolute; - transform-origin: 50% 0; } .invisible { @@ -1634,6 +1633,22 @@ a.account__display-name { ul { list-style: none; } + + &.left { + transform-origin: 100% 50%; + } + + &.top { + transform-origin: 50% 100%; + } + + &.bottom { + transform-origin: 50% 0; + } + + &.right { + transform-origin: 0 50%; + } } .dropdown-menu__arrow { @@ -3300,7 +3315,14 @@ a.status-card { border-radius: 4px; margin-left: 40px; overflow: hidden; - transform-origin: 50% 0; + + &.top { + transform-origin: 50% 100%; + } + + &.bottom { + transform-origin: 50% 0; + } } .privacy-dropdown__option { @@ -3372,6 +3394,10 @@ a.status-card { } } + &.top .privacy-dropdown__value { + border-radius: 0 0 4px 4px; + } + .privacy-dropdown__dropdown { display: block; box-shadow: 2px 4px 6px rgba($base-shadow-color, 0.1); From 28163528e5655bba164ebe5c4d1a1deb1056c468 Mon Sep 17 00:00:00 2001 From: ThibG Date: Sun, 19 Aug 2018 18:44:18 +0200 Subject: [PATCH 06/16] Don't filter own toots (fixes #8289) (#8298) --- app/javascript/mastodon/selectors/index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/javascript/mastodon/selectors/index.js b/app/javascript/mastodon/selectors/index.js index 106198f74..70f08a8eb 100644 --- a/app/javascript/mastodon/selectors/index.js +++ b/app/javascript/mastodon/selectors/index.js @@ -1,5 +1,6 @@ import { createSelector } from 'reselect'; import { List as ImmutableList } from 'immutable'; +import { me } from '../initial_state'; const getAccountBase = (state, id) => state.getIn(['accounts', id], null); const getAccountCounters = (state, id) => state.getIn(['accounts_counters', id], null); @@ -83,7 +84,7 @@ export const makeGetStatus = () => { statusReblog = null; } - const regex = regexFromFilters(filters); + const regex = (accountReblog || accountBase).get('id') !== me && regexFromFilters(filters); const filtered = regex && regex.test(statusBase.get('reblog') ? statusReblog.get('search_index') : statusBase.get('search_index')); return statusBase.withMutations(map => { From 1d1e0171ec52806ff42e2cafa67b1c68afe1f7bb Mon Sep 17 00:00:00 2001 From: Oskari Noppa Date: Sun, 19 Aug 2018 20:16:40 +0300 Subject: [PATCH 07/16] Fix a variable for a Finnish translation (#8299) --- config/locales/fi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/locales/fi.yml b/config/locales/fi.yml index 7fa2bc8de..96339e35e 100644 --- a/config/locales/fi.yml +++ b/config/locales/fi.yml @@ -725,7 +725,7 @@ fi: tip_local_timeline: Paikallinen aikajana näyttää instanssin %{instance} käyttäjien julkaisut. He ovat naapureitasi! tip_mobile_webapp: Jos voit lisätä Mastodonin mobiiliselaimen kautta aloitusnäytöllesi, voit vastaanottaa push-ilmoituksia. Toiminta vastaa monin tavoin tavanomaista sovellusta! tips: Vinkkejä - title: Tervetuloa mukaan, %name}! + title: Tervetuloa mukaan, %{name}! users: invalid_email: Virheellinen sähköpostiosoite invalid_otp_token: Virheellinen kaksivaiheisen todentamisen koodi From 25f6f41052b52a765a74e9e16d7411750ae46221 Mon Sep 17 00:00:00 2001 From: Annika Backstrom Date: Sun, 19 Aug 2018 21:44:16 -0400 Subject: [PATCH 08/16] Add hotkey for follow requests (#8307) --- app/javascript/mastodon/features/ui/index.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/javascript/mastodon/features/ui/index.js b/app/javascript/mastodon/features/ui/index.js index 67484fc63..34d52a7d2 100644 --- a/app/javascript/mastodon/features/ui/index.js +++ b/app/javascript/mastodon/features/ui/index.js @@ -89,6 +89,7 @@ const keyMap = { goToProfile: 'g u', goToBlocked: 'g b', goToMuted: 'g m', + goToRequests: 'g r', toggleHidden: 'x', }; @@ -427,6 +428,10 @@ export default class UI extends React.PureComponent { this.context.router.history.push('/mutes'); } + handleHotkeyGoToRequests = () => { + this.context.router.history.push('/follow_requests'); + } + render () { const { draggingOver } = this.state; const { children, isComposing, location, dropdownMenuIsOpen } = this.props; @@ -449,6 +454,7 @@ export default class UI extends React.PureComponent { goToProfile: this.handleHotkeyGoToProfile, goToBlocked: this.handleHotkeyGoToBlocked, goToMuted: this.handleHotkeyGoToMuted, + goToRequests: this.handleHotkeyGoToRequests, }; return ( From 6226aa83d7eb0e03cc7d577eee1fe129cf0f36fc Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 20 Aug 2018 13:28:05 +0200 Subject: [PATCH 09/16] Increase reach of Delete->Actor activities (#8305) Fix #7316 --- app/services/suspend_account_service.rb | 8 +++++--- spec/services/suspend_account_service_spec.rb | 13 +++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb index 0a98f5fb9..8fc79b8ad 100644 --- a/app/services/suspend_account_service.rb +++ b/app/services/suspend_account_service.rb @@ -23,9 +23,7 @@ class SuspendAccountService < BaseService def purge_content! if @account.local? - ActivityPub::RawDistributionWorker.perform_async(delete_actor_json, @account.id) - - ActivityPub::DeliveryWorker.push_bulk(Relay.enabled.pluck(:inbox_url)) do |inbox_url| + ActivityPub::DeliveryWorker.push_bulk(delivery_inboxes) do |inbox_url| [delete_actor_json, @account.id, inbox_url] end end @@ -75,4 +73,8 @@ class SuspendAccountService < BaseService @delete_actor_json = Oj.dump(ActivityPub::LinkedDataSignature.new(payload).sign!(@account)) end + + def delivery_inboxes + Account.inboxes + Relay.enabled.pluck(:inbox_url) + end end diff --git a/spec/services/suspend_account_service_spec.rb b/spec/services/suspend_account_service_spec.rb index fd303a9d5..8a5bd3301 100644 --- a/spec/services/suspend_account_service_spec.rb +++ b/spec/services/suspend_account_service_spec.rb @@ -2,6 +2,11 @@ require 'rails_helper' RSpec.describe SuspendAccountService, type: :service do describe '#call' do + before do + stub_request(:post, "https://alice.com/inbox").to_return(status: 201) + stub_request(:post, "https://bob.com/inbox").to_return(status: 201) + end + subject do -> { described_class.new.call(account) } end @@ -14,6 +19,8 @@ RSpec.describe SuspendAccountService, type: :service do let!(:active_relationship) { Fabricate(:follow, account: account) } let!(:passive_relationship) { Fabricate(:follow, target_account: account) } let!(:subscription) { Fabricate(:subscription, account: account) } + let!(:remote_alice) { Fabricate(:account, inbox_url: 'https://alice.com/inbox', protocol: :activitypub) } + let!(:remote_bob) { Fabricate(:account, inbox_url: 'https://bob.com/inbox', protocol: :activitypub) } it 'deletes associated records' do is_expected.to change { @@ -29,5 +36,11 @@ RSpec.describe SuspendAccountService, type: :service do ].map(&:count) }.from([1, 1, 1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0, 0, 0]) end + + it 'sends a delete actor activity to all known inboxes' do + subject.call + expect(a_request(:post, "https://alice.com/inbox")).to have_been_made.once + expect(a_request(:post, "https://bob.com/inbox")).to have_been_made.once + end end end From 5fbf12bbb881181eb64c4d3e1b5f49b475572dc5 Mon Sep 17 00:00:00 2001 From: ThibG Date: Mon, 20 Aug 2018 13:32:43 +0200 Subject: [PATCH 10/16] Skip pagination logic entirely for pinned toots (fixes #8302) (#8310) --- app/javascript/mastodon/actions/timelines.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/javascript/mastodon/actions/timelines.js b/app/javascript/mastodon/actions/timelines.js index 11a199db6..e8fd441e1 100644 --- a/app/javascript/mastodon/actions/timelines.js +++ b/app/javascript/mastodon/actions/timelines.js @@ -55,7 +55,7 @@ export function expandTimeline(timelineId, path, params = {}, done = noOp) { return; } - if (!params.max_id && timeline.get('items', ImmutableList()).size > 0) { + if (!params.max_id && !params.pinned && timeline.get('items', ImmutableList()).size > 0) { params.since_id = timeline.getIn(['items', 0]); } From b34d6238cbd2be08e5df90b1dd99f266948fab8b Mon Sep 17 00:00:00 2001 From: ThibG Date: Mon, 20 Aug 2018 18:46:04 +0200 Subject: [PATCH 11/16] Add API endpoint to list featured accounts (fixes #8315) (#8317) --- .../api/v1/endorsements_controller.rb | 72 +++++++++++++++++++ app/models/account_pin.rb | 1 + config/routes.rb | 15 ++-- 3 files changed, 81 insertions(+), 7 deletions(-) create mode 100644 app/controllers/api/v1/endorsements_controller.rb diff --git a/app/controllers/api/v1/endorsements_controller.rb b/app/controllers/api/v1/endorsements_controller.rb new file mode 100644 index 000000000..0f04b488f --- /dev/null +++ b/app/controllers/api/v1/endorsements_controller.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +class Api::V1::EndorsementsController < Api::BaseController + before_action -> { doorkeeper_authorize! :read, :'read:accounts' } + before_action :require_user! + after_action :insert_pagination_headers + + respond_to :json + + def index + @accounts = load_accounts + render json: @accounts, each_serializer: REST::AccountSerializer + end + + private + + def load_accounts + if unlimited? + endorsed_accounts.all + else + endorsed_accounts.paginate_by_max_id( + limit_param(DEFAULT_ACCOUNTS_LIMIT), + params[:max_id], + params[:since_id] + ) + end + end + + def endorsed_accounts + current_account.endorsed_accounts + end + + def insert_pagination_headers + set_pagination_headers(next_path, prev_path) + end + + def next_path + return if unlimited? + + if records_continue? + api_v1_endorsements_url pagination_params(max_id: pagination_max_id) + end + end + + def prev_path + return if unlimited? + + unless @accounts.empty? + api_v1_endorsements_url pagination_params(since_id: pagination_since_id) + end + end + + def pagination_max_id + @accounts.last.id + end + + def pagination_since_id + @accounts.first.id + end + + def records_continue? + @accounts.size == limit_param(DEFAULT_ACCOUNTS_LIMIT) + end + + def pagination_params(core_params) + params.slice(:limit).permit(:limit).merge(core_params) + end + + def unlimited? + params[:limit] == '0' + end +end diff --git a/app/models/account_pin.rb b/app/models/account_pin.rb index 9a21c3405..b51d3d4cd 100644 --- a/app/models/account_pin.rb +++ b/app/models/account_pin.rb @@ -11,6 +11,7 @@ # class AccountPin < ApplicationRecord + include Paginable include RelationshipCacheable belongs_to :account diff --git a/config/routes.rb b/config/routes.rb index a8716aae5..da7cb8061 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -261,13 +261,14 @@ Rails.application.routes.draw do get '/search', to: 'search#index', as: :search - resources :follows, only: [:create] - resources :media, only: [:create, :update] - resources :blocks, only: [:index] - resources :mutes, only: [:index] - resources :favourites, only: [:index] - resources :reports, only: [:index, :create] - resources :filters, only: [:index, :create, :show, :update, :destroy] + resources :follows, only: [:create] + resources :media, only: [:create, :update] + resources :blocks, only: [:index] + resources :mutes, only: [:index] + resources :favourites, only: [:index] + resources :reports, only: [:index, :create] + resources :filters, only: [:index, :create, :show, :update, :destroy] + resources :endorsements, only: [:index] namespace :apps do get :verify_credentials, to: 'credentials#show' From 8ac5e651e1f6cdf4fdd9e1092abbf02e48b6bbc9 Mon Sep 17 00:00:00 2001 From: ThibG Date: Mon, 20 Aug 2018 22:42:02 +0200 Subject: [PATCH 12/16] Upgrade doorkeeper to 4.4.2 (#8321) --- Gemfile | 2 +- Gemfile.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 5a6afe9ad..31c3c8086 100644 --- a/Gemfile +++ b/Gemfile @@ -42,7 +42,7 @@ gem 'omniauth-cas', '~> 1.1' gem 'omniauth-saml', '~> 1.10' gem 'omniauth', '~> 1.2' -gem 'doorkeeper', '~> 4.2', '< 4.3' +gem 'doorkeeper', '~> 4.4' gem 'fast_blank', '~> 1.0' gem 'fastimage' gem 'goldfinger', '~> 2.1' diff --git a/Gemfile.lock b/Gemfile.lock index 32851c67b..fbffc0c2d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -181,7 +181,7 @@ GEM docile (1.3.0) domain_name (0.5.20180417) unf (>= 0.0.5, < 1.0.0) - doorkeeper (4.2.6) + doorkeeper (4.4.2) railties (>= 4.2) dotenv (2.2.2) dotenv-rails (2.2.2) @@ -672,7 +672,7 @@ DEPENDENCIES devise (~> 4.4) devise-two-factor (~> 3.0) devise_pam_authenticatable2 (~> 9.1) - doorkeeper (~> 4.2, < 4.3) + doorkeeper (~> 4.4) dotenv-rails (~> 2.2, < 2.3) fabrication (~> 2.20) faker (~> 1.8) From 6a0d4d36adb3b2efbd145a40248ee40bdbbb163f Mon Sep 17 00:00:00 2001 From: NecroTechno Date: Mon, 20 Aug 2018 22:58:57 +0100 Subject: [PATCH 13/16] reorder simple form ul to list vertically (fixes #8236) (#8322) * reorder simple form ul to list vertically (fixes #8236) * remove browser specific prefixes --- app/javascript/styles/mastodon/forms.scss | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/javascript/styles/mastodon/forms.scss b/app/javascript/styles/mastodon/forms.scss index 375c7b64b..22dbfa8cf 100644 --- a/app/javascript/styles/mastodon/forms.scss +++ b/app/javascript/styles/mastodon/forms.scss @@ -154,9 +154,8 @@ code { margin-bottom: 15px; } - li { - float: left; - width: 50%; + ul { + columns: 2; } } From 83746b63646088396232a348f4024b101f39c78a Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 21 Aug 2018 02:49:51 +0200 Subject: [PATCH 14/16] Remove annoying Notification#cache_ids scope warning (#8333) --- app/models/notification.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/models/notification.rb b/app/models/notification.rb index 4f6ec8e8e..b9bec0808 100644 --- a/app/models/notification.rb +++ b/app/models/notification.rb @@ -39,8 +39,6 @@ class Notification < ApplicationRecord validates :account_id, uniqueness: { scope: [:activity_type, :activity_id] } validates :activity_type, inclusion: { in: TYPE_CLASS_MAP.values } - scope :cache_ids, -> { select(:id, :updated_at, :activity_type, :activity_id) } - scope :browserable, ->(exclude_types = []) { types = TYPE_CLASS_MAP.values - activity_types_from_types(exclude_types + [:follow_request]) where(activity_type: types) @@ -68,6 +66,10 @@ class Notification < ApplicationRecord end class << self + def cache_ids + select(:id, :updated_at, :activity_type, :activity_id) + end + def reload_stale_associations!(cached_items) account_ids = (cached_items.map(&:from_account_id) + cached_items.map { |item| item.target_status&.account_id }.compact).uniq From d98de8ada743886c3cd48b2ad942d46b805af7a9 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 21 Aug 2018 12:25:50 +0200 Subject: [PATCH 15/16] Get rid of all batch order warnings (#8334) --- app/models/form/status_batch.rb | 4 ++-- app/services/after_block_domain_from_account_service.rb | 4 ++-- app/services/backup_service.rb | 4 ++-- app/services/block_domain_service.rb | 6 +++--- app/services/remove_status_service.rb | 4 ++-- app/workers/refollow_worker.rb | 2 +- app/workers/scheduler/backup_cleanup_scheduler.rb | 2 +- app/workers/scheduler/email_scheduler.rb | 2 +- app/workers/scheduler/user_cleanup_scheduler.rb | 2 +- lib/tasks/mastodon.rake | 8 ++++---- 10 files changed, 19 insertions(+), 19 deletions(-) diff --git a/app/models/form/status_batch.rb b/app/models/form/status_batch.rb index 4f08a3049..8f5fd1fa2 100644 --- a/app/models/form/status_batch.rb +++ b/app/models/form/status_batch.rb @@ -23,7 +23,7 @@ class Form::StatusBatch media_attached_status_ids = MediaAttachment.where(status_id: status_ids).pluck(:status_id) ApplicationRecord.transaction do - Status.where(id: media_attached_status_ids).find_each do |status| + Status.where(id: media_attached_status_ids).reorder(nil).find_each do |status| status.update!(sensitive: sensitive) log_action :update, status end @@ -35,7 +35,7 @@ class Form::StatusBatch end def delete_statuses - Status.where(id: status_ids).find_each do |status| + Status.where(id: status_ids).reorder(nil).find_each do |status| RemovalWorker.perform_async(status.id) log_action :destroy, status end diff --git a/app/services/after_block_domain_from_account_service.rb b/app/services/after_block_domain_from_account_service.rb index 0f1a8505d..56cc819fb 100644 --- a/app/services/after_block_domain_from_account_service.rb +++ b/app/services/after_block_domain_from_account_service.rb @@ -15,13 +15,13 @@ class AfterBlockDomainFromAccountService < BaseService private def reject_existing_followers! - @account.passive_relationships.where(account: Account.where(domain: @domain)).includes(:account).find_each do |follow| + @account.passive_relationships.where(account: Account.where(domain: @domain)).includes(:account).reorder(nil).find_each do |follow| reject_follow!(follow) end end def reject_pending_follow_requests! - FollowRequest.where(target_account: @account).where(account: Account.where(domain: @domain)).includes(:account).find_each do |follow_request| + FollowRequest.where(target_account: @account).where(account: Account.where(domain: @domain)).includes(:account).reorder(nil).find_each do |follow_request| reject_follow!(follow_request) end end diff --git a/app/services/backup_service.rb b/app/services/backup_service.rb index 8492c1117..da7db6462 100644 --- a/app/services/backup_service.rb +++ b/app/services/backup_service.rb @@ -18,7 +18,7 @@ class BackupService < BaseService def build_json! @collection = serialize(collection_presenter, ActivityPub::CollectionSerializer) - account.statuses.with_includes.find_in_batches do |statuses| + account.statuses.with_includes.reorder(nil).find_in_batches do |statuses| statuses.each do |status| item = serialize(status, ActivityPub::ActivitySerializer) item.delete(:'@context') @@ -60,7 +60,7 @@ class BackupService < BaseService end def dump_media_attachments!(tar) - MediaAttachment.attached.where(account: account).find_in_batches do |media_attachments| + MediaAttachment.attached.where(account: account).reorder(nil).find_in_batches do |media_attachments| media_attachments.each do |m| download_to_tar(tar, m.file, m.file.path) end diff --git a/app/services/block_domain_service.rb b/app/services/block_domain_service.rb index d082de40b..a1fe93665 100644 --- a/app/services/block_domain_service.rb +++ b/app/services/block_domain_service.rb @@ -43,14 +43,14 @@ class BlockDomainService < BaseService end def suspend_accounts! - blocked_domain_accounts.where(suspended: false).find_each do |account| + blocked_domain_accounts.where(suspended: false).reorder(nil).find_each do |account| UnsubscribeService.new.call(account) if account.subscribed? SuspendAccountService.new.call(account) end end def clear_account_images! - blocked_domain_accounts.find_each do |account| + blocked_domain_accounts.reorder(nil).find_each do |account| account.avatar.destroy if account.avatar.exists? account.header.destroy if account.header.exists? account.save @@ -58,7 +58,7 @@ class BlockDomainService < BaseService end def clear_account_attachments! - media_from_blocked_domain.find_each do |attachment| + media_from_blocked_domain.reorder(nil).find_each do |attachment| @affected_status_ids << attachment.status_id if attachment.status_id.present? attachment.file.destroy if attachment.file.exists? diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb index fb889140b..1a53093b8 100644 --- a/app/services/remove_status_service.rb +++ b/app/services/remove_status_service.rb @@ -43,13 +43,13 @@ class RemoveStatusService < BaseService end def remove_from_followers - @account.followers_for_local_distribution.find_each do |follower| + @account.followers_for_local_distribution.reorder(nil).find_each do |follower| FeedManager.instance.unpush_from_home(follower, @status) end end def remove_from_lists - @account.lists_for_local_distribution.select(:id, :account_id).find_each do |list| + @account.lists_for_local_distribution.select(:id, :account_id).reorder(nil).find_each do |list| FeedManager.instance.unpush_from_list(list, @status) end end diff --git a/app/workers/refollow_worker.rb b/app/workers/refollow_worker.rb index 66bcd27c3..12f2bf671 100644 --- a/app/workers/refollow_worker.rb +++ b/app/workers/refollow_worker.rb @@ -9,7 +9,7 @@ class RefollowWorker target_account = Account.find(target_account_id) return unless target_account.protocol == :activitypub - target_account.followers.where(domain: nil).find_each do |follower| + target_account.followers.where(domain: nil).reorder(nil).find_each do |follower| # Locally unfollow remote account follower.unfollow!(target_account) diff --git a/app/workers/scheduler/backup_cleanup_scheduler.rb b/app/workers/scheduler/backup_cleanup_scheduler.rb index cfdf6f4af..023a77307 100644 --- a/app/workers/scheduler/backup_cleanup_scheduler.rb +++ b/app/workers/scheduler/backup_cleanup_scheduler.rb @@ -6,7 +6,7 @@ class Scheduler::BackupCleanupScheduler sidekiq_options unique: :until_executed def perform - old_backups.find_each(&:destroy!) + old_backups.reorder(nil).find_each(&:destroy!) end private diff --git a/app/workers/scheduler/email_scheduler.rb b/app/workers/scheduler/email_scheduler.rb index 1a5a1c826..24117e424 100644 --- a/app/workers/scheduler/email_scheduler.rb +++ b/app/workers/scheduler/email_scheduler.rb @@ -6,7 +6,7 @@ class Scheduler::EmailScheduler sidekiq_options unique: :until_executed def perform - eligible_users.find_each do |user| + eligible_users.reorder(nil).find_each do |user| next unless user.allows_digest_emails? DigestMailerWorker.perform_async(user.id) end diff --git a/app/workers/scheduler/user_cleanup_scheduler.rb b/app/workers/scheduler/user_cleanup_scheduler.rb index dde195bff..626fb1652 100644 --- a/app/workers/scheduler/user_cleanup_scheduler.rb +++ b/app/workers/scheduler/user_cleanup_scheduler.rb @@ -6,7 +6,7 @@ class Scheduler::UserCleanupScheduler sidekiq_options unique: :until_executed def perform - User.where('confirmed_at is NULL AND confirmation_sent_at <= ?', 2.days.ago).find_in_batches do |batch| + User.where('confirmed_at is NULL AND confirmation_sent_at <= ?', 2.days.ago).reorder(nil).find_in_batches do |batch| Account.where(id: batch.map(&:account_id)).delete_all User.where(id: batch.map(&:id)).delete_all end diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index f693c8b5a..191ce634c 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -503,7 +503,7 @@ namespace :mastodon do desc 'Remove media attachments attributed to silenced accounts' task remove_silenced: :environment do nb_media_attachments = 0 - MediaAttachment.where(account: Account.silenced).select(:id).find_in_batches do |media_attachments| + MediaAttachment.where(account: Account.silenced).select(:id).reorder(nil).find_in_batches do |media_attachments| nb_media_attachments += media_attachments.length Maintenance::DestroyMediaWorker.push_bulk(media_attachments.map(&:id)) end @@ -515,7 +515,7 @@ namespace :mastodon do time_ago = ENV.fetch('NUM_DAYS') { 7 }.to_i.days.ago nb_media_attachments = 0 - MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id).find_in_batches do |media_attachments| + MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id).reorder(nil).find_in_batches do |media_attachments| nb_media_attachments += media_attachments.length Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id)) end @@ -535,7 +535,7 @@ namespace :mastodon do accounts = accounts.where(domain: ENV['DOMAIN']) if ENV['DOMAIN'].present? nb_accounts = 0 - accounts.select(:id).find_in_batches do |accounts_batch| + accounts.select(:id).reorder(nil).find_in_batches do |accounts_batch| nb_accounts += accounts_batch.length Maintenance::RedownloadAccountMediaWorker.push_bulk(accounts_batch.map(&:id)) end @@ -570,7 +570,7 @@ namespace :mastodon do desc 'Generates home timelines for users who logged in in the past two weeks' task build: :environment do - User.active.select(:id, :account_id).find_in_batches do |users| + User.active.select(:id, :account_id).reorder(nil).find_in_batches do |users| RegenerationWorker.push_bulk(users.map(&:account_id)) end end From f06fa099625e928e5858ea81a20be1eddf6c6fbb Mon Sep 17 00:00:00 2001 From: ThibG Date: Tue, 21 Aug 2018 17:53:01 +0200 Subject: [PATCH 16/16] Revert to using Paperclip's filesystem storage, and fix dangling records in remove_remote (#8339) * Fix uncaching worker * Revert to using Paperclip's filesystem backend instead of fog-local fog-local has lots of concurrency issues, causing failure to delete files, dangling file records, and spurious errors UncacheMediaWorker --- Gemfile | 1 - Gemfile.lock | 3 --- app/workers/maintenance/uncache_media_worker.rb | 2 +- config/initializers/paperclip.rb | 12 ++++-------- 4 files changed, 5 insertions(+), 13 deletions(-) diff --git a/Gemfile b/Gemfile index 31c3c8086..516d397a2 100644 --- a/Gemfile +++ b/Gemfile @@ -16,7 +16,6 @@ gem 'dotenv-rails', '~> 2.2', '< 2.3' gem 'aws-sdk-s3', '~> 1.9', require: false gem 'fog-core', '~> 1.45' -gem 'fog-local', '~> 0.5', require: false gem 'fog-openstack', '~> 0.1', require: false gem 'paperclip', '~> 6.0' gem 'paperclip-av-transcoder', '~> 0.6' diff --git a/Gemfile.lock b/Gemfile.lock index fbffc0c2d..ea0e3f0cd 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -220,8 +220,6 @@ GEM fog-json (1.0.2) fog-core (~> 1.0) multi_json (~> 1.10) - fog-local (0.5.0) - fog-core (>= 1.27, < 3.0) fog-openstack (0.1.25) fog-core (~> 1.40) fog-json (>= 1.0) @@ -679,7 +677,6 @@ DEPENDENCIES fast_blank (~> 1.0) fastimage fog-core (~> 1.45) - fog-local (~> 0.5) fog-openstack (~> 0.1) fuubar (~> 2.2) goldfinger (~> 2.1) diff --git a/app/workers/maintenance/uncache_media_worker.rb b/app/workers/maintenance/uncache_media_worker.rb index f6a51a1b8..2d1a670a7 100644 --- a/app/workers/maintenance/uncache_media_worker.rb +++ b/app/workers/maintenance/uncache_media_worker.rb @@ -8,7 +8,7 @@ class Maintenance::UncacheMediaWorker def perform(media_attachment_id) media = MediaAttachment.find(media_attachment_id) - return unless media.file.exists? + return if media.file.blank? media.file.destroy media.save diff --git a/config/initializers/paperclip.rb b/config/initializers/paperclip.rb index c134bc5b8..59ab9b9a1 100644 --- a/config/initializers/paperclip.rb +++ b/config/initializers/paperclip.rb @@ -74,14 +74,10 @@ elsif ENV['SWIFT_ENABLED'] == 'true' fog_public: true ) else - require 'fog/local' - Paperclip::Attachment.default_options.merge!( - fog_credentials: { - provider: 'Local', - local_root: ENV.fetch('PAPERCLIP_ROOT_PATH') { Rails.root.join('public', 'system') }, - }, - fog_directory: '', - fog_host: ENV.fetch('PAPERCLIP_ROOT_URL') { '/system' } + storage: :filesystem, + use_timestamp: true, + path: (ENV['PAPERCLIP_ROOT_PATH'] || ':rails_root/public/system') + '/:class/:attachment/:id_partition/:style/:filename', + url: (ENV['PAPERCLIP_ROOT_URL'] || '/system') + '/:class/:attachment/:id_partition/:style/:filename', ) end