luna/mastodon
0
0
Fork 0
mirror of https://github.com/lunaisnotaboy/mastodon.git synced 2024-12-06 21:45:35 +00:00
Code Issues Projects Releases Wiki Activity

Merge commit 'b1d89071384ef028c97a3d28cf8cf6bc0ca6c4ce' into glitch-soc/merge-upstream

Browse source
This commit is contained in:
Claire 2023-09-15 21:06:45 +02:00
parent 9f3df3edee b1d8907138
commit 08b495d95c
33 changed files with 759 additions and 327 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

113
.github/workflows/test-ruby.yml vendored
View file

@ -250,3 +250,116 @@ jobs:
with:
name: e2e-screenshots
path: tmp/screenshots/
test-search:
name: Testing search
runs-on: ubuntu-latest
needs:
- build
services:
postgres:
image: postgres:14-alpine
env:
POSTGRES_PASSWORD: postgres
POSTGRES_USER: postgres
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432:5432
redis:
image: redis:7-alpine
options: >-
--health-cmd "redis-cli ping"
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 6379:6379
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.17.13
env:
discovery.type: single-node
xpack.security.enabled: false
options: >-
--health-cmd "curl http://localhost:9200/_cluster/health"
--health-interval 10s
--health-timeout 5s
--health-retries 10
ports:
- 9200:9200
env:
DB_HOST: localhost
DB_USER: postgres
DB_PASS: postgres
DISABLE_SIMPLECOV: true
RAILS_ENV: test
BUNDLE_WITH: test
ES_ENABLED: true
ES_HOST: localhost
ES_PORT: 9200
strategy:
fail-fast: false
matrix:
ruby-version:
- '3.0'
- '3.1'
- '.ruby-version'
steps:
- uses: actions/checkout@v3
- uses: actions/download-artifact@v3
with:
path: './public'
name: ${{ github.sha }}
- name: Update package index
run: sudo apt-get update
- name: Set up Node.js
uses: actions/setup-node@v3
with:
cache: yarn
node-version-file: '.nvmrc'
- name: Install native Ruby dependencies
run: sudo apt-get install -y libicu-dev libidn11-dev
- name: Install additional system dependencies
run: sudo apt-get install -y ffmpeg imagemagick
- name: Set up bundler cache
uses: ruby/setup-ruby@v1
with:
ruby-version: ${{ matrix.ruby-version}}
bundler-cache: true
- run: yarn --frozen-lockfile
- name: Load database schema
run: './bin/rails db:create db:schema:load db:seed'
- run: bundle exec rake spec:search
- name: Archive logs
uses: actions/upload-artifact@v3
if: failure()
with:
name: test-search-logs-${{ matrix.ruby-version }}
path: log/
- name: Archive test screenshots
uses: actions/upload-artifact@v3
if: failure()
with:
name: test-search-screenshots
path: tmp/screenshots/

2
.nvmrc
View file

@ -1 +1 @@
16.20
20.6

2
Dockerfile
View file

@ -1,6 +1,6 @@
# syntax=docker/dockerfile:1.4
# This needs to be bookworm-slim because the Ruby image is built on bookworm-slim
ARG NODE_VERSION="16.20-bookworm-slim"
ARG NODE_VERSION="20.6-bookworm-slim"
FROM ghcr.io/moritzheiber/ruby-jemalloc:3.2.2-slim as ruby
FROM node:${NODE_VERSION} as build

2
Gemfile.lock
View file

@ -640,7 +640,7 @@ GEM
sidekiq (>= 5, < 8)
rspec-support (3.12.1)
rspec_chunked (0.6)
rubocop (1.56.2)
rubocop (1.56.3)
base64 (~> 0.1.1)
json (~> 2.3)
language_server-protocol (>= 3.17.0)

3
Vagrantfile vendored
View file

@ -76,7 +76,8 @@ path.logs: /var/log/elasticsearch
network.host: 0.0.0.0
http.port: 9200
discovery.seed_hosts: ["localhost"]
cluster.initial_master_nodes: ["node-1"]' > /etc/elasticsearch/elasticsearch.yml
cluster.initial_master_nodes: ["node-1"]
xpack.security.enabled: false' > /etc/elasticsearch/elasticsearch.yml
sudo systemctl restart elasticsearch

4
app/controllers/api/v1/directories_controller.rb
View file

@ -16,7 +16,9 @@ class Api::V1::DirectoriesController < Api::BaseController
end
def set_accounts
@accounts = accounts_scope.offset(params[:offset]).limit(limit_param(DEFAULT_ACCOUNTS_LIMIT))
with_read_replica do
@accounts = accounts_scope.offset(params[:offset]).limit(limit_param(DEFAULT_ACCOUNTS_LIMIT))
end
end
def accounts_scope

45
app/javascript/mastodon/actions/search.js
View file

@ -1,3 +1,7 @@
import { fromJS } from 'immutable';
import { searchHistory } from 'mastodon/settings';
import api from '../api';
import { fetchRelationships } from './accounts';
@ -15,8 +19,7 @@ export const SEARCH_EXPAND_REQUEST = 'SEARCH_EXPAND_REQUEST';
export const SEARCH_EXPAND_SUCCESS = 'SEARCH_EXPAND_SUCCESS';
export const SEARCH_EXPAND_FAIL = 'SEARCH_EXPAND_FAIL';
export const SEARCH_RESULT_CLICK = 'SEARCH_RESULT_CLICK';
export const SEARCH_RESULT_FORGET = 'SEARCH_RESULT_FORGET';
export const SEARCH_HISTORY_UPDATE = 'SEARCH_HISTORY_UPDATE';
export function changeSearch(value) {
return {
@ -170,16 +173,34 @@ export const openURL = (value, history, onFailure) => (dispatch, getState) => {
});
};
export const clickSearchResult = (q, type) => ({
type: SEARCH_RESULT_CLICK,
export const clickSearchResult = (q, type) => (dispatch, getState) => {
const previous = getState().getIn(['search', 'recent']);
const me = getState().getIn(['meta', 'me']);
const current = previous.add(fromJS({ type, q })).takeLast(4);
result: {
type,
q,
},
searchHistory.set(me, current.toJS());
dispatch(updateSearchHistory(current));
};
export const forgetSearchResult = q => (dispatch, getState) => {
const previous = getState().getIn(['search', 'recent']);
const me = getState().getIn(['meta', 'me']);
const current = previous.filterNot(result => result.get('q') === q);
searchHistory.set(me, current.toJS());
dispatch(updateSearchHistory(current));
};
export const updateSearchHistory = recent => ({
type: SEARCH_HISTORY_UPDATE,
recent,
});
export const forgetSearchResult = q => ({
type: SEARCH_RESULT_FORGET,
q,
});
export const hydrateSearch = () => (dispatch, getState) => {
const me = getState().getIn(['meta', 'me']);
const history = searchHistory.get(me);
if (history !== null) {
dispatch(updateSearchHistory(history));
}
};

2
app/javascript/mastodon/actions/store.js
View file

@ -2,6 +2,7 @@ import { Iterable, fromJS } from 'immutable';
import { hydrateCompose } from './compose';
import { importFetchedAccounts } from './importer';
import { hydrateSearch } from './search';
export const STORE_HYDRATE = 'STORE_HYDRATE';
export const STORE_HYDRATE_LAZY = 'STORE_HYDRATE_LAZY';
@ -20,6 +21,7 @@ export function hydrateStore(rawState) {
});
dispatch(hydrateCompose());
dispatch(hydrateSearch());
dispatch(importFetchedAccounts(Object.values(rawState.accounts)));
};
}

15
app/javascript/mastodon/features/audio/index.jsx
View file

@ -205,11 +205,11 @@ class Audio extends PureComponent {
};
toggleMute = () => {
const muted = !this.state.muted;
const muted = !(this.state.muted || this.state.volume === 0);
this.setState({ muted }, () => {
this.setState((state) => ({ muted, volume: Math.max(state.volume || 0.5, 0.05) }), () => {
if (this.gainNode) {
this.gainNode.gain.value = muted ? 0 : this.state.volume;
this.gainNode.gain.value = this.state.muted ? 0 : this.state.volume;
}
});
};
@ -287,7 +287,7 @@ class Audio extends PureComponent {
const { x } = getPointerPosition(this.volume, e);
if(!isNaN(x)) {
this.setState({ volume: x }, () => {
this.setState((state) => ({ volume: x, muted: state.muted && x === 0 }), () => {
if (this.gainNode) {
this.gainNode.gain.value = this.state.muted ? 0 : x;
}
@ -466,8 +466,9 @@ class Audio extends PureComponent {
render () {
const { src, intl, alt, lang, editable, autoPlay, sensitive, blurhash } = this.props;
const { paused, muted, volume, currentTime, duration, buffer, dragging, revealed } = this.state;
const { paused, volume, currentTime, duration, buffer, dragging, revealed } = this.state;
const progress = Math.min((currentTime / duration) * 100, 100);
const muted = this.state.muted || volume === 0;
let warning;
@ -557,12 +558,12 @@ class Audio extends PureComponent {
<button type='button' title={intl.formatMessage(muted ? messages.unmute : messages.mute)} aria-label={intl.formatMessage(muted ? messages.unmute : messages.mute)} className='player-button' onClick={this.toggleMute}><Icon id={muted ? 'volume-off' : 'volume-up'} fixedWidth /></button>
<div className={classNames('video-player__volume', { active: this.state.hovered })} ref={this.setVolumeRef} onMouseDown={this.handleVolumeMouseDown}>
<div className='video-player__volume__current' style={{ width: `${volume * 100}%`, backgroundColor: this._getAccentColor() }} />
<div className='video-player__volume__current' style={{ width: `${muted ? 0 : volume * 100}%`, backgroundColor: this._getAccentColor() }} />
<span
className='video-player__volume__handle'
tabIndex={0}
style={{ left: `${volume * 100}%`, backgroundColor: this._getAccentColor() }}
style={{ left: `${muted ? 0 : volume * 100}%`, backgroundColor: this._getAccentColor() }}
/>
</div>

25
app/javascript/mastodon/features/compose/components/search.jsx
View file

@ -16,6 +16,17 @@ const messages = defineMessages({
placeholderSignedIn: { id: 'search.search_or_paste', defaultMessage: 'Search or paste URL' },
});
const labelForRecentSearch = search => {
switch(search.get('type')) {
case 'account':
return `@${search.get('q')}`;
case 'hashtag':
return `#${search.get('q')}`;
default:
return search.get('q');
}
};
class Search extends PureComponent {
static contextTypes = {
@ -187,12 +198,16 @@ class Search extends PureComponent {
};
handleRecentSearchClick = search => {
const { onChange } = this.props;
const { router } = this.context;
if (search.get('type') === 'account') {
router.history.push(`/@${search.get('q')}`);
} else if (search.get('type') === 'hashtag') {
router.history.push(`/tags/${search.get('q')}`);
} else {
onChange(search.get('q'));
this._submit(search.get('type'));
}
this._unfocus();
@ -221,11 +236,15 @@ class Search extends PureComponent {
}
_submit (type) {
const { onSubmit, openInRoute } = this.props;
const { onSubmit, openInRoute, value, onClickSearchResult } = this.props;
const { router } = this.context;
onSubmit(type);
if (value) {
onClickSearchResult(value, type);
}
if (openInRoute) {
router.history.push('/search');
}
@ -243,7 +262,7 @@ class Search extends PureComponent {
const { recent } = this.props;
return recent.toArray().map(search => ({
label: search.get('type') === 'account' ? `@${search.get('q')}` : `#${search.get('q')}`,
label: labelForRecentSearch(search),
action: () => this.handleRecentSearchClick(search),
@ -359,7 +378,7 @@ class Search extends PureComponent {
{searchEnabled ? (
<div className='search__popout__menu'>
{this.defaultOptions.map(({ key, label, action }, i) => (
<button key={key} onMouseDown={action} className={classNames('search__popout__menu__item', { selected: selectedOption === (options.length + i) })}>
<button key={key} onMouseDown={action} className={classNames('search__popout__menu__item', { selected: selectedOption === ((options.length || recent.size) + i) })}>
{label}
</button>
))}

2
app/javascript/mastodon/features/compose/containers/search_container.js
View file

@ -15,7 +15,7 @@ import Search from '../components/search';
const mapStateToProps = state => ({
value: state.getIn(['search', 'value']),
submitted: state.getIn(['search', 'submitted']),
recent: state.getIn(['search', 'recent']),
recent: state.getIn(['search', 'recent']).reverse(),
});
const mapDispatchToProps = dispatch => ({

17
app/javascript/mastodon/features/video/index.jsx
View file

@ -217,8 +217,9 @@ class Video extends PureComponent {
const { x } = getPointerPosition(this.volume, e);
if(!isNaN(x)) {
this.setState({ volume: x }, () => {
this.setState((state) => ({ volume: x, muted: state.muted && x === 0 }), () => {
this.video.volume = x;
this.video.muted = this.state.muted;
});
}
}, 15);
@ -425,10 +426,11 @@ class Video extends PureComponent {
};
toggleMute = () => {
const muted = !this.video.muted;
const muted = !(this.video.muted || this.state.volume === 0);
this.setState({ muted }, () => {
this.video.muted = muted;
this.setState((state) => ({ muted, volume: Math.max(state.volume || 0.5, 0.05) }), () => {
this.video.volume = this.state.volume;
this.video.muted = this.state.muted;
});
};
@ -501,8 +503,9 @@ class Video extends PureComponent {
render () {
const { preview, src, aspectRatio, onOpenVideo, onCloseVideo, intl, alt, lang, detailed, sensitive, editable, blurhash, autoFocus } = this.props;
const { currentTime, duration, volume, buffer, dragging, paused, fullscreen, hovered, muted, revealed } = this.state;
const { currentTime, duration, volume, buffer, dragging, paused, fullscreen, hovered, revealed } = this.state;
const progress = Math.min((currentTime / duration) * 100, 100);
const muted = this.state.muted || volume === 0;
let preload;
@ -593,12 +596,12 @@ class Video extends PureComponent {
<button type='button' title={intl.formatMessage(muted ? messages.unmute : messages.mute)} aria-label={intl.formatMessage(muted ? messages.unmute : messages.mute)} className='player-button' onClick={this.toggleMute}><Icon id={muted ? 'volume-off' : 'volume-up'} fixedWidth /></button>
<div className={classNames('video-player__volume', { active: this.state.hovered })} onMouseDown={this.handleVolumeMouseDown} ref={this.setVolumeRef}>
<div className='video-player__volume__current' style={{ width: `${volume * 100}%` }} />
<div className='video-player__volume__current' style={{ width: `${muted ? 0 : volume * 100}%` }} />
<span
className={classNames('video-player__volume__handle')}
tabIndex={0}
style={{ left: `${volume * 100}%` }}
style={{ left: `${muted ? 0 : volume * 100}%` }}
/>
</div>

9
app/javascript/mastodon/reducers/search.js
View file

@ -14,8 +14,7 @@ import {
SEARCH_SHOW,
SEARCH_EXPAND_REQUEST,
SEARCH_EXPAND_SUCCESS,
SEARCH_RESULT_CLICK,
SEARCH_RESULT_FORGET,
SEARCH_HISTORY_UPDATE,
} from '../actions/search';
const initialState = ImmutableMap({
@ -73,10 +72,8 @@ export default function search(state = initialState, action) {
case SEARCH_EXPAND_SUCCESS:
const results = action.searchType === 'hashtags' ? ImmutableOrderedSet(fromJS(action.results.hashtags)) : action.results[action.searchType].map(item => item.id);
return state.updateIn(['results', action.searchType], list => list.union(results));
case SEARCH_RESULT_CLICK:
return state.update('recent', set => set.add(fromJS(action.result)));
case SEARCH_RESULT_FORGET:
return state.update('recent', set => set.filterNot(result => result.get('q') === action.q));
case SEARCH_HISTORY_UPDATE:
return state.set('recent', ImmutableOrderedSet(fromJS(action.recent)));
default:
return state;
}

1
app/javascript/mastodon/settings.js
View file

@ -46,3 +46,4 @@ export default class Settings {
export const pushNotificationsSetting = new Settings('mastodon_push_notification_data');
export const tagHistory = new Settings('mastodon_tag_history');
export const bannerSettings = new Settings('mastodon_banner_settings');
export const searchHistory = new Settings('mastodon_search_history');

3
app/javascript/styles/mastodon/components.scss
View file

@ -8877,7 +8877,6 @@ noscript {
border-radius: 8px;
border: 1px solid $highlight-text-color;
background: rgba($highlight-text-color, 0.15);
padding-inline-end: 45px;
overflow: hidden;
&__background-image {
@ -8940,7 +8939,7 @@ noscript {
position: absolute;
inset-inline-end: 0;
top: 0;
padding: 10px;
padding: 15px 10px;
.icon-button {
color: $highlight-text-color;

2
app/lib/search_query_parser.rb
View file

@ -8,7 +8,7 @@ class SearchQueryParser < Parslet::Parser
rule(:operator) { (str('+') | str('-')).as(:operator) }
rule(:prefix) { term >> colon }
rule(:shortcode) { (colon >> term >> colon.maybe).as(:shortcode) }
rule(:phrase) { (quote >> (term >> space.maybe).repeat >> quote).as(:phrase) }
rule(:phrase) { (quote >> (match('[^\s"]').repeat(1).as(:term) >> space.maybe).repeat >> quote).as(:phrase) }
rule(:clause) { (operator.maybe >> prefix.maybe.as(:prefix) >> (phrase | term | shortcode)).as(:clause) | prefix.as(:clause) | quote.as(:junk) }
rule(:query) { (clause >> space.maybe).repeat.as(:query) }
root(:query)

13
app/lib/search_query_transformer.rb
View file

@ -121,7 +121,7 @@ class SearchQueryTransformer < Parslet::Transform
def to_query
if @term.start_with?('#')
{ match: { tags: { query: @term } } }
{ match: { tags: { query: @term, operator: 'and' } } }
else
{ multi_match: { type: 'most_fields', query: @term, fields: ['text', 'text.stemmed'], operator: 'and' } }
end
@ -225,17 +225,16 @@ class SearchQueryTransformer < Parslet::Transform
rule(clause: subtree(:clause)) do
prefix = clause[:prefix][:term].to_s if clause[:prefix]
operator = clause[:operator]&.to_s
term = clause[:phrase] ? clause[:phrase].map { |term| term[:term].to_s }.join(' ') : clause[:term].to_s
if clause[:prefix] && SUPPORTED_PREFIXES.include?(prefix)
PrefixClause.new(prefix, operator, clause[:term].to_s, current_account: current_account)
PrefixClause.new(prefix, operator, term, current_account: current_account)
elsif clause[:prefix]
TermClause.new(operator, "#{prefix} #{clause[:term]}")
TermClause.new(operator, "#{prefix} #{term}")
elsif clause[:term]
TermClause.new(operator, clause[:term].to_s)
elsif clause[:shortcode]
TermClause.new(operator, ":#{clause[:term]}:")
TermClause.new(operator, term)
elsif clause[:phrase]
PhraseClause.new(operator, clause[:phrase].is_a?(Array) ? clause[:phrase].map { |p| p[:term].to_s }.join(' ') : clause[:phrase].to_s)
PhraseClause.new(operator, term)
else
raise "Unexpected clause type: #{clause}"
end

6
app/models/account.rb
View file

@ -129,10 +129,10 @@ class Account < ApplicationRecord
scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
scope :without_unapproved, -> { left_outer_joins(:user).merge(User.approved.confirmed).or(remote) }
scope :searchable, -> { without_unapproved.without_suspended.where(moved_to_account_id: nil) }
scope :discoverable, -> { searchable.without_silenced.where(discoverable: true).left_outer_joins(:account_stat) }
scope :discoverable, -> { searchable.without_silenced.where(discoverable: true).joins(:account_stat) }
scope :followable_by, ->(account) { joins(arel_table.join(Follow.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(Follow.arel_table[:target_account_id]).and(Follow.arel_table[:account_id].eq(account.id))).join_sources).where(Follow.arel_table[:id].eq(nil)).joins(arel_table.join(FollowRequest.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(FollowRequest.arel_table[:target_account_id]).and(FollowRequest.arel_table[:account_id].eq(account.id))).join_sources).where(FollowRequest.arel_table[:id].eq(nil)) }
scope :by_recent_status, -> { order(Arel.sql('(case when account_stats.last_status_at is null then 1 else 0 end) asc, account_stats.last_status_at desc, accounts.id desc')) }
scope :by_recent_sign_in, -> { order(Arel.sql('(case when users.current_sign_in_at is null then 1 else 0 end) asc, users.current_sign_in_at desc, accounts.id desc')) }
scope :by_recent_status, -> { order(Arel.sql('account_stats.last_status_at DESC NULLS LAST')) }
scope :by_recent_sign_in, -> { order(Arel.sql('users.current_sign_in_at DESC NULLS LAST')) }
scope :popular, -> { order('account_stats.followers_count desc') }
scope :by_domain_and_subdomains, ->(domain) { where(domain: Instance.by_domain_and_subdomains(domain).select(:domain)) }
scope :not_excluded_by_account, ->(account) { where.not(id: account.excluded_from_timeline_account_ids) }

2
app/models/media_attachment.rb
View file

@ -171,7 +171,7 @@ class MediaAttachment < ApplicationRecord
DEFAULT_STYLES = [:original].freeze
GLOBAL_CONVERT_OPTIONS = {
all: '-quality 90 +profile "!icc,*" +set modify-date +set create-date',
all: '-quality 90 +profile "!icc,*" +set modify-date -define jpeg:dct-method=float +set create-date',
}.freeze
belongs_to :account, inverse_of: :media_attachments, optional: true

22
app/services/statuses_search_service.rb
View file

@ -8,6 +8,7 @@ class StatusesSearchService < BaseService
@limit = options[:limit].to_i
@offset = options[:offset].to_i
convert_deprecated_options!
status_search_results
end
@ -28,4 +29,25 @@ class StatusesSearchService < BaseService
def parsed_query
SearchQueryTransformer.new.apply(SearchQueryParser.new.parse(@query), current_account: @account)
end
def convert_deprecated_options!
syntax_options = []
if @options[:account_id]
username = Account.select(:username, :domain).find(@options[:account_id]).acct
syntax_options << "from:@#{username}"
end
if @options[:min_id]
timestamp = Mastodon::Snowflake.to_time(@options[:min_id])
syntax_options << "after:\"#{timestamp.iso8601}\""
end
if @options[:max_id]
timestamp = Mastodon::Snowflake.to_time(@options[:max_id])
syntax_options << "before:\"#{timestamp.iso8601}\""
end
@query = "#{@query} #{syntax_options.join(' ')}".strip if syntax_options.any?
end
end

2
config/webpack/production.js
View file

@ -4,7 +4,7 @@ const { createHash } = require('crypto');
const { readFileSync } = require('fs');
const { resolve } = require('path');
const CompressionPlugin = require('compression-webpack-plugin');
const CompressionPlugin = require('@renchap/compression-webpack-plugin');
const TerserPlugin = require('terser-webpack-plugin');
const { BundleAnalyzerPlugin } = require('webpack-bundle-analyzer');
const { merge } = require('webpack-merge');

9
db/migrate/20230907150100_add_index_account_stats_on_last_status_at_and_account_id.rb Normal file
View file

@ -0,0 +1,9 @@
# frozen_string_literal: true
class AddIndexAccountStatsOnLastStatusAtAndAccountId < ActiveRecord::Migration[7.0]
disable_ddl_transaction!
def change
add_index :account_stats, [:last_status_at, :account_id], order: { last_status_at: 'DESC NULLS LAST' }, algorithm: :concurrently
end
end

3
db/schema.rb
View file

@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema[7.0].define(version: 2023_09_04_134623) do
ActiveRecord::Schema[7.0].define(version: 2023_09_07_150100) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
@ -99,6 +99,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_09_04_134623) do
t.datetime "updated_at", precision: nil, null: false
t.datetime "last_status_at", precision: nil
t.index ["account_id"], name: "index_account_stats_on_account_id", unique: true
t.index ["last_status_at", "account_id"], name: "index_account_stats_on_last_status_at_and_account_id", order: { last_status_at: "DESC NULLS LAST" }
end
create_table "account_statuses_cleanup_policies", force: :cascade do |t|

4
lib/mastodon/snowflake.rb
View file

@ -104,6 +104,10 @@ module Mastodon::Snowflake
id
end
def to_time(id)
Time.at((id >> 16) / 1000).utc
end
private
def already_defined?

10
lib/tasks/spec.rake
View file

@ -9,3 +9,13 @@ if Rake::Task.task_defined?('spec:system')
Rake::Task['spec:system'].enhance ['spec:enable_system_specs']
end
if Rake::Task.task_defined?('spec:search')
namespace :spec do
task :enable_search_specs do # rubocop:disable Rails/RakeEnvironment
ENV['RUN_SEARCH_SPECS'] = 'true'
end
end
Rake::Task['spec:search'].enhance ['spec:enable_search_specs']
end

4
package.json
View file

@ -47,6 +47,7 @@
"@material-design-icons/svg": "^0.14.10",
"@rails/ujs": "^7.0.6",
"@reduxjs/toolkit": "^1.9.5",
"@renchap/compression-webpack-plugin": "^6.1.4",
"@svgr/webpack": "^5.5.0",
"abortcontroller-polyfill": "^1.7.5",
"atrament": "0.2.4",
@ -64,7 +65,6 @@
"classnames": "^2.3.2",
"cocoon-js-vanilla": "^1.3.0",
"color-blend": "^4.0.0",
"compression-webpack-plugin": "^6.1.1",
"core-js": "^3.30.2",
"cross-env": "^7.0.3",
"css-loader": "^5.2.7",
@ -139,7 +139,7 @@
"tiny-queue": "^0.2.1",
"twitter-text": "3.1.0",
"uuid": "^9.0.0",
"webpack": "^4.46.0",
"webpack": "^4.47.0",
"webpack-assets-manifest": "^4.0.6",
"webpack-bundle-analyzer": "^4.8.0",
"webpack-cli": "^3.3.12",

33
spec/controllers/api/v1/directories_controller_spec.rb
View file

@ -15,12 +15,13 @@ describe Api::V1::DirectoriesController do
describe 'GET #show' do
context 'with no params' do
before do
_local_unconfirmed_account = Fabricate(
local_unconfirmed_account = Fabricate(
:account,
domain: nil,
user: Fabricate(:user, confirmed_at: nil, approved: true),
username: 'local_unconfirmed'
)
local_unconfirmed_account.create_account_stat!
local_unapproved_account = Fabricate(
:account,
@ -28,15 +29,17 @@ describe Api::V1::DirectoriesController do
user: Fabricate(:user, confirmed_at: 10.days.ago),
username: 'local_unapproved'
)
local_unapproved_account.create_account_stat!
local_unapproved_account.user.update(approved: false)
_local_undiscoverable_account = Fabricate(
local_undiscoverable_account = Fabricate(
:account,
domain: nil,
user: Fabricate(:user, confirmed_at: 10.days.ago, approved: true),
discoverable: false,
username: 'local_undiscoverable'
)
local_undiscoverable_account.create_account_stat!
excluded_from_timeline_account = Fabricate(
:account,
@ -44,18 +47,20 @@ describe Api::V1::DirectoriesController do
discoverable: true,
username: 'remote_excluded_from_timeline'
)
excluded_from_timeline_account.create_account_stat!
Fabricate(:block, account: user.account, target_account: excluded_from_timeline_account)
_domain_blocked_account = Fabricate(
domain_blocked_account = Fabricate(
:account,
domain: 'test.example',
discoverable: true,
username: 'remote_domain_blocked'
)
domain_blocked_account.create_account_stat!
Fabricate(:account_domain_block, account: user.account, domain: 'test.example')
end
it 'returns only the local discoverable account' do
it 'returns the local discoverable account and the remote discoverable account' do
local_discoverable_account = Fabricate(
:account,
domain: nil,
@ -63,6 +68,7 @@ describe Api::V1::DirectoriesController do
discoverable: true,
username: 'local_discoverable'
)
local_discoverable_account.create_account_stat!
eligible_remote_account = Fabricate(
:account,
@ -70,13 +76,13 @@ describe Api::V1::DirectoriesController do
discoverable: true,
username: 'eligible_remote'
)
eligible_remote_account.create_account_stat!
get :show
expect(response).to have_http_status(200)
expect(body_as_json.size).to eq(2)
expect(body_as_json.first[:id]).to include(eligible_remote_account.id.to_s)
expect(body_as_json.second[:id]).to include(local_discoverable_account.id.to_s)
expect(body_as_json.pluck(:id)).to contain_exactly(eligible_remote_account.id.to_s, local_discoverable_account.id.to_s)
end
end
@ -85,6 +91,8 @@ describe Api::V1::DirectoriesController do
user = Fabricate(:user, confirmed_at: 10.days.ago, approved: true)
local_account = Fabricate(:account, domain: nil, user: user)
remote_account = Fabricate(:account, domain: 'host.example')
local_account.create_account_stat!
remote_account.create_account_stat!
get :show, params: { local: '1' }
@ -97,24 +105,23 @@ describe Api::V1::DirectoriesController do
context 'when ordered by active' do
it 'returns accounts in order of most recent status activity' do
status_old = Fabricate(:status)
travel_to 10.seconds.from_now
status_new = Fabricate(:status)
old_stat = Fabricate(:account_stat, last_status_at: 1.day.ago)
new_stat = Fabricate(:account_stat, last_status_at: 1.minute.ago)
get :show, params: { order: 'active' }
expect(response).to have_http_status(200)
expect(body_as_json.size).to eq(2)
expect(body_as_json.first[:id]).to include(status_new.account.id.to_s)
expect(body_as_json.second[:id]).to include(status_old.account.id.to_s)
expect(body_as_json.first[:id]).to include(new_stat.account_id.to_s)
expect(body_as_json.second[:id]).to include(old_stat.account_id.to_s)
end
end
context 'when ordered by new' do
it 'returns accounts in order of creation' do
account_old = Fabricate(:account)
account_old = Fabricate(:account_stat).account
travel_to 10.seconds.from_now
account_new = Fabricate(:account)
account_new = Fabricate(:account_stat).account
get :show, params: { order: 'new' }

20
spec/lib/search_query_transformer_spec.rb
View file

@ -57,4 +57,24 @@ describe SearchQueryTransformer do
expect(subject.send(:filter_clauses)).to be_empty
end
end
context 'with \'"hello world"\'' do
let(:query) { '"hello world"' }
it 'transforms clauses' do
expect(subject.send(:must_clauses).map(&:phrase)).to contain_exactly('hello world')
expect(subject.send(:must_not_clauses)).to be_empty
expect(subject.send(:filter_clauses)).to be_empty
end
end
context 'with \'before:"2022-01-01 23:00"\'' do
let(:query) { 'before:"2022-01-01 23:00"' }
it 'transforms clauses' do
expect(subject.send(:must_clauses)).to be_empty
expect(subject.send(:must_not_clauses)).to be_empty
expect(subject.send(:filter_clauses).map(&:term)).to contain_exactly(lt: '2022-01-01 23:00', time_zone: 'UTC')
end
end
end

29
spec/rails_helper.rb
View file

@ -4,11 +4,17 @@ ENV['RAILS_ENV'] ||= 'test'
# This needs to be defined before Rails is initialized
RUN_SYSTEM_SPECS = ENV.fetch('RUN_SYSTEM_SPECS', false)
RUN_SEARCH_SPECS = ENV.fetch('RUN_SEARCH_SPECS', false)
if RUN_SYSTEM_SPECS
STREAMING_PORT = ENV.fetch('TEST_STREAMING_PORT', '4020')
ENV['STREAMING_API_BASE_URL'] = "http://localhost:#{STREAMING_PORT}"
end
if RUN_SEARCH_SPECS
# Include any configuration or setups specific to search tests here
end
require File.expand_path('../config/environment', __dir__)
abort('The Rails environment is running in production mode!') if Rails.env.production?
@ -30,6 +36,7 @@ Sidekiq.logger = nil
# System tests config
DatabaseCleaner.strategy = [:deletion]
streaming_server_manager = StreamingServerManager.new
search_data_manager = SearchDataManager.new
Devise::Test::ControllerHelpers.module_eval do
alias_method :original_sign_in, :sign_in
@ -69,7 +76,14 @@ end
RSpec.configure do |config|
# This is set before running spec:system, see lib/tasks/tests.rake
config.filter_run_excluding type: :system unless RUN_SYSTEM_SPECS
config.filter_run_excluding type: lambda { |type|
case type
when :system
!RUN_SYSTEM_SPECS
when :search
!RUN_SEARCH_SPECS
end
}
config.fixture_path = Rails.root.join('spec', 'fixtures')
config.use_transactional_fixtures = true
config.order = 'random'
@ -113,10 +127,17 @@ RSpec.configure do |config|
Webpacker.compile
streaming_server_manager.start(port: STREAMING_PORT)
end
if RUN_SEARCH_SPECS
Chewy.strategy(:urgent)
search_data_manager.prepare_test_data
end
end
config.after :suite do
streaming_server_manager.stop
search_data_manager.cleanup_test_data if RUN_SEARCH_SPECS
end
config.around :each, type: :system do |example|
@ -137,6 +158,12 @@ RSpec.configure do |config|
self.use_transactional_tests = true
end
config.around :each, type: :search do |example|
search_data_manager.populate_indexes
example.run
search_data_manager.remove_indexes
end
config.before(:each) do |example|
unless example.metadata[:paperclip_processing]
allow_any_instance_of(Paperclip::Attachment).to receive(:post_process).and_return(true) # rubocop:disable RSpec/AnyInstance

51
spec/search/models/concerns/account_search_spec.rb Normal file
View file

@ -0,0 +1,51 @@
# frozen_string_literal: true
require 'rails_helper'
describe AccountSearch do
describe 'a non-discoverable account becoming discoverable' do
let(:account) { Account.find_by(username: 'search_test_account_1') }
context 'when picking a non-discoverable account' do
it 'its bio is not in the AccountsIndex' do
results = AccountsIndex.filter(term: { username: account.username })
expect(results.count).to eq(1)
expect(results.first.text).to be_nil
end
end
context 'when the non-discoverable account becomes discoverable' do
it 'its bio is added to the AccountsIndex' do
account.discoverable = true
account.save!
results = AccountsIndex.filter(term: { username: account.username })
expect(results.count).to eq(1)
expect(results.first.text).to eq(account.note)
end
end
end
describe 'a discoverable account becoming non-discoverable' do
let(:account) { Account.find_by(username: 'search_test_account_0') }
context 'when picking an discoverable account' do
it 'has its bio in the AccountsIndex' do
results = AccountsIndex.filter(term: { username: account.username })
expect(results.count).to eq(1)
expect(results.first.text).to eq(account.note)
end
end
context 'when the discoverable account becomes non-discoverable' do
it 'its bio is removed from the AccountsIndex' do
account.discoverable = false
account.save!
results = AccountsIndex.filter(term: { username: account.username })
expect(results.count).to eq(1)
expect(results.first.text).to be_nil
end
end
end
end

53
spec/search/models/concerns/account_statuses_search_spec.rb Normal file
View file

@ -0,0 +1,53 @@
# frozen_string_literal: true
require 'rails_helper'
describe AccountStatusesSearch do
describe 'a non-indexable account becoming indexable' do
let(:account) { Account.find_by(username: 'search_test_account_1') }
context 'when picking a non-indexable account' do
it 'has no statuses in the PublicStatusesIndex' do
expect(PublicStatusesIndex.filter(term: { account_id: account.id }).count).to eq(0)
end
it 'has statuses in the StatusesIndex' do
expect(StatusesIndex.filter(term: { account_id: account.id }).count).to eq(account.statuses.count)
end
end
context 'when the non-indexable account becomes indexable' do
it 'adds the public statuses to the PublicStatusesIndex' do
account.indexable = true
account.save!
expect(PublicStatusesIndex.filter(term: { account_id: account.id }).count).to eq(account.statuses.where(visibility: :public).count)
expect(StatusesIndex.filter(term: { account_id: account.id }).count).to eq(account.statuses.count)
end
end
end
describe 'an indexable account becoming non-indexable' do
let(:account) { Account.find_by(username: 'search_test_account_0') }
context 'when picking an indexable account' do
it 'has statuses in the PublicStatusesIndex' do
expect(PublicStatusesIndex.filter(term: { account_id: account.id }).count).to eq(account.statuses.where(visibility: :public).count)
end
it 'has statuses in the StatusesIndex' do
expect(StatusesIndex.filter(term: { account_id: account.id }).count).to eq(account.statuses.count)
end
end
context 'when the indexable account becomes non-indexable' do
it 'removes the statuses from the PublicStatusesIndex' do
account.indexable = false
account.save!
expect(PublicStatusesIndex.filter(term: { account_id: account.id }).count).to eq(0)
expect(StatusesIndex.filter(term: { account_id: account.id }).count).to eq(account.statuses.count)
end
end
end
end

42
spec/spec_helper.rb
View file

@ -129,3 +129,45 @@ class StreamingServerManager
@running_thread.join
end
end
class SearchDataManager
def prepare_test_data
4.times do |i|
username = "search_test_account_#{i}"
account = Fabricate.create(:account, username: username, indexable: i.even?, discoverable: i.even?, note: "Lover of #{i}.")
2.times do |j|
Fabricate.create(:status, account: account, text: "#{username}'s #{j} post", visibility: j.even? ? :public : :private)
end
end
3.times do |i|
Fabricate.create(:tag, name: "search_test_tag_#{i}")
end
end
def indexes
[
AccountsIndex,
PublicStatusesIndex,
StatusesIndex,
TagsIndex,
]
end
def populate_indexes
indexes.each do |index_class|
index_class.purge!
index_class.import!
end
end
def remove_indexes
indexes.each(&:delete!)
end
def cleanup_test_data
Status.destroy_all
Account.destroy_all
Tag.destroy_all
end
end

536
yarn.lock
View file

@ -88,7 +88,7 @@
"@jridgewell/trace-mapping" "^0.3.17"
jsesc "^2.5.1"
"@babel/generator@^7.22.5", "@babel/generator@^7.7.2":
"@babel/generator@^7.7.2":
version "7.22.10"
resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.22.10.tgz#c92254361f398e160645ac58831069707382b722"
integrity sha512-79KIf7YiWjjdZ81JnLujDRApWtl7BxTqWD88+FFdQEIOG8LJ0etDOM7CXuIgGJa55sGOwZVwuEsaLEm0PJ5/+A==
@ -262,7 +262,7 @@
dependencies:
"@babel/types" "^7.22.5"
"@babel/helper-split-export-declaration@^7.22.5", "@babel/helper-split-export-declaration@^7.22.6":
"@babel/helper-split-export-declaration@^7.22.6":
version "7.22.6"
resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz#322c61b7310c0997fe4c323955667f18fcefb91c"
integrity sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==
@ -320,16 +320,16 @@
chalk "^2.4.2"
js-tokens "^4.0.0"
"@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.20.7":
"@babel/parser@^7.1.0", "@babel/parser@^7.20.7", "@babel/parser@^7.22.15", "@babel/parser@^7.22.5":
version "7.22.16"
resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.22.16.tgz#180aead7f247305cce6551bea2720934e2fa2c95"
integrity sha512-+gPfKv8UWeKKeJTUxe59+OobVcrYHETCsORl61EmSkmgymguYk/X5bp7GuUIXaFsc6y++v8ZxPsLSSuujqDphA==
"@babel/parser@^7.14.7":
version "7.22.10"
resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.22.10.tgz#e37634f9a12a1716136c44624ef54283cabd3f55"
integrity sha512-lNbdGsQb9ekfsnjFGhEiF4hfFqGgfOP3H3d27re3n+CGhNuTSUEQdfWk556sTLNTloczcdM5TYF2LhzmDQKyvQ==
"@babel/parser@^7.22.15", "@babel/parser@^7.22.5":
version "7.22.15"
resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.22.15.tgz#d34592bfe288a32e741aa0663dbc4829fcd55160"
integrity sha512-RWmQ/sklUN9BvGGpCDgSubhHWfAx24XDTDObup4ffvxaYsptOg2P3KG0j+1eWKLxpkX0j0uHxmpq2Z1SP/VhxA==
"@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@^7.22.15":
version "7.22.15"
resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.22.15.tgz#02dc8a03f613ed5fdc29fb2f728397c78146c962"
@ -1112,23 +1112,7 @@
"@babel/parser" "^7.22.5"
"@babel/types" "^7.22.5"
"@babel/traverse@7":
version "7.22.5"
resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.22.5.tgz#44bd276690db6f4940fdb84e1cb4abd2f729ccd1"
integrity sha512-7DuIjPgERaNo6r+PZwItpjCZEa5vyw4eJGufeLxrPdBXBoLcCJCIasvK6pK/9DVNrLZTLFhUGqaC6X/PA007TQ==
dependencies:
"@babel/code-frame" "^7.22.5"
"@babel/generator" "^7.22.5"
"@babel/helper-environment-visitor" "^7.22.5"
"@babel/helper-function-name" "^7.22.5"
"@babel/helper-hoist-variables" "^7.22.5"
"@babel/helper-split-export-declaration" "^7.22.5"
"@babel/parser" "^7.22.5"
"@babel/types" "^7.22.5"
debug "^4.1.0"
globals "^11.1.0"
"@babel/traverse@^7.22.15":
"@babel/traverse@7", "@babel/traverse@^7.22.15":
version "7.22.15"
resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.22.15.tgz#75be4d2d6e216e880e93017f4e2389aeb77ef2d9"
integrity sha512-DdHPwvJY0sEeN4xJU5uRLmZjgMMDIvMPniLuYzUVXj/GGzysPl0/fwt44JBkyUIzGJPV8QgHMcQdQ34XFuKTYQ==
@ -1144,16 +1128,16 @@
debug "^4.1.0"
globals "^11.1.0"
"@babel/types@^7.0.0", "@babel/types@^7.20.7", "@babel/types@^7.3.3":
version "7.22.10"
resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.22.10.tgz#4a9e76446048f2c66982d1a989dd12b8a2d2dc03"
integrity sha512-obaoigiLrlDZ7TUQln/8m4mSqIW2QFeOrCQc9r+xsaHGNoplVNYlRVpsfE8Vj35GEm2ZH4ZhrNYogs/3fj85kg==
"@babel/types@^7.0.0", "@babel/types@^7.12.11", "@babel/types@^7.20.7", "@babel/types@^7.22.10", "@babel/types@^7.22.15", "@babel/types@^7.22.5", "@babel/types@^7.4.4":
version "7.22.15"
resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.22.15.tgz#266cb21d2c5fd0b3931e7a91b6dd72d2f617d282"
integrity sha512-X+NLXr0N8XXmN5ZsaQdm9U2SSC3UbIYq/doL++sueHOTisgZHoKaQtZxGuV2cUPQHMfjKEfg/g6oy7Hm6SKFtA==
dependencies:
"@babel/helper-string-parser" "^7.22.5"
"@babel/helper-validator-identifier" "^7.22.5"
"@babel/helper-validator-identifier" "^7.22.15"
to-fast-properties "^2.0.0"
"@babel/types@^7.0.0-beta.49", "@babel/types@^7.12.11", "@babel/types@^7.12.6":
"@babel/types@^7.0.0-beta.49", "@babel/types@^7.12.6":
version "7.22.5"
resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.22.5.tgz#cd93eeaab025880a3a47ec881f4b096a5b786fbe"
integrity sha512-zo3MIHGOkPOfoRXitsgHLjEXmlDaD/5KU1Uzuc9GNiZPhSqVxVRtxuPaSBZDsYZ9qV88AjtMtWW7ww98loJ9KA==
@ -1162,13 +1146,13 @@
"@babel/helper-validator-identifier" "^7.22.5"
to-fast-properties "^2.0.0"
"@babel/types@^7.22.10", "@babel/types@^7.22.15", "@babel/types@^7.22.5", "@babel/types@^7.4.4":
version "7.22.15"
resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.22.15.tgz#266cb21d2c5fd0b3931e7a91b6dd72d2f617d282"
integrity sha512-X+NLXr0N8XXmN5ZsaQdm9U2SSC3UbIYq/doL++sueHOTisgZHoKaQtZxGuV2cUPQHMfjKEfg/g6oy7Hm6SKFtA==
"@babel/types@^7.3.3":
version "7.22.10"
resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.22.10.tgz#4a9e76446048f2c66982d1a989dd12b8a2d2dc03"
integrity sha512-obaoigiLrlDZ7TUQln/8m4mSqIW2QFeOrCQc9r+xsaHGNoplVNYlRVpsfE8Vj35GEm2ZH4ZhrNYogs/3fj85kg==
dependencies:
"@babel/helper-string-parser" "^7.22.5"
"@babel/helper-validator-identifier" "^7.22.15"
"@babel/helper-validator-identifier" "^7.22.5"
to-fast-properties "^2.0.0"
"@bcoe/v8-coverage@^0.2.3":
@ -1325,10 +1309,10 @@
minimatch "^3.1.2"
strip-json-comments "^3.1.1"
"@eslint/js@8.48.0":
version "8.48.0"
resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.48.0.tgz#642633964e217905436033a2bd08bf322849b7fb"
integrity sha512-ZSjtmelB7IJfWD2Fvb7+Z+ChTIKWq6kjda95fLcQKNS5aheVHn4IkfgRQE3sIIzTcSLwLcLZUD9UBt+V7+h+Pw==
"@eslint/js@8.49.0":
version "8.49.0"
resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.49.0.tgz#86f79756004a97fa4df866835093f1df3d03c333"
integrity sha512-1S8uAY/MTJqVx0SC4epBq+N2yhuwtNwLbJYNZyhL2pO1ZVKn5HFXav5T41Ryzy9K9V7ZId2JB2oy/W4aCd9/2w==