Merge commit 'a83615edc9fc1ce3363ddcf1fc676806421a5f65' into glitch-soc/merge-upstream

Conflicts:
- `streaming/index.js`:
  Upstream has added a parameter to `streamFrom`, while glitch-soc had an
  extra parameter.
  Ported upstream changes.
This commit is contained in:
Claire 2023-09-19 12:59:23 +02:00
commit 22ee4e057d
5 changed files with 134 additions and 79 deletions

View file

@ -546,7 +546,7 @@ class Status extends ImmutablePureComponent {
const visibilityIcon = visibilityIconInfo[status.get('visibility')];
const {statusContentProps, hashtagBar} = getHashtagBarForStatus(status);
const expanded = !status.get('hidden')
const expanded = !status.get('hidden') || status.get('spoiler_text').length === 0;
return (
<HotKeys handlers={handlers}>

View file

@ -293,7 +293,7 @@ class DetailedStatus extends ImmutablePureComponent {
}
const {statusContentProps, hashtagBar} = getHashtagBarForStatus(status);
const expanded = !status.get('hidden')
const expanded = !status.get('hidden') || status.get('spoiler_text').length === 0;
return (
<div style={outerStyle}>

View file

@ -41,11 +41,14 @@ module Paperclip
@output_options['vframes'] = 1
when 'mp4'
unless eligible_to_passthrough?(metadata)
bitrate = (metadata.width * metadata.height * 30 * BITS_PER_PIXEL) / 1_000
size_limit_in_bits = MediaAttachment::VIDEO_LIMIT * 8
desired_bitrate = (metadata.width * metadata.height * 30 * BITS_PER_PIXEL).floor
maximum_bitrate = (size_limit_in_bits / metadata.duration).floor - 192_000 # Leave some space for the audio stream
bitrate = [desired_bitrate, maximum_bitrate].min
@output_options['b:v'] = "#{bitrate}k"
@output_options['maxrate'] = "#{bitrate + 192}k"
@output_options['bufsize'] = "#{bitrate * 5}k"
@output_options['b:v'] = bitrate
@output_options['maxrate'] = bitrate + 192_000
@output_options['bufsize'] = bitrate * 5
if high_vfr?(metadata)
@output_options['vsync'] = 'vfr'

View file

@ -152,6 +152,28 @@ const redisConfigFromEnv = (env) => {
};
};
const PUBLIC_CHANNELS = [
'public',
'public:media',
'public:local',
'public:local:media',
'public:remote',
'public:remote:media',
'hashtag',
'hashtag:local',
];
// Used for priming the counters/gauges for the various metrics that are
// per-channel
const CHANNEL_NAMES = [
'system',
'user',
'user:notification',
'list',
'direct',
...PUBLIC_CHANNELS
];
const startServer = async () => {
const app = express();
@ -203,9 +225,6 @@ const startServer = async () => {
labelNames: ['type'],
});
connectedClients.set({ type: 'websocket' }, 0);
connectedClients.set({ type: 'eventsource' }, 0);
const connectedChannels = new metrics.Gauge({
name: 'connected_channels',
help: 'The number of channels the streaming server is streaming to',
@ -217,6 +236,35 @@ const startServer = async () => {
help: 'The number of Redis channels the streaming server is subscribed to',
});
const redisMessagesReceived = new metrics.Counter({
name: 'redis_messages_received_total',
help: 'The total number of messages the streaming server has received from redis subscriptions'
});
const messagesSent = new metrics.Counter({
name: 'messages_sent_total',
help: 'The total number of messages the streaming server sent to clients per connection type',
labelNames: [ 'type' ]
});
// Prime the gauges so we don't loose metrics between restarts:
redisSubscriptions.set(0);
connectedClients.set({ type: 'websocket' }, 0);
connectedClients.set({ type: 'eventsource' }, 0);
// For each channel, initialize the gauges at zero; There's only a finite set of channels available
CHANNEL_NAMES.forEach(( channel ) => {
connectedChannels.set({ type: 'websocket', channel }, 0);
connectedChannels.set({ type: 'eventsource', channel }, 0);
})
// Prime the counters so that we don't loose metrics between restarts.
// Unfortunately counters don't support the set() API, so instead I'm using
// inc(0) to achieve the same result.
redisMessagesReceived.inc(0);
messagesSent.inc({ type: 'websocket' }, 0);
messagesSent.inc({ type: 'eventsource' }, 0);
// When checking metrics in the browser, the favicon is requested this
// prevents the request from falling through to the API Router, which would
// error for this endpoint:
@ -262,6 +310,8 @@ const startServer = async () => {
* @param {string} message
*/
const onRedisMessage = (channel, message) => {
redisMessagesReceived.inc();
const callbacks = subs[channel];
log.silly(`New message on channel ${redisPrefix}${channel}`);
@ -490,17 +540,6 @@ const startServer = async () => {
}
};
const PUBLIC_CHANNELS = [
'public',
'public:media',
'public:local',
'public:local:media',
'public:remote',
'public:remote:media',
'hashtag',
'hashtag:local',
];
/**
* @param {any} req
* @param {string|undefined} channelName
@ -705,11 +744,12 @@ const startServer = async () => {
* @param {any} req
* @param {function(string, string): void} output
* @param {undefined | function(string[], SubscriptionListener): void} attachCloseHandler
* @param {'websocket' | 'eventsource'} destinationType
* @param {boolean=} needsFiltering
* @param {boolean=} allowLocalOnly
* @returns {SubscriptionListener}
*/
const streamFrom = (ids, req, output, attachCloseHandler, needsFiltering = false, allowLocalOnly = false) => {
const streamFrom = (ids, req, output, attachCloseHandler, destinationType, needsFiltering = false, allowLocalOnly = false) => {
const accountId = req.accountId || req.remoteAddress;
log.verbose(req.requestId, `Starting stream from ${ids.join(', ')} for ${accountId}`);
@ -718,6 +758,8 @@ const startServer = async () => {
// TODO: Replace "string"-based delete payloads with object payloads:
const encodedPayload = typeof payload === 'object' ? JSON.stringify(payload) : payload;
messagesSent.labels({ type: destinationType }).inc(1);
log.silly(req.requestId, `Transmitting for ${accountId}: ${event} ${encodedPayload}`);
output(event, encodedPayload);
};
@ -1038,7 +1080,7 @@ const startServer = async () => {
const onSend = streamToHttp(req, res);
const onEnd = streamHttpEnd(req, subscriptionHeartbeat(channelIds));
streamFrom(channelIds, req, onSend, onEnd, options.needsFiltering, options.allowLocalOnly);
streamFrom(channelIds, req, onSend, onEnd, 'eventsource', options.needsFiltering, options.allowLocalOnly);
}).catch(err => {
log.verbose(req.requestId, 'Subscription error:', err.toString());
httpNotFound(res);
@ -1262,7 +1304,7 @@ const startServer = async () => {
const onSend = streamToWs(request, socket, streamNameFromChannelName(channelName, params));
const stopHeartbeat = subscriptionHeartbeat(channelIds);
const listener = streamFrom(channelIds, request, onSend, undefined, options.needsFiltering, options.allowLocalOnly);
const listener = streamFrom(channelIds, request, onSend, undefined, 'websocket', options.needsFiltering, options.allowLocalOnly);
connectedChannels.labels({ type: 'websocket', channel: channelName }).inc();

122
yarn.lock
View file

@ -1298,7 +1298,12 @@
dependencies:
eslint-visitor-keys "^3.3.0"
"@eslint-community/regexpp@^4.5.1", "@eslint-community/regexpp@^4.6.1":
"@eslint-community/regexpp@^4.5.1":
version "4.8.1"
resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.8.1.tgz#8c4bb756cc2aa7eaf13cfa5e69c83afb3260c20c"
integrity sha512-PWiOzLIUAjN/w5K17PoF4n6sKBw0gqLHPhywmYHP4t1VFQQVYeb1yWsJwnMVEMl3tUHME7X/SJPZLmtG7XBDxQ==
"@eslint-community/regexpp@^4.6.1":
version "4.8.0"
resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.8.0.tgz#11195513186f68d42fbf449f9a7136b2c0c92005"
integrity sha512-JylOEEzDiOryeUnFbQz+oViCXS0KsvR1mvHkoMiu5+UiBvy+RYX7tzlIIIEstF/gVa2tj9AQXk3dgnxv6KxhFg==
@ -2254,11 +2259,16 @@
"@types/tough-cookie" "*"
parse5 "^7.0.0"
"@types/json-schema@*", "@types/json-schema@^7.0.12", "@types/json-schema@^7.0.5", "@types/json-schema@^7.0.8":
"@types/json-schema@*", "@types/json-schema@^7.0.5", "@types/json-schema@^7.0.8":
version "7.0.12"
resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.12.tgz#d70faba7039d5fca54c83c7dbab41051d2b6f6cb"
integrity sha512-Hr5Jfhc9eYOQNPYO5WLDq/n4jqijdHNlDXjuAQkkt+mWdQR+XJToOHrsD4cPaMXpn6KO7y2+wM8AZEs8VpBLVA==
"@types/json-schema@^7.0.12":
version "7.0.13"
resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.13.tgz#02c24f4363176d2d18fc8b70b9f3c54aba178a85"
integrity sha512-RbSSoHliUbnXj3ny0CNFOoxrIDV6SUGyStHsvDqosw6CkdPV8TtWGlfecuK4ToyMEAql6pzNxgCFKanovUzlgQ==
"@types/json-stable-stringify@^1.0.32":
version "1.0.34"
resolved "https://registry.yarnpkg.com/@types/json-stable-stringify/-/json-stable-stringify-1.0.34.tgz#c0fb25e4d957e0ee2e497c1f553d7f8bb668fd75"
@ -2500,9 +2510,9 @@
integrity sha512-5cJ8CB4yAx7BH1oMvdU0Jh9lrEXyPkar6F9G/ERswkCuvP4KQZfZkSjcMbAICCpQTN4OuZn8tz0HiKv9TGZgrQ==
"@types/semver@^7.5.0":
version "7.5.1"
resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.1.tgz#0480eeb7221eb9bc398ad7432c9d7e14b1a5a367"
integrity sha512-cJRQXpObxfNKkFAZbJl2yjWtJCqELQIdShsogr1d2MilP8dKD9TE/nEKHkJgUNHdGKCQaf9HbIynuV2csLGVLg==
version "7.5.2"
resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.2.tgz#31f6eec1ed7ec23f4f05608d3a2d381df041f564"
integrity sha512-7aqorHYgdNO4DM36stTiGO3DvKoex9TQRwsJU6vMaFGyqpBA1MNZkz+PG3gaNUPpTAOYhT1WR7M1JyA3fbS9Cw==
"@types/send@*":
version "0.17.1"
@ -2601,15 +2611,15 @@
"@types/yargs-parser" "*"
"@typescript-eslint/eslint-plugin@^6.0.0":
version "6.6.0"
resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.6.0.tgz#19ba09aa34fd504696445100262e5a9e1b1d7024"
integrity sha512-CW9YDGTQnNYMIo5lMeuiIG08p4E0cXrXTbcZ2saT/ETE7dWUrNxlijsQeU04qAAKkILiLzdQz+cGFxCJjaZUmA==
version "6.7.2"
resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.7.2.tgz#f18cc75c9cceac8080a9dc2e7d166008c5207b9f"
integrity sha512-ooaHxlmSgZTM6CHYAFRlifqh1OAr3PAQEwi7lhYhaegbnXrnh7CDcHmc3+ihhbQC7H0i4JF0psI5ehzkF6Yl6Q==
dependencies:
"@eslint-community/regexpp" "^4.5.1"
"@typescript-eslint/scope-manager" "6.6.0"
"@typescript-eslint/type-utils" "6.6.0"
"@typescript-eslint/utils" "6.6.0"
"@typescript-eslint/visitor-keys" "6.6.0"
"@typescript-eslint/scope-manager" "6.7.2"
"@typescript-eslint/type-utils" "6.7.2"
"@typescript-eslint/utils" "6.7.2"
"@typescript-eslint/visitor-keys" "6.7.2"
debug "^4.3.4"
graphemer "^1.4.0"
ignore "^5.2.4"
@ -2618,31 +2628,31 @@
ts-api-utils "^1.0.1"
"@typescript-eslint/parser@^6.0.0":
version "6.6.0"
resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-6.6.0.tgz#fe323a7b4eafb6d5ea82b96216561810394a739e"
integrity sha512-setq5aJgUwtzGrhW177/i+DMLqBaJbdwGj2CPIVFFLE0NCliy5ujIdLHd2D1ysmlmsjdL2GWW+hR85neEfc12w==
version "6.7.2"
resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-6.7.2.tgz#e0ae93771441b9518e67d0660c79e3a105497af4"
integrity sha512-KA3E4ox0ws+SPyxQf9iSI25R6b4Ne78ORhNHeVKrPQnoYsb9UhieoiRoJgrzgEeKGOXhcY1i8YtOeCHHTDa6Fw==
dependencies:
"@typescript-eslint/scope-manager" "6.6.0"
"@typescript-eslint/types" "6.6.0"
"@typescript-eslint/typescript-estree" "6.6.0"
"@typescript-eslint/visitor-keys" "6.6.0"
"@typescript-eslint/scope-manager" "6.7.2"
"@typescript-eslint/types" "6.7.2"
"@typescript-eslint/typescript-estree" "6.7.2"
"@typescript-eslint/visitor-keys" "6.7.2"
debug "^4.3.4"
"@typescript-eslint/scope-manager@6.6.0":
version "6.6.0"
resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-6.6.0.tgz#57105d4419d6de971f7d2c30a2ff4ac40003f61a"
integrity sha512-pT08u5W/GT4KjPUmEtc2kSYvrH8x89cVzkA0Sy2aaOUIw6YxOIjA8ilwLr/1fLjOedX1QAuBpG9XggWqIIfERw==
"@typescript-eslint/scope-manager@6.7.2":
version "6.7.2"
resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-6.7.2.tgz#cf59a2095d2f894770c94be489648ad1c78dc689"
integrity sha512-bgi6plgyZjEqapr7u2mhxGR6E8WCzKNUFWNh6fkpVe9+yzRZeYtDTbsIBzKbcxI+r1qVWt6VIoMSNZ4r2A+6Yw==
dependencies:
"@typescript-eslint/types" "6.6.0"
"@typescript-eslint/visitor-keys" "6.6.0"
"@typescript-eslint/types" "6.7.2"
"@typescript-eslint/visitor-keys" "6.7.2"
"@typescript-eslint/type-utils@6.6.0":
version "6.6.0"
resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-6.6.0.tgz#14f651d13b884915c4fca0d27adeb652a4499e86"
integrity sha512-8m16fwAcEnQc69IpeDyokNO+D5spo0w1jepWWY2Q6y5ZKNuj5EhVQXjtVAeDDqvW6Yg7dhclbsz6rTtOvcwpHg==
"@typescript-eslint/type-utils@6.7.2":
version "6.7.2"
resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-6.7.2.tgz#ed921c9db87d72fa2939fee242d700561454f367"
integrity sha512-36F4fOYIROYRl0qj95dYKx6kybddLtsbmPIYNK0OBeXv2j9L5nZ17j9jmfy+bIDHKQgn2EZX+cofsqi8NPATBQ==
dependencies:
"@typescript-eslint/typescript-estree" "6.6.0"
"@typescript-eslint/utils" "6.6.0"
"@typescript-eslint/typescript-estree" "6.7.2"
"@typescript-eslint/utils" "6.7.2"
debug "^4.3.4"
ts-api-utils "^1.0.1"
@ -2651,10 +2661,10 @@
resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-5.62.0.tgz#258607e60effa309f067608931c3df6fed41fd2f"
integrity sha512-87NVngcbVXUahrRTqIK27gD2t5Cu1yuCXxbLcFtCzZGlfyVWWh8mLHkoxzjsB6DDNnvdL+fW8MiwPEJyGJQDgQ==
"@typescript-eslint/types@6.6.0":
version "6.6.0"
resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-6.6.0.tgz#95e7ea650a2b28bc5af5ea8907114a48f54618c2"
integrity sha512-CB6QpJQ6BAHlJXdwUmiaXDBmTqIE2bzGTDLADgvqtHWuhfNP3rAOK7kAgRMAET5rDRr9Utt+qAzRBdu3AhR3sg==
"@typescript-eslint/types@6.7.2":
version "6.7.2"
resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-6.7.2.tgz#75a615a6dbeca09cafd102fe7f465da1d8a3c066"
integrity sha512-flJYwMYgnUNDAN9/GAI3l8+wTmvTYdv64fcH8aoJK76Y+1FCZ08RtI5zDerM/FYT5DMkAc+19E4aLmd5KqdFyg==
"@typescript-eslint/typescript-estree@5.62.0":
version "5.62.0"
@ -2669,30 +2679,30 @@
semver "^7.3.7"
tsutils "^3.21.0"
"@typescript-eslint/typescript-estree@6.6.0":
version "6.6.0"
resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-6.6.0.tgz#373c420d2e12c28220f4a83352280a04823a91b7"
integrity sha512-hMcTQ6Al8MP2E6JKBAaSxSVw5bDhdmbCEhGW/V8QXkb9oNsFkA4SBuOMYVPxD3jbtQ4R/vSODBsr76R6fP3tbA==
"@typescript-eslint/typescript-estree@6.7.2":
version "6.7.2"
resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-6.7.2.tgz#ce5883c23b581a5caf878af641e49dd0349238c7"
integrity sha512-kiJKVMLkoSciGyFU0TOY0fRxnp9qq1AzVOHNeN1+B9erKFCJ4Z8WdjAkKQPP+b1pWStGFqezMLltxO+308dJTQ==
dependencies:
"@typescript-eslint/types" "6.6.0"
"@typescript-eslint/visitor-keys" "6.6.0"
"@typescript-eslint/types" "6.7.2"
"@typescript-eslint/visitor-keys" "6.7.2"
debug "^4.3.4"
globby "^11.1.0"
is-glob "^4.0.3"
semver "^7.5.4"
ts-api-utils "^1.0.1"
"@typescript-eslint/utils@6.6.0":
version "6.6.0"
resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-6.6.0.tgz#2d686c0f0786da6362d909e27a9de1c13ba2e7dc"
integrity sha512-mPHFoNa2bPIWWglWYdR0QfY9GN0CfvvXX1Sv6DlSTive3jlMTUy+an67//Gysc+0Me9pjitrq0LJp0nGtLgftw==
"@typescript-eslint/utils@6.7.2":
version "6.7.2"
resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-6.7.2.tgz#b9ef0da6f04932167a9222cb4ac59cb187165ebf"
integrity sha512-ZCcBJug/TS6fXRTsoTkgnsvyWSiXwMNiPzBUani7hDidBdj1779qwM1FIAmpH4lvlOZNF3EScsxxuGifjpLSWQ==
dependencies:
"@eslint-community/eslint-utils" "^4.4.0"
"@types/json-schema" "^7.0.12"
"@types/semver" "^7.5.0"
"@typescript-eslint/scope-manager" "6.6.0"
"@typescript-eslint/types" "6.6.0"
"@typescript-eslint/typescript-estree" "6.6.0"
"@typescript-eslint/scope-manager" "6.7.2"
"@typescript-eslint/types" "6.7.2"
"@typescript-eslint/typescript-estree" "6.7.2"
semver "^7.5.4"
"@typescript-eslint/visitor-keys@5.62.0":
@ -2703,12 +2713,12 @@
"@typescript-eslint/types" "5.62.0"
eslint-visitor-keys "^3.3.0"
"@typescript-eslint/visitor-keys@6.6.0":
version "6.6.0"
resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-6.6.0.tgz#1109088b4346c8b2446f3845db526374d9a3bafc"
integrity sha512-L61uJT26cMOfFQ+lMZKoJNbAEckLe539VhTxiGHrWl5XSKQgA0RTBZJW2HFPy5T0ZvPVSD93QsrTKDkfNwJGyQ==
"@typescript-eslint/visitor-keys@6.7.2":
version "6.7.2"
resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-6.7.2.tgz#4cb2bd786f1f459731b0ad1584c9f73e1c7a4d5c"
integrity sha512-uVw9VIMFBUTz8rIeaUT3fFe8xIUx8r4ywAdlQv1ifH+6acn/XF8Y6rwJ7XNmkNMDrTW+7+vxFFPIF40nJCVsMQ==
dependencies:
"@typescript-eslint/types" "6.6.0"
"@typescript-eslint/types" "6.7.2"
eslint-visitor-keys "^3.4.1"
"@webassemblyjs/ast@1.9.0":
@ -5417,9 +5427,9 @@ eslint-plugin-import@~2.28.0:
tsconfig-paths "^3.14.2"
eslint-plugin-jsdoc@^46.1.0:
version "46.6.0"
resolved "https://registry.yarnpkg.com/eslint-plugin-jsdoc/-/eslint-plugin-jsdoc-46.6.0.tgz#513e41911a3081a15fe6a94490002e8d8f7b71d9"
integrity sha512-T/1gzsvnX45qABzyPEonEhFDttkTn7Igm/X89TXIkTLBOsNl2GYtyBqQPZGXZZ8J5VBzEhiCMvI2P2kXX4dnFw==
version "46.8.1"
resolved "https://registry.yarnpkg.com/eslint-plugin-jsdoc/-/eslint-plugin-jsdoc-46.8.1.tgz#cfc649c15d460903fe8e86eda582023bb97f283a"
integrity sha512-uTce7IBluPKXIQMWJkIwFsI1gv7sZRmLjctca2K5DIxPi8fSBj9f4iru42XmGwuiMyH2f3nfc60sFmnSGv4Z/A==
dependencies:
"@es-joy/jsdoccomment" "~0.40.1"
are-docs-informative "^0.0.2"