Merge branch 'fix/pagination' into 'develop'

Draft: fix(backend): requested limit to be fulfilled if possible

Closes #10867

See merge request firefish/firefish!10696

locales/ca-ES.yml

@@ -2301,3 +2301,6 @@ getQrCode: Mostrar el codi QR
 copyRemoteFollowUrl: Còpia la adreça URL del seguidor remot
 foldNotification: Agrupar les notificacions similars
 slashQuote: Cita encadenada
+i18nServerInfo: Els nous clients els trobares en {language} per defecte.
+i18nServerChange: Fes servir {language} en comptes.
+i18nServerSet: Fes servir {language} per els nous clients.

packages/backend/src/models/repositories/note-favorite.ts

@@ -2,6 +2,7 @@ import { db } from "@/db/postgre.js";
 import { NoteFavorite } from "@/models/entities/note-favorite.js";
 import { Notes } from "../index.js";
 import type { User } from "@/models/entities/user.js";
+import Logger from "@/services/logger.js";
 
 export const NoteFavoriteRepository = db.getRepository(NoteFavorite).extend({
 	async pack(
@@ -23,9 +24,16 @@ export const NoteFavoriteRepository = db.getRepository(NoteFavorite).extend({
 	packMany(favorites: any[], me: { id: User["id"] }) {
 		return Promise.allSettled(favorites.map((x) => this.pack(x, me))).then(
 			(promises) =>
-				promises.flatMap((result) =>
-					result.status === "fulfilled" ? [result.value] : [],
-				),
+				promises.flatMap((result, i) => {
+					if (result.status === "fulfilled") {
+						return [result.value];
+					}
+					const logger = new Logger("models-note-favorite");
+					logger.error(
+						`dropping note favorite due to violating visibility restrictions, note favorite ${favorites[i].id} user ${me.id}`,
+					);
+					return [];
+				}),
 		);
 	},
 });

packages/backend/src/models/repositories/note-reaction.ts

@@ -4,6 +4,7 @@ import { Notes, Users } from "../index.js";
 import type { Packed } from "@/misc/schema.js";
 import { decodeReaction } from "backend-rs";
 import type { User } from "@/models/entities/user.js";
+import Logger from "@/services/logger.js";
 
 export const NoteReactionRepository = db.getRepository(NoteReaction).extend({
 	async pack(
@@ -49,8 +50,15 @@ export const NoteReactionRepository = db.getRepository(NoteReaction).extend({
 		);
 
 		// filter out rejected promises, only keep fulfilled values
-		return reactions.flatMap((result) =>
-			result.status === "fulfilled" ? [result.value] : [],
-		);
+		return reactions.flatMap((result, i) => {
+			if (result.status === "fulfilled") {
+				return [result.value];
+			}
+			const logger = new Logger("models-note-reaction");
+			logger.error(
+				`dropping note reaction due to violating visibility restrictions, reason is ${result.reason}`,
+			);
+			return [];
+		});
 	},
 });

packages/backend/src/models/repositories/note.ts

@@ -23,6 +23,7 @@ import {
 } from "@/misc/populate-emojis.js";
 import { db } from "@/db/postgre.js";
 import { IdentifiableError } from "@/misc/identifiable-error.js";
+import Logger from "@/services/logger.js";
 
 export async function populatePoll(note: Note, meId: User["id"] | null) {
 	const poll = await Polls.findOneByOrFail({ noteId: note.id });
@@ -343,8 +344,15 @@ export const NoteRepository = db.getRepository(Note).extend({
 		);
 
 		// filter out rejected promises, only keep fulfilled values
-		return promises.flatMap((result) =>
-			result.status === "fulfilled" ? [result.value] : [],
-		);
+		return promises.flatMap((result, i) => {
+			if (result.status === "fulfilled") {
+				return [result.value];
+			}
+			const logger = new Logger("models-note");
+			logger.error(
+				`dropping note due to violating visibility restrictions, note ${notes[i].id} user ${meId}`,
+			);
+			return [];
+		});
 	},
 });

packages/backend/src/server/activitypub/outbox.ts

@@ -73,7 +73,21 @@ export default async (ctx: Router.RouterContext) => {
 			)
 			.andWhere("note.localOnly = FALSE");
 
-		const notes = await query.take(limit).getMany();
+		// We fetch more than requested because some may be filtered out, and if there's less than
+		// requested, this is not normal behavior of any API.
+		const notes = [];
+		const take = Math.floor(limit * 1.5);
+		let skip = 0;
+		while (notes.length < limit) {
+			const notes_query = await query.take(take).skip(skip).getMany();
+			notes.push(...(await Notes.packMany(notes_query)));
+			skip += take;
+			if (notes_query.length < take) break;
+		}
+
+		if (notes.length > limit) {
+			notes.length = limit;
+		}
 
 		if (sinceId) notes.reverse();
 

packages/backend/src/server/api/endpoints/antennas/notes.ts

@@ -117,11 +117,25 @@ export default define(meta, paramDef, async (ps, user) => {
 	generateMutedUserQuery(query, user);
 	generateBlockedUserQuery(query, user);
 
-	const notes = await query.take(limit).getMany();
-
-	if (notes.length > 0) {
-		readNote(user.id, notes);
+	// We fetch more than requested because some may be filtered out, and if there's less than
+	// requested, the pagination stops.
+	const found = [];
+	const take = Math.floor(ps.limit * 1.5);
+	let skip = 0;
+	while (found.length < ps.limit) {
+		const notes = await query.take(take).skip(skip).getMany();
+		found.push(...(await Notes.packMany(notes)));
+		skip += take;
+		if (notes.length < take) break;
 	}
 
-	return await Notes.packMany(notes, user);
+	if (found.length > ps.limit) {
+		found.length = ps.limit;
+	}
+
+	if (found.length > 0) {
+		readNote(user.id, found);
+	}
+
+	return found;
 });

packages/backend/src/server/api/endpoints/channels/timeline.ts

@@ -76,9 +76,23 @@ export default define(meta, paramDef, async (ps, user) => {
 		.leftJoinAndSelect("note.channel", "channel");
 	//#endregion
 
-	const timeline = await query.take(ps.limit).getMany();
+	// We fetch more than requested because some may be filtered out, and if there's less than
+	// requested, the pagination stops.
+	const found = [];
+	const take = Math.floor(ps.limit * 1.5);
+	let skip = 0;
+	while (found.length < ps.limit) {
+		const timeline = await query.take(take).skip(skip).getMany();
+		found.push(...(await Notes.packMany(timeline, user)));
+		skip += take;
+		if (timeline.length < take) break;
+	}
+
+	if (found.length > ps.limit) {
+		found.length = ps.limit;
+	}
 
 	if (user) activeUsersChart.read(user);
 
-	return await Notes.packMany(timeline, user);
+	return found;
 });

packages/backend/src/server/api/endpoints/clips/notes.ts

@@ -88,7 +88,21 @@ export default define(meta, paramDef, async (ps, user) => {
 		generateBlockedUserQuery(query, user);
 	}
 
-	const notes = await query.take(ps.limit).getMany();
+	// We fetch more than requested because some may be filtered out, and if there's less than
+	// requested, the pagination stops.
+	const found = [];
+	const take = Math.floor(ps.limit * 1.5);
+	let skip = 0;
+	while (found.length < ps.limit) {
+		const notes = await query.take(take).skip(skip).getMany();
+		found.push(...(await Notes.packMany(notes, user)));
+		skip += take;
+		if (notes.length < take) break;
+	}
 
-	return await Notes.packMany(notes, user);
+	if (found.length > ps.limit) {
+		found.length = ps.limit;
+	}
+
+	return found;
 });

packages/backend/src/server/api/endpoints/notes.ts

@@ -85,7 +85,21 @@ export default define(meta, paramDef, async (ps) => {
 	//	query.isBot = bot;
 	//}
 
-	const notes = await query.take(ps.limit).getMany();
+	// We fetch more than requested because some may be filtered out, and if there's less than
+	// requested, the pagination stops.
+	const found = [];
+	const take = Math.floor(ps.limit * 1.5);
+	let skip = 0;
+	while (found.length < ps.limit) {
+		const notes = await query.take(take).skip(skip).getMany();
+		found.push(...(await Notes.packMany(notes)));
+		skip += take;
+		if (notes.length < take) break;
+	}
 
-	return await Notes.packMany(notes);
+	if (found.length > ps.limit) {
+		found.length = ps.limit;
+	}
+
+	return found;
 });

packages/backend/src/server/api/endpoints/notes/children.ts

@@ -57,7 +57,25 @@ export default define(meta, paramDef, async (ps, user) => {
 		generateBlockedUserQuery(query, user);
 	}
 
-	const notes = await query.getMany();
+	// We fetch more than requested because some may be filtered out, and if there's less than
+	// requested, the pagination stops.
+	const found = [];
+	const take = Math.floor(ps.limit * 1.5);
+	let skip = 0;
+	while (found.length < ps.limit) {
+		const notes = await query.take(take).skip(skip).getMany();
+		found.push(
+			...(await Notes.packMany(notes, user, {
+				detail: false,
+			})),
+		);
+		skip += take;
+		if (notes.length < take) break;
+	}
 
-	return await Notes.packMany(notes, user, { detail: false });
+	if (found.length > ps.limit) {
+		found.length = ps.limit;
+	}
+
+	return found;
 });

packages/backend/src/server/api/endpoints/users/notes.ts

@@ -138,7 +138,21 @@ export default define(meta, paramDef, async (ps, me) => {
 
 	//#endregion
 
-	const timeline = await query.take(ps.limit).getMany();
+	// We fetch more than requested because some may be filtered out, and if there's less than
+	// requested, the pagination stops.
+	const found = [];
+	const take = Math.floor(ps.limit * 1.5);
+	let skip = 0;
+	while (found.length < ps.limit) {
+		const timeline = await query.take(take).skip(skip).getMany();
+		found.push(...(await Notes.packMany(timeline, user)));
+		skip += take;
+		if (timeline.length < take) break;
+	}
 
-	return await Notes.packMany(timeline, me);
+	if (found.length > ps.limit) {
+		found.length = ps.limit;
+	}
+
+	return found;
 });

packages/backend/src/server/api/endpoints/users/reactions.ts

@@ -70,7 +70,23 @@ export default define(meta, paramDef, async (ps, me) => {
 
 	generateVisibilityQuery(query, me);
 
-	const reactions = await query.take(ps.limit).getMany();
+	// We fetch more than requested because some may be filtered out, and if there's less than
+	// requested, the pagination stops.
+	const found = [];
+	const take = Math.floor(ps.limit * 1.5);
+	let skip = 0;
+	while (found.length < ps.limit) {
+		const reactions = await query.take(take).skip(skip).getMany();
+		found.push(
+			...(await NoteReactions.packMany(reactions, me, { withNote: true })),
+		);
+		skip += take;
+		if (reactions.length < take) break;
+	}
 
-	return await NoteReactions.packMany(reactions, me, { withNote: true });
+	if (found.length > ps.limit) {
+		found.length = ps.limit;
+	}
+
+	return found;
 });

packages/client/src/components/global/MkTime.vue

@@ -25,15 +25,21 @@ const props = withDefaults(
 	},
 );
 
+function getDateSafe(n: Date | string | number) {
+	try {
+		if (n instanceof Date) {
+			return n;
+		}
+		return new Date(n);
+	} catch (err) {
+		return {
+			getTime: () => Number.NaN,
+		};
+	}
+}
+
 const _time = computed(() =>
-	props.time == null
-		? Number.NaN
-		: typeof props.time === "number"
-			? props.time
-			: (props.time instanceof Date
-					? props.time
-					: new Date(props.time)
-				).getTime(),
+	props.time == null ? Number.NaN : getDateSafe(props.time).getTime(),
 );
 const invalid = computed(() => Number.isNaN(_time.value));
 const absolute = computed(() =>
@@ -41,45 +47,57 @@ const absolute = computed(() =>
 );
 
 const now = ref(props.origin?.getTime() ?? Date.now());
+
 const relative = computed<string>(() => {
 	if (props.mode === "absolute") return ""; // absoluteではrelativeを使わないので計算しない
 	if (invalid.value) return i18n.ts._ago.invalid;
 
 	const ago = (now.value - _time.value) / 1000; /* ms */
-	return ago >= 31536000
-		? i18n.t("_ago.yearsAgo", { n: Math.floor(ago / 31536000).toString() })
-		: ago >= 2592000
-			? i18n.t("_ago.monthsAgo", {
-					n: Math.floor(ago / 2592000).toString(),
-				})
-			: ago >= 604800
-				? i18n.t("_ago.weeksAgo", {
-						n: Math.floor(ago / 604800).toString(),
-					})
-				: ago >= 86400
-					? i18n.t("_ago.daysAgo", {
-							n: Math.floor(ago / 86400).toString(),
-						})
-					: ago >= 3600
-						? i18n.t("_ago.hoursAgo", {
-								n: Math.floor(ago / 3600).toString(),
-							})
-						: ago >= 60
-							? i18n.t("_ago.minutesAgo", {
-									n: (~~(ago / 60)).toString(),
-								})
-							: ago >= 10
-								? i18n.t("_ago.secondsAgo", {
-										n: (~~(ago % 60)).toString(),
-									})
-								: ago >= -1
-									? i18n.ts._ago.justNow
-									: i18n.ts._ago.future;
+
+	if (ago >= 31536000) {
+		return i18n.t("_ago.yearsAgo", {
+			n: Math.floor(ago / 31536000).toString(),
+		});
+	}
+	if (ago >= 2592000) {
+		return i18n.t("_ago.monthsAgo", {
+			n: Math.floor(ago / 2592000).toString(),
+		});
+	}
+	if (ago >= 604800) {
+		return i18n.t("_ago.weeksAgo", {
+			n: Math.floor(ago / 604800).toString(),
+		});
+	}
+	if (ago >= 86400) {
+		return i18n.t("_ago.daysAgo", {
+			n: Math.floor(ago / 86400).toString(),
+		});
+	}
+	if (ago >= 3600) {
+		return i18n.t("_ago.hoursAgo", {
+			n: Math.floor(ago / 3600).toString(),
+		});
+	}
+	if (ago >= 60) {
+		return i18n.t("_ago.minutesAgo", {
+			n: (~~(ago / 60)).toString(),
+		});
+	}
+	if (ago >= 10) {
+		return i18n.t("_ago.secondsAgo", {
+			n: (~~(ago % 60)).toString(),
+		});
+	}
+	if (ago >= -1) {
+		return i18n.ts._ago.justNow;
+	}
+	return i18n.ts._ago.future;
 });
 
 let tickId: number | undefined;
 
-function tick() {
+function tick(forceUpdateTicker = false) {
 	if (
 		invalid.value ||
 		props.origin ||
@@ -101,13 +119,16 @@ function tick() {
 
 	if (!tickId) {
 		tickId = window.setInterval(tick, next);
-	} else if (prev < next) {
+	} else if (prev < next || forceUpdateTicker) {
 		window.clearInterval(tickId);
 		tickId = window.setInterval(tick, next);
 	}
 }
 
-watch(() => props.time, tick);
+watch(
+	() => props.time,
+	() => tick(true),
+);
 
 onMounted(() => {
 	tick();