Merge branch 'fix/post_import_if' into 'develop'

Draft: fix: questionable if statements in note import

Co-authored-by: naskya <m@naskya.net>

See merge request firefish/firefish!10771

.gitignore

@@ -50,16 +50,12 @@ api-docs.json
 *.log
 *.code-workspace
 .DS_Store
-files/
+/files
 ormconfig.json
 packages/backend/assets/instance.css
 packages/backend/assets/sounds/None.mp3
 packages/backend/assets/LICENSE
 
-!/packages/backend/queue/processors/db
-!/packages/backend/src/db
-!/packages/backend/src/server/api/endpoints/drive/files
-
 packages/megalodon/lib
 packages/megalodon/.idea
 

.gitlab-ci.yml

@@ -3,8 +3,10 @@ image: docker.io/rust:slim-bookworm
 services:
   - name: docker.io/groonga/pgroonga:latest-alpine-12-slim
     alias: postgres
+    pull_policy: if-not-present
   - name: docker.io/redis:7-alpine
     alias: redis
+    pull_policy: if-not-present
 
 workflow:
   rules:
@@ -12,6 +14,11 @@ workflow:
       when: always
     - if: $CI_MERGE_REQUEST_PROJECT_PATH == 'firefish/firefish'
       when: always
+    - if: $CI_PROJECT_PATH != 'firefish/firefish'
+      changes:
+        paths:
+          - .gitlab-ci.yml
+      when: never
     - when: never
 
 cache:
@@ -79,10 +86,7 @@ client_build_test:
           - packages/client/*
           - packages/firefish-js/*
           - packages/sw/*
-          - scripts/**/*
           - locales/**/*
-          - package.json
-          - pnpm-lock.yaml
     - if: $CI_PIPELINE_SOURCE == 'push' || $CI_PIPELINE_SOURCE == 'merge_request_event'
       changes:
         paths:
@@ -93,9 +97,18 @@ client_build_test:
           - Cargo.toml
           - Cargo.lock
       when: never
+  services: []
+  before_script:
+    - apt-get update && apt-get -y upgrade
+    - apt-get -y --no-install-recommends install curl
+    - curl -fsSL 'https://deb.nodesource.com/setup_18.x' | bash -
+    - apt-get install -y --no-install-recommends build-essential python3 perl nodejs
+    - corepack enable
+    - corepack prepare pnpm@latest --activate
+    - cp .config/ci.yml .config/default.yml
   script:
     - pnpm install --frozen-lockfile
-    - pnpm --filter 'client' --filter 'sw' run build:debug
+    - pnpm --filter 'firefish-js' --filter 'client' --filter 'sw' run build:debug
 
 container_image_build:
   stage: build
@@ -119,8 +132,21 @@ container_image_build:
     - apt-get install -y --no-install-recommends buildah ca-certificates fuse-overlayfs
     - buildah login --username "${CI_REGISTRY_USER}" --password "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
     - export IMAGE_TAG="${CI_REGISTRY}/${CI_PROJECT_PATH}/develop:not-for-production"
+    - export IMAGE_CACHE="${CI_REGISTRY}/${CI_PROJECT_PATH}/develop/cache"
   script:
-    - buildah build --isolation chroot --device /dev/fuse:rw --security-opt seccomp=unconfined --security-opt apparmor=unconfined --cap-add all --tag "${IMAGE_TAG}" --platform linux/amd64 .
+    - |-
+      buildah build \
+        --isolation chroot \
+        --device /dev/fuse:rw \
+        --security-opt seccomp=unconfined \
+        --security-opt apparmor=unconfined \
+        --cap-add all \
+        --platform linux/amd64 \
+        --layers \
+        --cache-to "${IMAGE_CACHE}" \
+        --cache-from "${IMAGE_CACHE}" \
+        --tag "${IMAGE_TAG}" \
+        .
     - buildah inspect "${IMAGE_TAG}"
     - buildah push "${IMAGE_TAG}"
 
@@ -157,8 +183,12 @@ cargo_clippy:
           - Cargo.lock
     - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'main'
       when: never
-  script:
+  services: []
+  before_script:
+    - apt-get install -y --no-install-recommends build-essential clang mold perl
+    - cp ci/cargo/config.toml /usr/local/cargo/config.toml
     - rustup component add clippy
+  script:
     - cargo clippy -- -D warnings
 
 renovate:

packages/backend-rs/index.d.ts

@@ -1292,7 +1292,6 @@ export interface AbuseUserReportLike {
   comment: string
 }
 export function publishToModerationStream(moderatorId: string, report: AbuseUserReportLike): void
-export function publishToNotesStream(note: Note): void
 export function getTimestamp(id: string): number
 /**
  * The generated ID results in the form of `[8 chars timestamp] + [cuid2]`.

packages/backend-rs/index.js

@@ -310,7 +310,7 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { SECOND, MINUTE, HOUR, DAY, USER_ONLINE_THRESHOLD, USER_ACTIVE_THRESHOLD, FILE_TYPE_BROWSERSAFE, loadEnv, loadConfig, stringToAcct, acctToString, addNoteToAntenna, isBlockedServer, isSilencedServer, isAllowedServer, checkWordMute, getFullApAccount, isSelfHost, isSameOrigin, extractHost, toPuny, isUnicodeEmoji, sqlLikeEscape, safeForSql, formatMilliseconds, getImageSizeFromUrl, getNoteSummary, isSafeUrl, latestVersion, toMastodonId, fromMastodonId, fetchMeta, metaToPugArgs, nyaify, hashPassword, verifyPassword, isOldPasswordAlgorithm, decodeReaction, countReactions, toDbReaction, removeOldAttestationChallenges, AntennaSrcEnum, DriveFileUsageHintEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, initializeRustLogger, fetchNodeinfo, nodeinfo_2_1, nodeinfo_2_0, Protocol, Inbound, Outbound, watchNote, unwatchNote, publishToChannelStream, ChatEvent, publishToChatStream, ChatIndexEvent, publishToChatIndexStream, publishToBroadcastStream, publishToGroupChatStream, publishToModerationStream, publishToNotesStream, getTimestamp, genId, genIdAt, secureRndstr } = nativeBinding
+const { SECOND, MINUTE, HOUR, DAY, USER_ONLINE_THRESHOLD, USER_ACTIVE_THRESHOLD, FILE_TYPE_BROWSERSAFE, loadEnv, loadConfig, stringToAcct, acctToString, addNoteToAntenna, isBlockedServer, isSilencedServer, isAllowedServer, checkWordMute, getFullApAccount, isSelfHost, isSameOrigin, extractHost, toPuny, isUnicodeEmoji, sqlLikeEscape, safeForSql, formatMilliseconds, getImageSizeFromUrl, getNoteSummary, isSafeUrl, latestVersion, toMastodonId, fromMastodonId, fetchMeta, metaToPugArgs, nyaify, hashPassword, verifyPassword, isOldPasswordAlgorithm, decodeReaction, countReactions, toDbReaction, removeOldAttestationChallenges, AntennaSrcEnum, DriveFileUsageHintEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, initializeRustLogger, fetchNodeinfo, nodeinfo_2_1, nodeinfo_2_0, Protocol, Inbound, Outbound, watchNote, unwatchNote, publishToChannelStream, ChatEvent, publishToChatStream, ChatIndexEvent, publishToChatIndexStream, publishToBroadcastStream, publishToGroupChatStream, publishToModerationStream, getTimestamp, genId, genIdAt, secureRndstr } = nativeBinding
 
 module.exports.SECOND = SECOND
 module.exports.MINUTE = MINUTE
@@ -381,7 +381,6 @@ module.exports.publishToChatIndexStream = publishToChatIndexStream
 module.exports.publishToBroadcastStream = publishToBroadcastStream
 module.exports.publishToGroupChatStream = publishToGroupChatStream
 module.exports.publishToModerationStream = publishToModerationStream
-module.exports.publishToNotesStream = publishToNotesStream
 module.exports.getTimestamp = getTimestamp
 module.exports.genId = genId
 module.exports.genIdAt = genIdAt

packages/backend-rs/src/service/stream.rs

@@ -5,7 +5,6 @@ pub mod chat_index;
 pub mod custom_emoji;
 pub mod group_chat;
 pub mod moderation;
-pub mod new_note;
 
 use crate::config::CONFIG;
 use crate::database::redis_conn;
@@ -26,7 +25,7 @@ pub enum Stream {
     #[strum(to_string = "noteStream:{note_id}")]
     Note { note_id: String },
     #[strum(serialize = "notesStream")]
-    NewNote,
+    Notes,
     #[strum(to_string = "userListStream:{list_id}")]
     UserList { list_id: String },
     #[strum(to_string = "mainStream:{user_id}")]

packages/backend-rs/src/service/stream/new_note.rs

@@ -1,10 +0,0 @@
-use crate::model::entity::note;
-use crate::service::stream::{publish_to_stream, Error, Stream};
-
-// for napi export (https://github.com/napi-rs/napi-rs/issues/2060)
-type Note = note::Model;
-
-#[crate::export(js_name = "publishToNotesStream")]
-pub fn publish(note: &Note) -> Result<(), Error> {
-    publish_to_stream(&Stream::NewNote, None, Some(serde_json::to_string(note)?))
-}

packages/backend/src/queue/processors/db/import-firefish-post.ts

@@ -59,18 +59,27 @@ export async function importCkPost(
 		userId: user.id,
 	});
 
-	// FIXME: What is this condition?
-	if (note != null && (note.fileIds?.length || 0) < files.length) {
+	// If an import is completely successful at once, the order should not be out of order.
+	// If it takes multiple imports to complete, the order is not guaranteed to be consistent.
+	if (note != null && files.length > 0) {
+		const addFiles: DriveFile[] = [];
+		for (const file of files) {
+			if (!note.fileIds.includes(file.id)) {
+				addFiles.push(file);
+			}
+		}
+
 		const update: Partial<Note> = {};
-		update.fileIds = files.map((x) => x.id);
+		update.fileIds = addFiles.map((x) => x.id);
 
 		if (update.fileIds != null) {
-			await NoteFiles.delete({ noteId: note.id });
 			await NoteFiles.insert(
 				update.fileIds.map((fileId) => ({ noteId: note?.id, fileId })),
 			);
 		}
 
+		update.fileIds = note.fileIds.concat(update.fileIds);
+
 		await Notes.update(note.id, update);
 		await NoteEdits.insert({
 			id: genId(),

packages/backend/src/queue/processors/db/import-masto-post.ts

@@ -85,18 +85,27 @@ export async function importMastoPost(
 		userId: user.id,
 	});
 
-	// FIXME: What is this condition?
-	if (note != null && (note.fileIds?.length || 0) < files.length) {
+	// If an import is completely successful at once, the order should not be out of order.
+	// If it takes multiple imports to complete, the order is not guaranteed to be consistent.
+	if (note != null && files.length > 0) {
+		const addFiles: DriveFile[] = [];
+		for (const file of files) {
+			if (!note.fileIds.includes(file.id)) {
+				addFiles.push(file);
+			}
+		}
+
 		const update: Partial<Note> = {};
-		update.fileIds = files.map((x) => x.id);
+		update.fileIds = addFiles.map((x) => x.id);
 
 		if (update.fileIds != null) {
-			await NoteFiles.delete({ noteId: note.id });
 			await NoteFiles.insert(
 				update.fileIds.map((fileId) => ({ noteId: note?.id, fileId })),
 			);
 		}
 
+		update.fileIds = note.fileIds.concat(update.fileIds);
+
 		await Notes.update(note.id, update);
 		await NoteEdits.insert({
 			id: genId(),

packages/backend/src/services/note/create.ts

@@ -1,5 +1,9 @@
 import * as mfm from "mfm-js";
-import { publishMainStream, publishNoteStream } from "@/services/stream.js";
+import {
+	publishMainStream,
+	publishNotesStream,
+	publishNoteStream,
+} from "@/services/stream.js";
 import DeliverManager from "@/remote/activitypub/deliver-manager.js";
 import renderNote from "@/remote/activitypub/renderer/note.js";
 import renderCreate from "@/remote/activitypub/renderer/create.js";
@@ -45,7 +49,6 @@ import {
 	genId,
 	genIdAt,
 	isSilencedServer,
-	publishToNotesStream,
 } from "backend-rs";
 import { countSameRenotes } from "@/misc/count-same-renotes.js";
 import { deliverToRelays, getCachedRelays } from "../relay.js";
@@ -508,7 +511,7 @@ export default async (
 								30,
 							);
 						}
-						publishToNotesStream(toRustObject(noteToPublish));
+						publishNotesStream(noteToPublish);
 					}
 				} finally {
 					await lock.release();

packages/backend/src/services/stream.ts

@@ -193,10 +193,9 @@ class Publisher {
 	// 	);
 	// };
 
-	/* ported to backend-rs */
-	// public publishNotesStream = (note: Note): void => {
-	// 	this.publish("notesStream", null, note);
-	// };
+	public publishNotesStream = (note: Note): void => {
+		this.publish("notesStream", null, note);
+	};
 
 	/* ported to backend-rs */
 	// public publishAdminStream = <K extends keyof AdminStreamTypes>(
@@ -222,7 +221,7 @@ export const publishUserEvent = publisher.publishUserEvent;
 export const publishMainStream = publisher.publishMainStream;
 export const publishDriveStream = publisher.publishDriveStream;
 export const publishNoteStream = publisher.publishNoteStream;
-// export const publishNotesStream = publisher.publishNotesStream;
+export const publishNotesStream = publisher.publishNotesStream;
 // export const publishChannelStream = publisher.publishChannelStream;
 export const publishUserListStream = publisher.publishUserListStream;
 // export const publishAntennaStream = publisher.publishAntennaStream;

packages/client/src/components/global/MkTime.vue

@@ -10,7 +10,7 @@
 </template>
 
 <script lang="ts" setup>
-import { computed, onMounted, onUnmounted, ref } from "vue";
+import { computed, onMounted, onUnmounted, ref, watch } from "vue";
 import { i18n } from "@/i18n";
 import { dateTimeFormat } from "@/scripts/intl-const";
 
@@ -25,7 +25,7 @@ const props = withDefaults(
 	},
 );
 
-const _time =
+const _time = computed(() =>
 	props.time == null
 		? Number.NaN
 		: typeof props.time === "number"
@@ -33,16 +33,19 @@ const _time =
 			: (props.time instanceof Date
 					? props.time
 					: new Date(props.time)
-				).getTime();
-const invalid = Number.isNaN(_time);
-const absolute = !invalid ? dateTimeFormat.format(_time) : i18n.ts._ago.invalid;
+				).getTime(),
+);
+const invalid = computed(() => Number.isNaN(_time.value));
+const absolute = computed(() =>
+	!invalid.value ? dateTimeFormat.format(_time.value) : i18n.ts._ago.invalid,
+);
 
 const now = ref(props.origin?.getTime() ?? Date.now());
 const relative = computed<string>(() => {
 	if (props.mode === "absolute") return ""; // absoluteではrelativeを使わないので計算しない
-	if (invalid) return i18n.ts._ago.invalid;
+	if (invalid.value) return i18n.ts._ago.invalid;
 
-	const ago = (now.value - _time) / 1000; /* ms */
+	const ago = (now.value - _time.value) / 1000; /* ms */
 	return ago >= 31536000
 		? i18n.t("_ago.yearsAgo", { n: Math.floor(ago / 31536000).toString() })
 		: ago >= 2592000
@@ -74,15 +77,25 @@ const relative = computed<string>(() => {
 									: i18n.ts._ago.future;
 });
 
-let tickId: number;
+let tickId: number | undefined;
 
 function tick() {
+	if (
+		invalid.value ||
+		props.origin ||
+		(props.mode !== "relative" && props.mode !== "detail")
+	) {
+		if (tickId) window.clearInterval(tickId);
+		tickId = undefined;
+		return;
+	}
+
 	const _now = Date.now();
-	const agoPrev = (now.value - _time) / 1000; /* ms */ // 現状のinterval
+	const agoPrev = (now.value - _time.value) / 1000; /* ms */ // 現状のinterval
 
 	now.value = _now;
 
-	const ago = (now.value - _time) / 1000; /* ms */ // 次のinterval
+	const ago = (now.value - _time.value) / 1000; /* ms */ // 次のinterval
 	const prev = agoPrev < 60 ? 10000 : agoPrev < 3600 ? 60000 : 180000;
 	const next = ago < 60 ? 10000 : ago < 3600 ? 60000 : 180000;
 
@@ -94,16 +107,13 @@ function tick() {
 	}
 }
 
-if (
-	!invalid &&
-	!props.origin &&
-	(props.mode === "relative" || props.mode === "detail")
-) {
-	onMounted(() => {
-		tick();
-	});
-	onUnmounted(() => {
-		if (tickId) window.clearInterval(tickId);
-	});
-}
+watch(() => props.time, tick);
+
+onMounted(() => {
+	tick();
+});
+
+onUnmounted(() => {
+	if (tickId) window.clearInterval(tickId);
+});
 </script>