Merge branch 'feat/drive-file-usage-hints' into 'develop'

feat: Add usageHint field to DriveFile

Co-authored-by: yumeko <yumeko@mainichi.social>

See merge request firefish/firefish!10750

docs/downgrade.sql

@@ -1,6 +1,7 @@
 BEGIN;
 
 DELETE FROM "migrations" WHERE name IN (
+    'AddDriveFileUsage1713451569342',
     'ConvertCwVarcharToText1713225866247',
     'FixChatFileConstraint1712855579316',
     'DropTimeZone1712425488543',
@@ -23,7 +24,11 @@ DELETE FROM "migrations" WHERE name IN (
     'RemoveNativeUtilsMigration1705877093218'
 );
 
---convert-cw-varchar-to-text
+-- AddDriveFileUsage
+ALTER TABLE "drive_file" DROP COLUMN "usageHint";
+DROP TYPE "drive_file_usage_hint_enum";
+
+-- convert-cw-varchar-to-text
 DROP INDEX "IDX_8e3bbbeb3df04d1a8105da4c8f";
 ALTER TABLE "note" ALTER COLUMN "cw" TYPE character varying(512);
 CREATE INDEX "IDX_8e3bbbeb3df04d1a8105da4c8f" ON "note" USING "pgroonga" ("cw" pgroonga_varchar_full_text_search_ops_v2);

packages/backend-rs/index.d.ts

@@ -348,6 +348,7 @@ export interface DriveFile {
   webpublicType: string | null
   requestHeaders: Json | null
   requestIp: string | null
+  usageHint: DriveFileUsageHintEnum | null
 }
 export interface DriveFolder {
   id: string
@@ -780,6 +781,10 @@ export enum AntennaSrcEnum {
   List = 'list',
   Users = 'users'
 }
+export enum DriveFileUsageHintEnum {
+  UserAvatar = 'userAvatar',
+  UserBanner = 'userBanner'
+}
 export enum MutedNoteReasonEnum {
   Manual = 'manual',
   Other = 'other',

packages/backend-rs/index.js

@@ -310,7 +310,7 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { readEnvironmentConfig, readServerConfig, stringToAcct, acctToString, checkWordMute, getFullApAccount, isSelfHost, isSameOrigin, extractHost, toPuny, isUnicodeEmoji, sqlLikeEscape, safeForSql, formatMilliseconds, getNoteSummary, toMastodonId, fromMastodonId, fetchMeta, metaToPugArgs, nyaify, hashPassword, verifyPassword, isOldPasswordAlgorithm, decodeReaction, countReactions, toDbReaction, AntennaSrcEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, initIdGenerator, getTimestamp, genId, secureRndstr } = nativeBinding
+const { readEnvironmentConfig, readServerConfig, stringToAcct, acctToString, checkWordMute, getFullApAccount, isSelfHost, isSameOrigin, extractHost, toPuny, isUnicodeEmoji, sqlLikeEscape, safeForSql, formatMilliseconds, getNoteSummary, toMastodonId, fromMastodonId, fetchMeta, metaToPugArgs, nyaify, hashPassword, verifyPassword, isOldPasswordAlgorithm, decodeReaction, countReactions, toDbReaction, AntennaSrcEnum, DriveFileUsageHintEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, initIdGenerator, getTimestamp, genId, secureRndstr } = nativeBinding
 
 module.exports.readEnvironmentConfig = readEnvironmentConfig
 module.exports.readServerConfig = readServerConfig
@@ -339,6 +339,7 @@ module.exports.decodeReaction = decodeReaction
 module.exports.countReactions = countReactions
 module.exports.toDbReaction = toDbReaction
 module.exports.AntennaSrcEnum = AntennaSrcEnum
+module.exports.DriveFileUsageHintEnum = DriveFileUsageHintEnum
 module.exports.MutedNoteReasonEnum = MutedNoteReasonEnum
 module.exports.NoteVisibilityEnum = NoteVisibilityEnum
 module.exports.NotificationTypeEnum = NotificationTypeEnum

packages/backend-rs/src/model/entity/drive_file.rs

@@ -1,5 +1,6 @@
 //! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.15
 
+use super::sea_orm_active_enums::DriveFileUsageHintEnum;
 use sea_orm::entity::prelude::*;
 
 #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
@@ -52,6 +53,8 @@ pub struct Model {
     pub request_headers: Option<Json>,
     #[sea_orm(column_name = "requestIp")]
     pub request_ip: Option<String>,
+    #[sea_orm(column_name = "usageHint")]
+    pub usage_hint: Option<DriveFileUsageHintEnum>,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

packages/backend-rs/src/model/entity/sea_orm_active_enums.rs

@@ -23,6 +23,20 @@ pub enum AntennaSrcEnum {
 #[derive(Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)]
 #[cfg_attr(not(feature = "napi"), derive(Clone))]
 #[cfg_attr(feature = "napi", napi_derive::napi(string_enum = "camelCase"))]
+#[sea_orm(
+    rs_type = "String",
+    db_type = "Enum",
+    enum_name = "drive_file_usage_hint_enum"
+)]
+pub enum DriveFileUsageHintEnum {
+    #[sea_orm(string_value = "userAvatar")]
+    UserAvatar,
+    #[sea_orm(string_value = "userBanner")]
+    UserBanner,
+}
+#[derive(Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)]
+#[cfg_attr(not(feature = "napi"), derive(Clone))]
+#[cfg_attr(feature = "napi", napi_derive::napi(string_enum = "camelCase"))]
 #[sea_orm(
     rs_type = "String",
     db_type = "Enum",

packages/backend/src/migration/1713451569342-AddDriveFileUsage.ts

@@ -0,0 +1,17 @@
+import type { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddDriveFileUsage1713451569342 implements MigrationInterface {
+	public async up(queryRunner: QueryRunner): Promise<void> {
+		await queryRunner.query(
+			`CREATE TYPE drive_file_usage_hint_enum AS ENUM ('userAvatar', 'userBanner')`,
+		);
+		await queryRunner.query(
+			`ALTER TABLE "drive_file" ADD "usageHint" drive_file_usage_hint_enum DEFAULT NULL`,
+		);
+	}
+
+	public async down(queryRunner: QueryRunner): Promise<void> {
+		await queryRunner.query(`ALTER TABLE "drive_file" DROP COLUMN "usageHint"`);
+		await queryRunner.query(`DROP TYPE drive_file_usage_hint_enum`);
+	}
+}

packages/backend/src/models/entities/drive-file.ts

@@ -16,6 +16,8 @@ import { DriveFolder } from "./drive-folder.js";
 import { DB_MAX_IMAGE_COMMENT_LENGTH } from "@/misc/hard-limits.js";
 import { NoteFile } from "./note-file.js";
 
+export type DriveFileUsageHint = "userAvatar" | "userBanner" | null;
+
 @Entity()
 @Index(["userId", "folderId", "id"])
 export class DriveFile {
@@ -177,6 +179,14 @@ export class DriveFile {
 	})
 	public isSensitive: boolean;
 
+	// Hint for what this file is used for
+	@Column({
+		type: "enum",
+		enum: ["userAvatar", "userBanner"],
+		nullable: true,
+	})
+	public usageHint: DriveFileUsageHint;
+
 	/**
 	 * 外部の(信頼されていない)URLへの直リンクか否か
 	 */

packages/backend/src/models/repositories/drive-file.ts

@@ -152,6 +152,7 @@ export const DriveFileRepository = db.getRepository(DriveFile).extend({
 			md5: file.md5,
 			size: file.size,
 			isSensitive: file.isSensitive,
+			usageHint: file.usageHint,
 			blurhash: file.blurhash,
 			properties: opts.self ? file.properties : this.getPublicProperties(file),
 			url: opts.self ? file.url : this.getPublicUrl(file, false),
@@ -193,6 +194,7 @@ export const DriveFileRepository = db.getRepository(DriveFile).extend({
 			md5: file.md5,
 			size: file.size,
 			isSensitive: file.isSensitive,
+			usageHint: file.usageHint,
 			blurhash: file.blurhash,
 			properties: opts.self ? file.properties : this.getPublicProperties(file),
 			url: opts.self ? file.url : this.getPublicUrl(file, false),

packages/backend/src/models/schema/drive-file.ts

@@ -44,6 +44,12 @@ export const packedDriveFileSchema = {
 			optional: false,
 			nullable: false,
 		},
+		usageHint: {
+			type: "string",
+			optional: false,
+			nullable: true,
+			enum: ["userAvatar", "userBanner"],
+		},
 		blurhash: {
 			type: "string",
 			optional: false,

packages/backend/src/remote/activitypub/models/image.ts

@@ -3,7 +3,10 @@ import type { CacheableRemoteUser } from "@/models/entities/user.js";
 import Resolver from "../resolver.js";
 import { fetchMeta } from "backend-rs";
 import { apLogger } from "../logger.js";
-import type { DriveFile } from "@/models/entities/drive-file.js";
+import type {
+	DriveFile,
+	DriveFileUsageHint,
+} from "@/models/entities/drive-file.js";
 import { DriveFiles } from "@/models/index.js";
 import { truncate } from "@/misc/truncate.js";
 import { DB_MAX_IMAGE_COMMENT_LENGTH } from "@/misc/hard-limits.js";
@@ -16,6 +19,7 @@ const logger = apLogger;
 export async function createImage(
 	actor: CacheableRemoteUser,
 	value: any,
+	usage: DriveFileUsageHint,
 ): Promise<DriveFile> {
 	// Skip if author is frozen.
 	if (actor.isSuspended) {
@@ -43,6 +47,7 @@ export async function createImage(
 		sensitive: image.sensitive,
 		isLink: !instance.cacheRemoteFiles,
 		comment: truncate(image.name, DB_MAX_IMAGE_COMMENT_LENGTH),
+		usageHint: usage,
 	});
 
 	if (file.isLink) {
@@ -73,9 +78,10 @@ export async function createImage(
 export async function resolveImage(
 	actor: CacheableRemoteUser,
 	value: any,
+	usage: DriveFileUsageHint,
 ): Promise<DriveFile> {
 	// TODO
 
 	// Fetch from remote server and register
-	return await createImage(actor, value);
+	return await createImage(actor, value, usage);
 }

packages/backend/src/remote/activitypub/models/note.ts

@@ -213,7 +213,8 @@ export async function createNote(
 		? (
 				await Promise.all(
 					note.attachment.map(
-						(x) => limit(() => resolveImage(actor, x)) as Promise<DriveFile>,
+						(x) =>
+							limit(() => resolveImage(actor, x, null)) as Promise<DriveFile>,
 					),
 				)
 			).filter((image) => image != null)
@@ -616,7 +617,7 @@ export async function updateNote(value: string | IObject, resolver?: Resolver) {
 			fileList.map(
 				(x) =>
 					limit(async () => {
-						const file = await resolveImage(actor, x);
+						const file = await resolveImage(actor, x, null);
 						const update: Partial<DriveFile> = {};
 
 						const altText = truncate(x.name, DB_MAX_IMAGE_COMMENT_LENGTH);

packages/backend/src/remote/activitypub/models/person.ts

@@ -10,6 +10,7 @@ import {
 	Followings,
 	UserProfiles,
 	UserPublickeys,
+	DriveFiles,
 } from "@/models/index.js";
 import type { IRemoteUser, CacheableUser } from "@/models/entities/user.js";
 import { User } from "@/models/entities/user.js";
@@ -362,10 +363,14 @@ export async function createPerson(
 
 	//#region Fetch avatar and header image
 	const [avatar, banner] = await Promise.all(
-		[person.icon, person.image].map((img) =>
+		[person.icon, person.image].map((img, index) =>
 			img == null
 				? Promise.resolve(null)
-				: resolveImage(user!, img).catch(() => null),
+				: resolveImage(
+						user,
+						img,
+						index === 0 ? "userAvatar" : index === 1 ? "userBanner" : null,
+					).catch(() => null),
 		),
 	);
 
@@ -438,10 +443,14 @@ export async function updatePerson(
 
 	// Fetch avatar and header image
 	const [avatar, banner] = await Promise.all(
-		[person.icon, person.image].map((img) =>
+		[person.icon, person.image].map((img, index) =>
 			img == null
 				? Promise.resolve(null)
-				: resolveImage(user, img).catch(() => null),
+				: resolveImage(
+						user,
+						img,
+						index === 0 ? "userAvatar" : index === 1 ? "userBanner" : null,
+					).catch(() => null),
 		),
 	);
 
@@ -561,10 +570,14 @@ export async function updatePerson(
 	} as Partial<User>;
 
 	if (avatar) {
+		if (user?.avatarId)
+			await DriveFiles.update(user.avatarId, { usageHint: null });
 		updates.avatarId = avatar.id;
 	}
 
 	if (banner) {
+		if (user?.bannerId)
+			await DriveFiles.update(user.bannerId, { usageHint: null });
 		updates.bannerId = banner.id;
 	}
 

packages/backend/src/server/api/endpoints/i/update.ts

@@ -13,6 +13,7 @@ import { normalizeForSearch } from "@/misc/normalize-for-search.js";
 import { verifyLink } from "@/services/fetch-rel-me.js";
 import { ApiError } from "@/server/api/error.js";
 import define from "@/server/api/define.js";
+import { DriveFile } from "@/models/entities/drive-file";
 
 export const meta = {
 	tags: ["account"],
@@ -241,8 +242,9 @@ export default define(meta, paramDef, async (ps, _user, token) => {
 	if (ps.emailNotificationTypes !== undefined)
 		profileUpdates.emailNotificationTypes = ps.emailNotificationTypes;
 
+	let avatar: DriveFile | null = null;
 	if (ps.avatarId) {
-		const avatar = await DriveFiles.findOneBy({ id: ps.avatarId });
+		avatar = await DriveFiles.findOneBy({ id: ps.avatarId });
 
 		if (avatar == null || avatar.userId !== user.id)
 			throw new ApiError(meta.errors.noSuchAvatar);
@@ -250,8 +252,9 @@ export default define(meta, paramDef, async (ps, _user, token) => {
 			throw new ApiError(meta.errors.avatarNotAnImage);
 	}
 
+	let banner: DriveFile | null = null;
 	if (ps.bannerId) {
-		const banner = await DriveFiles.findOneBy({ id: ps.bannerId });
+		banner = await DriveFiles.findOneBy({ id: ps.bannerId });
 
 		if (banner == null || banner.userId !== user.id)
 			throw new ApiError(meta.errors.noSuchBanner);
@@ -328,6 +331,20 @@ export default define(meta, paramDef, async (ps, _user, token) => {
 	updateUsertags(user, tags);
 	//#endregion
 
+	// Update old/new avatar usage hints
+	if (avatar) {
+		if (user.avatarId)
+			await DriveFiles.update(user.avatarId, { usageHint: null });
+		await DriveFiles.update(avatar.id, { usageHint: "userAvatar" });
+	}
+
+	// Update old/new banner usage hints
+	if (banner) {
+		if (user.bannerId)
+			await DriveFiles.update(user.bannerId, { usageHint: null });
+		await DriveFiles.update(banner.id, { usageHint: "userBanner" });
+	}
+
 	if (Object.keys(updates).length > 0) await Users.update(user.id, updates);
 	if (Object.keys(profileUpdates).length > 0)
 		await UserProfiles.update(user.id, profileUpdates);

packages/backend/src/services/drive/add-file.ts

@@ -16,6 +16,7 @@ import {
 	UserProfiles,
 } from "@/models/index.js";
 import { DriveFile } from "@/models/entities/drive-file.js";
+import type { DriveFileUsageHint } from "@/models/entities/drive-file.js";
 import type { IRemoteUser, User } from "@/models/entities/user.js";
 import { genId } from "backend-rs";
 import { isDuplicateKeyValueError } from "@/misc/is-duplicate-key-value-error.js";
@@ -65,6 +66,7 @@ function urlPathJoin(
  * @param type Content-Type for original
  * @param hash Hash for original
  * @param size Size for original
+ * @param usage Optional usage hint for file (f.e. "userAvatar")
  */
 async function save(
 	file: DriveFile,
@@ -73,6 +75,7 @@ async function save(
 	type: string,
 	hash: string,
 	size: number,
+	usage: DriveFileUsageHint = null,
 ): Promise<DriveFile> {
 	// thunbnail, webpublic を必要なら生成
 	const alts = await generateAlts(path, type, !file.uri);
@@ -161,6 +164,7 @@ async function save(
 		file.md5 = hash;
 		file.size = size;
 		file.storedInternal = false;
+		file.usageHint = usage ?? null;
 
 		return await DriveFiles.insert(file).then((x) =>
 			DriveFiles.findOneByOrFail(x.identifiers[0]),
@@ -204,6 +208,7 @@ async function save(
 		file.type = type;
 		file.md5 = hash;
 		file.size = size;
+		file.usageHint = usage ?? null;
 
 		return await DriveFiles.insert(file).then((x) =>
 			DriveFiles.findOneByOrFail(x.identifiers[0]),
@@ -450,6 +455,9 @@ type AddFileArgs = {
 
 	requestIp?: string | null;
 	requestHeaders?: Record<string, string> | null;
+
+	/** Whether this file has a known use case, like user avatar or instance icon */
+	usageHint?: DriveFileUsageHint;
 };
 
 /**
@@ -469,6 +477,7 @@ export async function addFile({
 	sensitive = null,
 	requestIp = null,
 	requestHeaders = null,
+	usageHint = null,
 }: AddFileArgs): Promise<DriveFile> {
 	const info = await getFileInfo(path);
 	logger.info(`${JSON.stringify(info)}`);
@@ -581,6 +590,7 @@ export async function addFile({
 	file.isLink = isLink;
 	file.requestIp = requestIp;
 	file.requestHeaders = requestHeaders;
+	file.usageHint = usageHint;
 	file.isSensitive = user
 		? Users.isLocalUser(user) &&
 			(instance!.markLocalFilesNsfwByDefault || profile!.alwaysMarkNsfw)
@@ -639,6 +649,7 @@ export async function addFile({
 			info.type.mime,
 			info.md5,
 			info.size,
+			usageHint,
 		);
 	}
 

packages/backend/src/services/drive/upload-from-url.ts

@@ -3,7 +3,10 @@ import type { User } from "@/models/entities/user.js";
 import { createTemp } from "@/misc/create-temp.js";
 import { downloadUrl, isPrivateIp } from "@/misc/download-url.js";
 import type { DriveFolder } from "@/models/entities/drive-folder.js";
-import type { DriveFile } from "@/models/entities/drive-file.js";
+import type {
+	DriveFile,
+	DriveFileUsageHint,
+} from "@/models/entities/drive-file.js";
 import { DriveFiles } from "@/models/index.js";
 import { driveLogger } from "./logger.js";
 import { addFile } from "./add-file.js";
@@ -13,7 +16,11 @@ const logger = driveLogger.createSubLogger("downloader");
 
 type Args = {
 	url: string;
-	user: { id: User["id"]; host: User["host"] } | null;
+	user: {
+		id: User["id"];
+		host: User["host"];
+		driveCapacityOverrideMb: User["driveCapacityOverrideMb"];
+	} | null;
 	folderId?: DriveFolder["id"] | null;
 	uri?: string | null;
 	sensitive?: boolean;
@@ -22,6 +29,7 @@ type Args = {
 	comment?: string | null;
 	requestIp?: string | null;
 	requestHeaders?: Record<string, string> | null;
+	usageHint?: DriveFileUsageHint;
 };
 
 export async function uploadFromUrl({
@@ -35,6 +43,7 @@ export async function uploadFromUrl({
 	comment = null,
 	requestIp = null,
 	requestHeaders = null,
+	usageHint = null,
 }: Args): Promise<DriveFile> {
 	const parsedUrl = new URL(url);
 	if (
@@ -75,9 +84,10 @@ export async function uploadFromUrl({
 			sensitive,
 			requestIp,
 			requestHeaders,
+			usageHint,
 		});
 		logger.succ(`Got: ${driveFile.id}`);
-		return driveFile!;
+		return driveFile;
 	} catch (e) {
 		logger.error(`Failed to create drive file:\n${inspect(e)}`);
 		throw e;