From 3917818c76bdd6de717e40a8dde4d0b5792b429f Mon Sep 17 00:00:00 2001 From: Lhcfl Date: Sun, 21 Apr 2024 11:05:19 +0800 Subject: [PATCH 1/3] fix: #10451 Emoji pack imports randomly stop processing files --- .../processors/db/import-custom-emojis.ts | 37 +++++++++++++++++-- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/queue/processors/db/import-custom-emojis.ts b/packages/backend/src/queue/processors/db/import-custom-emojis.ts index f2371429f2..176928ba9e 100644 --- a/packages/backend/src/queue/processors/db/import-custom-emojis.ts +++ b/packages/backend/src/queue/processors/db/import-custom-emojis.ts @@ -11,14 +11,28 @@ import { addFile } from "@/services/drive/add-file.js"; import { genId } from "backend-rs"; import { db } from "@/db/postgre.js"; import probeImageSize from "probe-image-size"; -import * as path from "path"; +import * as path from "node:path"; const logger = queueLogger.createSubLogger("import-custom-emojis"); +// probeImageSize acceptable extensions +const acceptableExtensions = [ + ".avif", + ".bmp", + ".gif", + ".ico", + ".jpeg", + ".png", + ".psd", + // ".svg", // Disable for secure issues + ".tiff", + ".webp", +]; + // TODO: 名前衝突時の動作を選べるようにする export async function importCustomEmojis( job: Bull.Job, - done: any, + done: () => void, ): Promise { logger.info("Importing custom emojis ..."); @@ -62,6 +76,14 @@ export async function importCustomEmojis( if (!record.downloaded) continue; const emojiInfo = record.emoji; const emojiPath = `${outputPath}/${record.fileName}`; + + const extname = path.extname(record.fileName); + + // Skip non-support files + if (!acceptableExtensions.includes(extname)) { + continue; + } + await Emojis.delete({ name: emojiInfo.name, }); @@ -92,7 +114,7 @@ export async function importCustomEmojis( } else { logger.info("starting emoji import without metadata"); // Since we lack metadata, we import into a randomized category name instead - let categoryName = genId(); + const categoryName = genId(); let containedEmojis = fs.readdirSync(outputPath); @@ -103,7 +125,14 @@ export async function importCustomEmojis( for (const emojiFilename of containedEmojis) { // strip extension and get filename to use as name - const name = path.basename(emojiFilename, path.extname(emojiFilename)); + const extname = path.extname(emojiFilename); + + // Skip non-emoji files, such as LICENSE + if (!acceptableExtensions.includes(extname)) { + continue; + } + + const name = path.basename(emojiFilename, extname); const emojiPath = `${outputPath}/${emojiFilename}`; logger.info(`importing ${name}`); From 1dfc7c443badaa1f50bdb7075527a691b83de59a Mon Sep 17 00:00:00 2001 From: Lhcfl Date: Sun, 21 Apr 2024 11:14:19 +0800 Subject: [PATCH 2/3] fix: Extensions should be converted to lowercase --- .../processors/db/import-custom-emojis.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/backend/src/queue/processors/db/import-custom-emojis.ts b/packages/backend/src/queue/processors/db/import-custom-emojis.ts index 176928ba9e..e14f58dce7 100644 --- a/packages/backend/src/queue/processors/db/import-custom-emojis.ts +++ b/packages/backend/src/queue/processors/db/import-custom-emojis.ts @@ -16,17 +16,17 @@ import * as path from "node:path"; const logger = queueLogger.createSubLogger("import-custom-emojis"); // probeImageSize acceptable extensions +// JPG, GIF, PNG, WebP, BMP, TIFF, SVG, PSD. const acceptableExtensions = [ - ".avif", - ".bmp", - ".gif", - ".ico", ".jpeg", + ".jpg", + ".gif", ".png", - ".psd", - // ".svg", // Disable for secure issues - ".tiff", ".webp", + ".bmp", + ".tiff", + // ".svg", // Disable for secure issues + ".psd", ]; // TODO: 名前衝突時の動作を選べるようにする @@ -80,7 +80,7 @@ export async function importCustomEmojis( const extname = path.extname(record.fileName); // Skip non-support files - if (!acceptableExtensions.includes(extname)) { + if (!acceptableExtensions.includes(extname.toLowerCase())) { continue; } @@ -128,7 +128,7 @@ export async function importCustomEmojis( const extname = path.extname(emojiFilename); // Skip non-emoji files, such as LICENSE - if (!acceptableExtensions.includes(extname)) { + if (!acceptableExtensions.includes(extname.toLowerCase())) { continue; } From a52340fa531427644480494f228d38ed8dea1bd7 Mon Sep 17 00:00:00 2001 From: Linca Date: Mon, 22 Apr 2024 00:31:38 +0000 Subject: [PATCH 3/3] fix: disable extensions that cannot be used as emojis --- .../backend/src/queue/processors/db/import-custom-emojis.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/queue/processors/db/import-custom-emojis.ts b/packages/backend/src/queue/processors/db/import-custom-emojis.ts index e14f58dce7..33b68afd48 100644 --- a/packages/backend/src/queue/processors/db/import-custom-emojis.ts +++ b/packages/backend/src/queue/processors/db/import-custom-emojis.ts @@ -24,9 +24,9 @@ const acceptableExtensions = [ ".png", ".webp", ".bmp", - ".tiff", + // ".tiff", // Cannot be used as emoji // ".svg", // Disable for secure issues - ".psd", + // ".psd", // Cannot be used as emoji ]; // TODO: 名前衝突時の動作を選べるようにする