Compare commits

...

17 Commits

Author SHA1 Message Date
laozhoubuluo 0c8d201b5a Merge branch 'fix/post_import_if' into 'develop'
Draft: fix: questionable if statements in note import


See merge request firefish/firefish!10771
2024-05-07 01:31:43 +00:00
naskya 82c98ae72f
ci: modify buildah args 2024-05-07 07:26:33 +09:00
naskya 5b3f93457b
dev: add renovate 2024-05-07 06:58:00 +09:00
naskya 4d9c0f8e7b
ci: fix syntax 2024-05-07 06:11:31 +09:00
naskya bf2b624bc9
ci: build OCI container image on develop 2024-05-07 05:52:43 +09:00
naskya 5261eb24b6
ci: restrict project path 2024-05-07 05:26:05 +09:00
naskya d440e9b388
ci: revise tasks 2024-05-07 04:58:59 +09:00
naskya 14b285f882 Merge branch 'refactor/is-safe-url' into 'develop'
refactor (backend): port isValidUrl to backend-rs


See merge request firefish/firefish!10795
2024-05-06 17:11:51 +00:00
naskya baa5c402db
ci: apt-get update first & fix paths 2024-05-07 01:54:29 +09:00
naskya 5b01d3574f
refactor (backend): port isValidUrl to backend-rs 2024-05-07 00:56:37 +09:00
naskya e3a98ebc72 Merge branch 'userLang' into 'develop'
Add server-side per-user UI language

Co-authored-by: eana <coder@apps.1a23.com>

See merge request firefish/firefish!10793
2024-05-06 15:31:18 +00:00
naskya 7fe7f90350
ci: revise build config 2024-05-07 00:22:51 +09:00
naskya 8ed942e00f
chore: update auto-generated files 2024-05-06 23:13:31 +09:00
naskya ddfdd038ad
chore: update downgrade.sql 2024-05-06 23:10:39 +09:00
naskya 7fdd44cf8d
locale: update translations 2024-05-06 23:07:57 +09:00
eana ef57735e6a fix typo 2024-05-06 05:26:38 +00:00
eana e7c33835b2 Add server-side per-user UI language 2024-05-06 05:14:44 +00:00
25 changed files with 215 additions and 47 deletions

View File

@ -1,4 +1,4 @@
image: docker.io/node:18-alpine
image: docker.io/rust:slim-bookworm
services:
- name: docker.io/groonga/pgroonga:latest-alpine-12-slim
@ -8,40 +8,113 @@ services:
workflow:
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
- if: $CI_PROJECT_PATH == 'firefish/firefish'
when: always
- if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'main'
when: never
- if: $CI_MERGE_REQUEST_PROJECT_PATH == 'firefish/firefish'
when: always
- when: never
cache:
paths:
- node_modules/
- target/
- node_modules
# - /usr/local/cargo/registry/index
# - /usr/local/cargo/registry/cache
- target/debug/deps
- target/debug/build
stages:
- test
- build
- dependency
variables:
POSTGRES_DB: firefish_db
POSTGRES_USER: firefish
POSTGRES_PASSWORD: password
POSTGRES_HOST_AUTH_METHOD: trust
POSTGRES_DB: 'firefish_db'
POSTGRES_USER: 'firefish'
POSTGRES_PASSWORD: 'password'
POSTGRES_HOST_AUTH_METHOD: 'trust'
DEBIAN_FRONTEND: 'noninteractive'
CARGO_PROFILE_DEV_OPT_LEVEL: '0'
CARGO_PROFILE_DEV_LTO: 'off'
CARGO_PROFILE_DEV_DEBUG: 'none'
default:
before_script:
- apk add --update build-base linux-headers curl ca-certificates python3 perl postgresql-client
- curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
- . "${HOME}/.cargo/env"
- apt-get update && apt-get -y upgrade
- apt-get -y --no-install-recommends install curl
- curl -fsSL 'https://deb.nodesource.com/setup_18.x' | bash -
- apt-get install -y --no-install-recommends build-essential clang mold python3 perl nodejs postgresql-client
- corepack enable
- corepack prepare pnpm@latest --activate
- cp .config/ci.yml .config/default.yml
- cp ci/cargo/config.toml /usr/local/cargo/config.toml
- export PGPASSWORD="${POSTGRES_PASSWORD}"
- psql --host postgres --user "${POSTGRES_USER}" --dbname "${POSTGRES_DB}" --command 'CREATE EXTENSION pgroonga'
build_and_cargo_unit_test:
build_test:
stage: test
script:
- pnpm install --frozen-lockfile
- pnpm run build:debug
- pnpm run migrate
container_image_build:
stage: build
image: docker.io/debian:bookworm-slim
services: []
before_script: []
rules:
- if: $CI_COMMIT_BRANCH == 'develop'
script:
- apt-get update && apt-get -y upgrade
- apt-get install -y --no-install-recommends buildah ca-certificates
- buildah login --username "${CI_REGISTRY_USER}" --password "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
- buildah build --security-opt seccomp=unconfined --cap-add all --tag "${CI_REGISTRY}/${CI_PROJECT_PATH}/develop:not-for-production" --platform linux/amd64 .
- buildah push "${CI_REGISTRY}/${CI_PROJECT_PATH}/develop:not-for-production" "docker://${CI_REGISTRY}/${CI_PROJECT_PATH}/develop:not-for-production"
cargo_unit_test:
stage: test
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
paths:
- packages/backend-rs/**/*
- packages/macro-rs/**/*
- Cargo.toml
- Cargo.lock
- if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'main'
when: never
script:
- cargo check --features napi
- pnpm install --frozen-lockfile
- mkdir packages/backend-rs/built
- cp packages/backend-rs/index.js packages/backend-rs/built/index.js
- cp packages/backend-rs/index.d.ts packages/backend-rs/built/index.d.ts
- pnpm --filter='!backend-rs' run build:debug
- cargo test
cargo_clippy:
stage: test
rules:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
changes:
paths:
- packages/backend-rs/**/*
- packages/macro-rs/**/*
- Cargo.toml
- Cargo.lock
- if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'main'
when: never
script:
- cargo clippy -- -D warnings
renovate:
stage: dependency
image:
name: docker.io/renovate/renovate:37-slim
entrypoint: [""]
rules:
- if: $RENOVATE && $CI_PIPELINE_SOURCE == 'schedule'
services: []
before_script: []
script:
- renovate --platform gitlab --token "${API_TOKEN}" --endpoint "${CI_SERVER_URL}/api/v4" "${CI_PROJECT_PATH}"

3
ci/cargo/config.toml Normal file
View File

@ -0,0 +1,3 @@
[target.x86_64-unknown-linux-gnu]
linker = "/usr/bin/clang"
rustflags = ["-C", "link-arg=--ld-path=/usr/bin/mold"]

View File

@ -2,6 +2,8 @@
Breaking changes are indicated by the :warning: icon.
- Adding `lang` to the response of `i` and the request parameter of `i/update`.
## v20240504
- :warning: Removed `release` endpoint.

View File

@ -1,6 +1,7 @@
BEGIN;
DELETE FROM "migrations" WHERE name IN (
'AddUserProfileLanguage1714888400293',
'DropUnusedIndexes1714643926317',
'AlterAkaType1714099399879',
'AddDriveFileUsage1713451569342',
@ -764,9 +765,6 @@ CREATE SEQUENCE public.__chart_day__users_id_seq
CACHE 1;
ALTER SEQUENCE public.__chart_day__users_id_seq OWNED BY public.__chart_day__users.id;
-- drop-user-profile-language
ALTER TABLE "user_profile" ADD COLUMN "lang" character varying(32);
-- emoji-moderator
ALTER TABLE "user" DROP COLUMN "emojiModPerm";
DROP TYPE "public"."user_emojimodperm_enum";

View File

@ -766,6 +766,9 @@ confirmToUnclipAlreadyClippedNote: "This post is already part of the \"{name}\"
public: "Public"
i18nInfo: "Firefish is being translated into various languages by volunteers. You
can help at {link}."
i18nServerInfo: "New clients will be in {language} by default."
i18nServerChange: "Use {language} instead."
i18nServerSet: "Use {language} for new clients."
manageAccessTokens: "Manage access tokens"
accountInfo: "Account Info"
notesCount: "Number of posts"

View File

@ -685,6 +685,9 @@ unclip: "クリップ解除"
confirmToUnclipAlreadyClippedNote: "この投稿はすでにクリップ「{name}」に含まれています。投稿をこのクリップから除外しますか?"
public: "公開"
i18nInfo: "Firefishは有志によって様々な言語に翻訳されています。{link}で翻訳に協力できます。"
i18nServerInfo: "新しい端末では{language}が既定の言語になります。"
i18nServerChange: "{language}に変更する。"
i18nServerSet: "新しい端末での表示言語を{language}にします。"
manageAccessTokens: "アクセストークンの管理"
accountInfo: "アカウント情報"
notesCount: "投稿の数"

View File

@ -667,6 +667,9 @@ unclip: "移除便签"
confirmToUnclipAlreadyClippedNote: "本帖已包含在便签 \"{name}\" 里。您想要将本帖从该便签中移除吗?"
public: "公开"
i18nInfo: "Firefish 已经被志愿者们翻译成了各种语言。如果您也有兴趣,可以通过 {link} 帮助翻译。"
i18nServerInfo: "新客户端将默认使用 {language}。"
i18nServerChange: "改为 {language}。"
i18nServerSet: "设定新客户端使用 {language}。"
manageAccessTokens: "管理访问令牌"
accountInfo: "账号信息"
notesCount: "帖子数量"

View File

@ -661,6 +661,9 @@ unclip: "解除摘錄"
confirmToUnclipAlreadyClippedNote: "此貼文已包含在摘錄「{name}」中。 你想將貼文從這個摘錄中排除嗎?"
public: "公開"
i18nInfo: "Firefish已經被志願者們翻譯成各種語言版本如果想要幫忙的話可以進入{link}幫助翻譯。"
i18nServerInfo: "新客戶端將默認使用 {language}。"
i18nServerChange: "改為 {language}。"
i18nServerSet: "設定新客戶端使用 {language}。"
manageAccessTokens: "管理存取權杖"
accountInfo: "帳戶資訊"
notesCount: "貼文數量"

View File

@ -268,6 +268,7 @@ export interface NoteLikeForGetNoteSummary {
hasPoll: boolean
}
export function getNoteSummary(note: NoteLikeForGetNoteSummary): string
export function isSafeUrl(url: string): boolean
export function latestVersion(): Promise<string>
export function toMastodonId(firefishId: string): string | null
export function fromMastodonId(mastodonId: string): string | null
@ -1129,6 +1130,7 @@ export interface UserProfile {
preventAiLearning: boolean
isIndexable: boolean
mutedPatterns: Array<string>
lang: string | null
}
export interface UserPublickey {
userId: string

View File

@ -310,7 +310,7 @@ if (!nativeBinding) {
throw new Error(`Failed to load native binding`)
}
const { SECOND, MINUTE, HOUR, DAY, USER_ONLINE_THRESHOLD, USER_ACTIVE_THRESHOLD, FILE_TYPE_BROWSERSAFE, loadEnv, loadConfig, stringToAcct, acctToString, addNoteToAntenna, isBlockedServer, isSilencedServer, isAllowedServer, checkWordMute, getFullApAccount, isSelfHost, isSameOrigin, extractHost, toPuny, isUnicodeEmoji, sqlLikeEscape, safeForSql, formatMilliseconds, getImageSizeFromUrl, getNoteSummary, latestVersion, toMastodonId, fromMastodonId, fetchMeta, metaToPugArgs, nyaify, hashPassword, verifyPassword, isOldPasswordAlgorithm, decodeReaction, countReactions, toDbReaction, removeOldAttestationChallenges, AntennaSrcEnum, DriveFileUsageHintEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, initializeRustLogger, fetchNodeinfo, nodeinfo_2_1, nodeinfo_2_0, Protocol, Inbound, Outbound, watchNote, unwatchNote, publishToChannelStream, ChatEvent, publishToChatStream, ChatIndexEvent, publishToChatIndexStream, publishToBroadcastStream, publishToGroupChatStream, publishToModerationStream, getTimestamp, genId, genIdAt, secureRndstr } = nativeBinding
const { SECOND, MINUTE, HOUR, DAY, USER_ONLINE_THRESHOLD, USER_ACTIVE_THRESHOLD, FILE_TYPE_BROWSERSAFE, loadEnv, loadConfig, stringToAcct, acctToString, addNoteToAntenna, isBlockedServer, isSilencedServer, isAllowedServer, checkWordMute, getFullApAccount, isSelfHost, isSameOrigin, extractHost, toPuny, isUnicodeEmoji, sqlLikeEscape, safeForSql, formatMilliseconds, getImageSizeFromUrl, getNoteSummary, isSafeUrl, latestVersion, toMastodonId, fromMastodonId, fetchMeta, metaToPugArgs, nyaify, hashPassword, verifyPassword, isOldPasswordAlgorithm, decodeReaction, countReactions, toDbReaction, removeOldAttestationChallenges, AntennaSrcEnum, DriveFileUsageHintEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, initializeRustLogger, fetchNodeinfo, nodeinfo_2_1, nodeinfo_2_0, Protocol, Inbound, Outbound, watchNote, unwatchNote, publishToChannelStream, ChatEvent, publishToChatStream, ChatIndexEvent, publishToChatIndexStream, publishToBroadcastStream, publishToGroupChatStream, publishToModerationStream, getTimestamp, genId, genIdAt, secureRndstr } = nativeBinding
module.exports.SECOND = SECOND
module.exports.MINUTE = MINUTE
@ -339,6 +339,7 @@ module.exports.safeForSql = safeForSql
module.exports.formatMilliseconds = formatMilliseconds
module.exports.getImageSizeFromUrl = getImageSizeFromUrl
module.exports.getNoteSummary = getNoteSummary
module.exports.isSafeUrl = isSafeUrl
module.exports.latestVersion = latestVersion
module.exports.toMastodonId = toMastodonId
module.exports.fromMastodonId = fromMastodonId

View File

@ -0,0 +1,34 @@
#[crate::export]
pub fn is_safe_url(url: &str) -> bool {
if let Ok(url) = url.parse::<url::Url>() {
if url.host_str().unwrap_or_default() == "unix"
|| !["http", "https"].contains(&url.scheme())
|| ![None, Some(80), Some(443)].contains(&url.port())
{
return false;
}
true
} else {
false
}
}
#[cfg(test)]
mod unit_test {
use super::is_safe_url;
#[test]
fn safe_url() {
assert!(is_safe_url("http://firefish.dev/firefish/firefish"));
assert!(is_safe_url("https://firefish.dev/firefish/firefish"));
assert!(is_safe_url("http://firefish.dev:80/firefish/firefish"));
assert!(is_safe_url("https://firefish.dev:80/firefish/firefish"));
assert!(is_safe_url("http://firefish.dev:443/firefish/firefish"));
assert!(is_safe_url("https://firefish.dev:443/firefish/firefish"));
assert!(!is_safe_url("https://unix/firefish/firefish"));
assert!(!is_safe_url("https://firefish.dev:35/firefish/firefish"));
assert!(!is_safe_url("ftp://firefish.dev/firefish/firefish"));
assert!(!is_safe_url("nyaa"));
assert!(!is_safe_url(""));
}
}

View File

@ -8,6 +8,7 @@ pub mod escape_sql;
pub mod format_milliseconds;
pub mod get_image_size;
pub mod get_note_summary;
pub mod is_safe_url;
pub mod latest_version;
pub mod mastodon_id;
pub mod meta;

View File

@ -78,6 +78,7 @@ pub struct Model {
pub is_indexable: bool,
#[sea_orm(column_name = "mutedPatterns")]
pub muted_patterns: Vec<String>,
pub lang: Option<String>,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

View File

@ -0,0 +1,13 @@
import type { MigrationInterface, QueryRunner } from "typeorm";
export class AddUserProfileLanguage1714888400293 implements MigrationInterface {
async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "user_profile" ADD COLUMN "lang" character varying(32)`,
);
}
async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "user_profile" DROP COLUMN "lang"`);
}
}

View File

@ -7,10 +7,10 @@ import chalk from "chalk";
import Logger from "@/services/logger.js";
import IPCIDR from "ip-cidr";
import PrivateIp from "private-ip";
import { isValidUrl } from "./is-valid-url.js";
import { isSafeUrl } from "backend-rs";
export async function downloadUrl(url: string, path: string): Promise<void> {
if (!isValidUrl(url)) {
if (!isSafeUrl(url)) {
throw new StatusError("Invalid URL", 400);
}
@ -43,8 +43,8 @@ export async function downloadUrl(url: string, path: string): Promise<void> {
limit: 0,
},
})
.on("redirect", (res: Got.Response, opts: Got.NormalizedOptions) => {
if (!isValidUrl(opts.url)) {
.on("redirect", (_res: Got.Response, opts: Got.NormalizedOptions) => {
if (!isSafeUrl(opts.url)) {
downloadLogger.warn(`Invalid URL: ${opts.url}`);
req.destroy();
}

View File

@ -5,7 +5,7 @@ import CacheableLookup from "cacheable-lookup";
import fetch, { type RequestRedirect } from "node-fetch";
import { HttpProxyAgent, HttpsProxyAgent } from "hpagent";
import { config } from "@/config.js";
import { isValidUrl } from "./is-valid-url.js";
import { isSafeUrl } from "backend-rs";
export async function getJson(
url: string,
@ -60,7 +60,7 @@ export async function getResponse(args: {
size?: number;
redirect?: RequestRedirect;
}) {
if (!isValidUrl(args.url)) {
if (!isSafeUrl(args.url)) {
throw new StatusError("Invalid URL", 400);
}
@ -83,7 +83,7 @@ export async function getResponse(args: {
});
if (args.redirect === "manual" && [301, 302, 307, 308].includes(res.status)) {
if (!isValidUrl(res.url)) {
if (!isSafeUrl(res.url)) {
throw new StatusError("Invalid URL", 400);
}
return res;

View File

@ -1,20 +0,0 @@
export function isValidUrl(url: string | URL | undefined): boolean {
if (process.env.NODE_ENV !== "production") return true;
try {
if (url == null) return false;
const u = typeof url === "string" ? new URL(url) : url;
if (!u.protocol.match(/^https?:$/) || u.hostname === "unix") {
return false;
}
if (u.port !== "" && !["80", "443"].includes(u.port)) {
return false;
}
return true;
} catch {
return false;
}
}

View File

@ -50,6 +50,12 @@ export class UserProfile {
verified?: boolean;
}[];
@Column("varchar", {
length: 32,
nullable: true,
})
public lang: string | null;
@Column("varchar", {
length: 512,
nullable: true,

View File

@ -512,6 +512,7 @@ export const UserRepository = db.getRepository(User).extend({
description: profile!.description,
location: profile!.location,
birthday: profile!.birthday,
lang: profile!.lang,
fields: profile!.fields,
followersCount: followersCount ?? null,
followingCount: followingCount ?? null,

View File

@ -204,6 +204,12 @@ export const packedUserDetailedNotMeOnlySchema = {
optional: false,
example: "2018-03-12",
},
lang: {
type: "string",
nullable: true,
optional: false,
example: "ja-JP",
},
fields: {
type: "array",
nullable: false,

View File

@ -5,8 +5,8 @@ import { StatusError, getResponse } from "@/misc/fetch.js";
import { createSignedPost, createSignedGet } from "./ap-request.js";
import type { Response } from "node-fetch";
import type { IObject } from "./type.js";
import { isValidUrl } from "@/misc/is-valid-url.js";
import { apLogger } from "@/remote/activitypub/logger.js";
import { isSafeUrl } from "backend-rs";
export default async (user: { id: User["id"] }, url: string, object: any) => {
const body = JSON.stringify(object);
@ -44,7 +44,7 @@ export async function apGet(
user?: ILocalUser,
redirects: boolean = true,
): Promise<{ finalUrl: string; content: IObject }> {
if (!isValidUrl(url)) {
if (!isSafeUrl(url)) {
throw new StatusError("Invalid URL", 400);
}

View File

@ -87,6 +87,7 @@ export const paramDef = {
description: { ...Users.descriptionSchema, nullable: true },
location: { ...Users.locationSchema, nullable: true },
birthday: { ...Users.birthdaySchema, nullable: true },
lang: { type: "string", nullable: true },
avatarId: { type: "string", format: "misskey:id", nullable: true },
bannerId: { type: "string", format: "misskey:id", nullable: true },
fields: {
@ -154,6 +155,7 @@ export default define(meta, paramDef, async (ps, _user, token) => {
if (ps.name !== undefined) updates.name = ps.name;
if (ps.description !== undefined) profileUpdates.description = ps.description;
if (typeof ps.lang === "string") profileUpdates.lang = ps.lang;
if (ps.location !== undefined) profileUpdates.location = ps.location;
if (ps.birthday !== undefined) profileUpdates.birthday = ps.birthday;
if (ps.ffVisibility !== undefined)

View File

@ -132,6 +132,9 @@ export async function signIn(token: Account["token"], redirect?: string) {
if (_DEV_) console.log("logging as token ", token);
const newAccount = await fetchAccount(token);
localStorage.setItem("account", JSON.stringify(newAccount));
if (newAccount.lang) {
localStorage.setItem("lang", newAccount.lang);
}
document.cookie = `token=${token}; path=/; max-age=31536000`; // bull dashboardの認証とかで使う
await addAccount(newAccount.id, token);

View File

@ -14,6 +14,12 @@
>
</template>
</I18n>
<I18n :src="i18n.ts.i18nServerInfo" v-if="serverLang" tag="span">
<template #language><strong>{{ langs.find(a => a[0] === serverLang)?.[1] ?? serverLang }}</strong></template>
</I18n>
<button class="_textButton" @click="updateServerLang" v-if="lang && lang !== serverLang">
{{i18n.t(serverLang ? "i18nServerChange" : "i18nServerSet", { language: langs.find(a => a[0] === lang)?.[1] ?? lang })}}
</button>
</template>
</FormSelect>
@ -404,6 +410,7 @@ import { deviceKind } from "@/scripts/device-kind";
import icon from "@/scripts/icon";
const lang = ref(localStorage.getItem("lang"));
const serverLang = ref(me?.lang);
const translateLang = ref(localStorage.getItem("translateLang"));
const fontSize = ref(localStorage.getItem("fontSize"));
const useSystemFont = ref(localStorage.getItem("useSystemFont") !== "f");
@ -559,6 +566,14 @@ const foldNotification = computed(
// });
// }
function updateServerLang() {
os.api("i/update", {
lang: lang.value,
}).then((i) => {
serverLang.value = i.lang;
});
}
watch(swipeOnDesktop, () => {
defaultStore.set("swipeOnMobile", true);
});

15
renovate.json Normal file
View File

@ -0,0 +1,15 @@
{
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
"extends": ["config:base"],
"rangeStrategy": "bump",
"branchConcurrentLimit": 5,
"enabledManagers": ["npm", "cargo"],
"baseBranches": ["develop"],
"lockFileMaintenance": {
"enabled": true,
"recreateWhen": "always",
"rebaseStalePrs": true,
"branchTopic": "lock-file-maintenance",
"commitMessageAction": "Lock file maintenance"
}
}