From ef724a49fe52050aed5b12e201f33c901f25f6f8 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Sun, 2 Jul 2023 20:37:46 -0400 Subject: [PATCH 1/7] store cache values to redis --- packages/backend/package.json | 2 +- packages/backend/src/misc/cache.ts | 86 +++++++++++++------ packages/backend/src/misc/emoji-meta.ts | 34 +++++--- packages/backend/src/misc/populate-emojis.ts | 9 +- .../src/remote/activitypub/models/person.ts | 6 +- .../src/services/chart/charts/active-users.ts | 6 +- .../backend/src/services/instance-actor.ts | 6 +- .../register-or-fetch-instance-doc.ts | 12 +-- packages/backend/src/services/relay.ts | 2 +- packages/backend/src/services/user-cache.ts | 33 +++---- pnpm-lock.yaml | 23 +++-- 11 files changed, 131 insertions(+), 88 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index b584a5691..6f6344102 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -34,6 +34,7 @@ "@koa/cors": "3.4.3", "@koa/multer": "3.0.2", "@koa/router": "9.0.1", + "@msgpack/msgpack": "3.0.0-beta2", "@peertube/http-signature": "1.7.0", "@redocly/openapi-core": "1.0.0-beta.120", "@sinonjs/fake-timers": "9.1.2", @@ -43,7 +44,6 @@ "ajv": "8.12.0", "archiver": "5.3.1", "argon2": "^0.30.3", - "async-mutex": "^0.4.0", "autobind-decorator": "2.4.0", "autolinker": "4.0.0", "autwh": "0.1.0", diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index 9abebc91c..d790313d1 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -1,43 +1,75 @@ +import { redisClient } from "@/db/redis.js"; +import { nativeRandomStr } from "native-utils/built/index.js"; +import { encode, decode } from "@msgpack/msgpack"; +import { ChainableCommander } from "ioredis"; + export class Cache { - public cache: Map; - private lifetime: number; + private ttl: number; + private fingerprint: string; - constructor(lifetime: Cache["lifetime"]) { - this.cache = new Map(); - this.lifetime = lifetime; + constructor(ttl: number) { + this.ttl = ttl; + this.fingerprint = `cache:${nativeRandomStr(32)}`; } - public set(key: string | null, value: T): void { - this.cache.set(key, { - date: Date.now(), - value, - }); + private prefixedKey(key: string | null): string { + return key ? `${this.fingerprint}:${key}` : this.fingerprint; } - public get(key: string | null): T | undefined { - const cached = this.cache.get(key); - if (cached == null) return undefined; - if (Date.now() - cached.date > this.lifetime) { - this.cache.delete(key); - return undefined; + public async set(key: string | null, value: T, transaction?: ChainableCommander): Promise { + const _key = this.prefixedKey(key); + const _value = Buffer.from(encode(value)); + const commander = transaction ?? redisClient; + if (this.ttl === Infinity) { + await commander.set(_key, _value); + } else { + await commander.set(_key, _value, "PX", this.ttl); } - return cached.value; } - public delete(key: string | null) { - this.cache.delete(key); + public async get(key: string | null): Promise { + const _key = this.prefixedKey(key); + const cached = await redisClient.getBuffer(_key); + if (cached === null) return undefined; + + return decode(cached) as T; + } + + public async getAll(): Promise> { + const keys = await redisClient.keys(`${this.fingerprint}*`); + const map = new Map(); + if (keys.length === 0) { + return map; + } + const values = await redisClient.mgetBuffer(keys); + + for (const [i, key] of keys.entries()) { + const val = values[i]; + if (val !== null) { + map.set(key, decode(val) as T); + } + } + + return map; + } + + public async delete(...keys: (string | null)[]): Promise { + if (keys.length > 0) { + const _keys = keys.map(this.prefixedKey); + await redisClient.del(_keys); + } } /** - * キャッシュがあればそれを返し、無ければfetcherを呼び出して結果をキャッシュ&返します - * optional: キャッシュが存在してもvalidatorでfalseを返すとキャッシュ無効扱いにします + * Returns if cached value exists. Otherwise, calls fetcher and caches. + * Overwrites cached value if invalidated by the optional validator. */ public async fetch( key: string | null, fetcher: () => Promise, validator?: (cachedValue: T) => boolean, ): Promise { - const cachedValue = this.get(key); + const cachedValue = await this.get(key); if (cachedValue !== undefined) { if (validator) { if (validator(cachedValue)) { @@ -52,20 +84,20 @@ export class Cache { // Cache MISS const value = await fetcher(); - this.set(key, value); + await this.set(key, value); return value; } /** - * キャッシュがあればそれを返し、無ければfetcherを呼び出して結果をキャッシュ&返します - * optional: キャッシュが存在してもvalidatorでfalseを返すとキャッシュ無効扱いにします + * Returns if cached value exists. Otherwise, calls fetcher and caches if the fetcher returns a value. + * Overwrites cached value if invalidated by the optional validator. */ public async fetchMaybe( key: string | null, fetcher: () => Promise, validator?: (cachedValue: T) => boolean, ): Promise { - const cachedValue = this.get(key); + const cachedValue = await this.get(key); if (cachedValue !== undefined) { if (validator) { if (validator(cachedValue)) { @@ -81,7 +113,7 @@ export class Cache { // Cache MISS const value = await fetcher(); if (value !== undefined) { - this.set(key, value); + await this.set(key, value); } return value; } diff --git a/packages/backend/src/misc/emoji-meta.ts b/packages/backend/src/misc/emoji-meta.ts index fd9d9baa5..45364bdcb 100644 --- a/packages/backend/src/misc/emoji-meta.ts +++ b/packages/backend/src/misc/emoji-meta.ts @@ -1,9 +1,10 @@ import probeImageSize from "probe-image-size"; -import { Mutex, withTimeout } from "async-mutex"; +import { Mutex } from "redis-semaphore"; import { FILE_TYPE_BROWSERSAFE } from "@/const.js"; import Logger from "@/services/logger.js"; import { Cache } from "./cache.js"; +import { redisClient } from "@/db/redis.js"; export type Size = { width: number; @@ -11,23 +12,30 @@ export type Size = { }; const cache = new Cache(1000 * 60 * 10); // once every 10 minutes for the same url -const mutex = withTimeout(new Mutex(), 1000); +const logger = new Logger("emoji"); export async function getEmojiSize(url: string): Promise { - const logger = new Logger("emoji"); + let attempted = true; - await mutex.runExclusive(() => { - const attempted = cache.get(url); - if (!attempted) { - cache.set(url, true); - } else { - logger.warn(`Attempt limit exceeded: ${url}`); - throw new Error("Too many attempts"); - } - }); + const lock = new Mutex(redisClient, "getEmojiSize"); + await lock.acquire(); try { - logger.info(`Retrieving emoji size from ${url}`); + attempted = (await cache.get(url)) === true; + if (!attempted) { + await cache.set(url, true); + } + } finally { + await lock.release(); + } + + if (attempted) { + logger.warn(`Attempt limit exceeded: ${url}`); + throw new Error("Too many attempts"); + } + + try { + logger.debug(`Retrieving emoji size from ${url}`); const { width, height, mime } = await probeImageSize(url, { timeout: 5000, }); diff --git a/packages/backend/src/misc/populate-emojis.ts b/packages/backend/src/misc/populate-emojis.ts index 7aee4ec25..ce25dd559 100644 --- a/packages/backend/src/misc/populate-emojis.ts +++ b/packages/backend/src/misc/populate-emojis.ts @@ -7,6 +7,7 @@ import { isSelfHost, toPunyNullable } from "./convert-host.js"; import { decodeReaction } from "./reaction-lib.js"; import config from "@/config/index.js"; import { query } from "@/prelude/url.js"; +import { redisClient } from "@/db/redis.js"; const cache = new Cache(1000 * 60 * 60 * 12); @@ -75,7 +76,7 @@ export async function populateEmoji( if (emoji && !(emoji.width && emoji.height)) { emoji = await queryOrNull(); - cache.set(cacheKey, emoji); + await cache.set(cacheKey, emoji); } if (emoji == null) return null; @@ -150,7 +151,7 @@ export async function prefetchEmojis( emojis: { name: string; host: string | null }[], ): Promise { const notCachedEmojis = emojis.filter( - (emoji) => cache.get(`${emoji.name} ${emoji.host}`) == null, + async (emoji) => !(await cache.get(`${emoji.name} ${emoji.host}`)), ); const emojisQuery: any[] = []; const hosts = new Set(notCachedEmojis.map((e) => e.host)); @@ -169,7 +170,9 @@ export async function prefetchEmojis( select: ["name", "host", "originalUrl", "publicUrl"], }) : []; + const trans = redisClient.multi(); for (const emoji of _emojis) { - cache.set(`${emoji.name} ${emoji.host}`, emoji); + cache.set(`${emoji.name} ${emoji.host}`, emoji, trans); } + await trans.exec(); } diff --git a/packages/backend/src/remote/activitypub/models/person.ts b/packages/backend/src/remote/activitypub/models/person.ts index f8208e6d7..c541e9ae5 100644 --- a/packages/backend/src/remote/activitypub/models/person.ts +++ b/packages/backend/src/remote/activitypub/models/person.ts @@ -135,14 +135,14 @@ export async function fetchPerson( ): Promise { if (typeof uri !== "string") throw new Error("uri is not string"); - const cached = uriPersonCache.get(uri); + const cached = await uriPersonCache.get(uri); if (cached) return cached; // Fetch from the database if the URI points to this server if (uri.startsWith(`${config.url}/`)) { const id = uri.split("/").pop(); const u = await Users.findOneBy({ id }); - if (u) uriPersonCache.set(uri, u); + if (u) await uriPersonCache.set(uri, u); return u; } @@ -150,7 +150,7 @@ export async function fetchPerson( const exist = await Users.findOneBy({ uri }); if (exist) { - uriPersonCache.set(uri, exist); + await uriPersonCache.set(uri, exist); return exist; } //#endregion diff --git a/packages/backend/src/services/chart/charts/active-users.ts b/packages/backend/src/services/chart/charts/active-users.ts index 7a0c45cfa..15317e68b 100644 --- a/packages/backend/src/services/chart/charts/active-users.ts +++ b/packages/backend/src/services/chart/charts/active-users.ts @@ -25,12 +25,12 @@ export default class ActiveUsersChart extends Chart { return {}; } - public async read(user: { + public read(user: { id: User["id"]; host: null; createdAt: User["createdAt"]; - }): Promise { - await this.commit({ + }) { + this.commit({ read: [user.id], registeredWithinWeek: Date.now() - user.createdAt.getTime() < week ? [user.id] : [], diff --git a/packages/backend/src/services/instance-actor.ts b/packages/backend/src/services/instance-actor.ts index 50ce227eb..9240f3107 100644 --- a/packages/backend/src/services/instance-actor.ts +++ b/packages/backend/src/services/instance-actor.ts @@ -9,7 +9,7 @@ const ACTOR_USERNAME = "instance.actor" as const; const cache = new Cache(Infinity); export async function getInstanceActor(): Promise { - const cached = cache.get(null); + const cached = await cache.get(null); if (cached) return cached; const user = (await Users.findOneBy({ @@ -18,11 +18,11 @@ export async function getInstanceActor(): Promise { })) as ILocalUser | undefined; if (user) { - cache.set(null, user); + await cache.set(null, user); return user; } else { const created = (await createSystemUser(ACTOR_USERNAME)) as ILocalUser; - cache.set(null, created); + await cache.set(null, created); return created; } } diff --git a/packages/backend/src/services/register-or-fetch-instance-doc.ts b/packages/backend/src/services/register-or-fetch-instance-doc.ts index 4c3570e90..ddb9ce241 100644 --- a/packages/backend/src/services/register-or-fetch-instance-doc.ts +++ b/packages/backend/src/services/register-or-fetch-instance-doc.ts @@ -9,25 +9,25 @@ const cache = new Cache(1000 * 60 * 60); export async function registerOrFetchInstanceDoc( host: string, ): Promise { - host = toPuny(host); + const _host = toPuny(host); - const cached = cache.get(host); + const cached = await cache.get(_host); if (cached) return cached; - const index = await Instances.findOneBy({ host }); + const index = await Instances.findOneBy({ host: _host }); if (index == null) { const i = await Instances.insert({ id: genId(), - host, + host: _host, caughtAt: new Date(), lastCommunicatedAt: new Date(), }).then((x) => Instances.findOneByOrFail(x.identifiers[0])); - cache.set(host, i); + await cache.set(_host, i); return i; } else { - cache.set(host, index); + await cache.set(_host, index); return index; } } diff --git a/packages/backend/src/services/relay.ts b/packages/backend/src/services/relay.ts index bec4b1f86..2325f76c6 100644 --- a/packages/backend/src/services/relay.ts +++ b/packages/backend/src/services/relay.ts @@ -90,7 +90,7 @@ async function updateRelaysCache() { const relays = await Relays.findBy({ status: "accepted", }); - relaysCache.set(null, relays); + await relaysCache.set(null, relays); } export async function relayRejected(id: string) { diff --git a/packages/backend/src/services/user-cache.ts b/packages/backend/src/services/user-cache.ts index 949244855..373fb8686 100644 --- a/packages/backend/src/services/user-cache.ts +++ b/packages/backend/src/services/user-cache.ts @@ -6,7 +6,7 @@ import type { import { User } from "@/models/entities/user.js"; import { Users } from "@/models/index.js"; import { Cache } from "@/misc/cache.js"; -import { subscriber } from "@/db/redis.js"; +import { redisClient, subscriber } from "@/db/redis.js"; export const userByIdCache = new Cache(Infinity); export const localUserByNativeTokenCache = new Cache( @@ -22,13 +22,12 @@ subscriber.on("message", async (_, data) => { const { type, body } = obj.message; switch (type) { case "localUserUpdated": { - userByIdCache.delete(body.id); - localUserByIdCache.delete(body.id); - localUserByNativeTokenCache.cache.forEach((v, k) => { - if (v.value?.id === body.id) { - localUserByNativeTokenCache.delete(k); - } - }); + await userByIdCache.delete(body.id); + await localUserByIdCache.delete(body.id); + const toDelete = Array.from(await localUserByNativeTokenCache.getAll()) + .filter((v) => v[1]?.id === body.id) + .map((v) => v[0]); + await localUserByNativeTokenCache.delete(...toDelete); break; } case "userChangeSuspendedState": @@ -36,15 +35,17 @@ subscriber.on("message", async (_, data) => { case "userChangeModeratorState": case "remoteUserUpdated": { const user = await Users.findOneByOrFail({ id: body.id }); - userByIdCache.set(user.id, user); - for (const [k, v] of uriPersonCache.cache.entries()) { - if (v.value?.id === user.id) { - uriPersonCache.set(k, user); + await userByIdCache.set(user.id, user); + const trans = redisClient.multi(); + for (const [k, v] of (await uriPersonCache.getAll()).entries()) { + if (v?.id === user.id) { + await uriPersonCache.set(k, user, trans); } } + await trans.exec(); if (Users.isLocalUser(user)) { - localUserByNativeTokenCache.set(user.token, user); - localUserByIdCache.set(user.id, user); + await localUserByNativeTokenCache.set(user.token, user); + await localUserByIdCache.set(user.id, user); } break; } @@ -52,8 +53,8 @@ subscriber.on("message", async (_, data) => { const user = (await Users.findOneByOrFail({ id: body.id, })) as ILocalUser; - localUserByNativeTokenCache.delete(body.oldToken); - localUserByNativeTokenCache.set(body.newToken, user); + await localUserByNativeTokenCache.delete(body.oldToken); + await localUserByNativeTokenCache.set(body.newToken, user); break; } default: diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1002a6f95..560bb55a3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -105,6 +105,9 @@ importers: '@koa/router': specifier: 9.0.1 version: 9.0.1 + '@msgpack/msgpack': + specifier: 3.0.0-beta2 + version: 3.0.0-beta2 '@peertube/http-signature': specifier: 1.7.0 version: 1.7.0 @@ -132,9 +135,6 @@ importers: argon2: specifier: ^0.30.3 version: 0.30.3 - async-mutex: - specifier: ^0.4.0 - version: 0.4.0 autobind-decorator: specifier: 2.4.0 version: 2.4.0 @@ -786,7 +786,7 @@ importers: version: 2.30.0 emojilib: specifier: github:thatonecalculator/emojilib - version: github.com/thatonecalculator/emojilib/542fcc1a25003afad78f3248ceee8ac6980ddeb8 + version: github.com/thatonecalculator/emojilib/06944984a61ee799b7083894258f5fa318d932d1 escape-regexp: specifier: 0.0.1 version: 0.0.1 @@ -2277,6 +2277,11 @@ packages: os-filter-obj: 2.0.0 dev: true + /@msgpack/msgpack@3.0.0-beta2: + resolution: {integrity: sha512-y+l1PNV0XDyY8sM3YtuMLK5vE3/hkfId+Do8pLo/OPxfxuFAUwcGz3oiiUuV46/aBpwTzZ+mRWVMtlSKbradhw==} + engines: {node: '>= 14'} + dev: false + /@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.2: resolution: {integrity: sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==} cpu: [arm64] @@ -4496,12 +4501,6 @@ packages: stream-exhaust: 1.0.2 dev: true - /async-mutex@0.4.0: - resolution: {integrity: sha512-eJFZ1YhRR8UN8eBLoNzcDPcy/jqjsg6I1AP+KvWQX80BqOSW1oJPJXDylPUEeMr2ZQvHgnQ//Lp6f3RQ1zI7HA==} - dependencies: - tslib: 2.6.0 - dev: false - /async-settle@1.0.0: resolution: {integrity: sha512-VPXfB4Vk49z1LHHodrEQ6Xf7W4gg1w0dAPROHngx7qgDjqmIQ+fXmwgGXTW/ITLai0YLSvWepJOP9EVpMnEAcw==} engines: {node: '>= 0.10'} @@ -15772,8 +15771,8 @@ packages: url-polyfill: 1.1.12 dev: true - github.com/thatonecalculator/emojilib/542fcc1a25003afad78f3248ceee8ac6980ddeb8: - resolution: {tarball: https://codeload.github.com/thatonecalculator/emojilib/tar.gz/542fcc1a25003afad78f3248ceee8ac6980ddeb8} + github.com/thatonecalculator/emojilib/06944984a61ee799b7083894258f5fa318d932d1: + resolution: {tarball: https://codeload.github.com/thatonecalculator/emojilib/tar.gz/06944984a61ee799b7083894258f5fa318d932d1} name: emojilib version: 3.0.10 dev: true From 9d26e0803207a8b0c0024fc95a5b6ea2c45f17c1 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Sun, 2 Jul 2023 20:55:20 -0400 Subject: [PATCH 2/7] add cache prefix --- packages/backend/src/misc/cache.ts | 11 +++++------ packages/backend/src/misc/check-hit-antenna.ts | 2 +- packages/backend/src/misc/emoji-meta.ts | 2 +- packages/backend/src/misc/keypair-store.ts | 2 +- packages/backend/src/misc/populate-emojis.ts | 2 +- packages/backend/src/models/repositories/user.ts | 2 +- .../backend/src/remote/activitypub/db-resolver.ts | 4 ++-- packages/backend/src/server/api/authenticate.ts | 2 +- packages/backend/src/server/nodeinfo.ts | 2 +- packages/backend/src/services/instance-actor.ts | 2 +- packages/backend/src/services/note/create.ts | 2 +- .../src/services/register-or-fetch-instance-doc.ts | 2 +- packages/backend/src/services/relay.ts | 2 +- packages/backend/src/services/user-cache.ts | 8 ++++---- 14 files changed, 22 insertions(+), 23 deletions(-) diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index d790313d1..c2695aa78 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -1,19 +1,18 @@ import { redisClient } from "@/db/redis.js"; -import { nativeRandomStr } from "native-utils/built/index.js"; import { encode, decode } from "@msgpack/msgpack"; import { ChainableCommander } from "ioredis"; export class Cache { private ttl: number; - private fingerprint: string; + private prefix: string; - constructor(ttl: number) { + constructor(prefix: string, ttl: number) { this.ttl = ttl; - this.fingerprint = `cache:${nativeRandomStr(32)}`; + this.prefix = `cache:${prefix}`; } private prefixedKey(key: string | null): string { - return key ? `${this.fingerprint}:${key}` : this.fingerprint; + return key ? `${this.prefix}:${key}` : this.prefix; } public async set(key: string | null, value: T, transaction?: ChainableCommander): Promise { @@ -36,7 +35,7 @@ export class Cache { } public async getAll(): Promise> { - const keys = await redisClient.keys(`${this.fingerprint}*`); + const keys = await redisClient.keys(`${this.prefix}*`); const map = new Map(); if (keys.length === 0) { return map; diff --git a/packages/backend/src/misc/check-hit-antenna.ts b/packages/backend/src/misc/check-hit-antenna.ts index 358fba0f3..c422cca94 100644 --- a/packages/backend/src/misc/check-hit-antenna.ts +++ b/packages/backend/src/misc/check-hit-antenna.ts @@ -11,7 +11,7 @@ import * as Acct from "@/misc/acct.js"; import type { Packed } from "./schema.js"; import { Cache } from "./cache.js"; -const blockingCache = new Cache(1000 * 60 * 5); +const blockingCache = new Cache("blocking", 1000 * 60 * 5); // NOTE: フォローしているユーザーのノート、リストのユーザーのノート、グループのユーザーのノート指定はパフォーマンス上の理由で無効になっている diff --git a/packages/backend/src/misc/emoji-meta.ts b/packages/backend/src/misc/emoji-meta.ts index 45364bdcb..d2d15411f 100644 --- a/packages/backend/src/misc/emoji-meta.ts +++ b/packages/backend/src/misc/emoji-meta.ts @@ -11,7 +11,7 @@ export type Size = { height: number; }; -const cache = new Cache(1000 * 60 * 10); // once every 10 minutes for the same url +const cache = new Cache("emojiMeta",1000 * 60 * 10); // once every 10 minutes for the same url const logger = new Logger("emoji"); export async function getEmojiSize(url: string): Promise { diff --git a/packages/backend/src/misc/keypair-store.ts b/packages/backend/src/misc/keypair-store.ts index 4551bfd98..b0e07c4ab 100644 --- a/packages/backend/src/misc/keypair-store.ts +++ b/packages/backend/src/misc/keypair-store.ts @@ -3,7 +3,7 @@ import type { User } from "@/models/entities/user.js"; import type { UserKeypair } from "@/models/entities/user-keypair.js"; import { Cache } from "./cache.js"; -const cache = new Cache(Infinity); +const cache = new Cache("keypairStore", Infinity); export async function getUserKeypair(userId: User["id"]): Promise { return await cache.fetch(userId, () => diff --git a/packages/backend/src/misc/populate-emojis.ts b/packages/backend/src/misc/populate-emojis.ts index ce25dd559..e6e6c2fb9 100644 --- a/packages/backend/src/misc/populate-emojis.ts +++ b/packages/backend/src/misc/populate-emojis.ts @@ -9,7 +9,7 @@ import config from "@/config/index.js"; import { query } from "@/prelude/url.js"; import { redisClient } from "@/db/redis.js"; -const cache = new Cache(1000 * 60 * 60 * 12); +const cache = new Cache("populateEmojis", 1000 * 60 * 60 * 12); /** * 添付用絵文字情報 diff --git a/packages/backend/src/models/repositories/user.ts b/packages/backend/src/models/repositories/user.ts index 48c8d75b3..2bd8d4fba 100644 --- a/packages/backend/src/models/repositories/user.ts +++ b/packages/backend/src/models/repositories/user.ts @@ -40,7 +40,7 @@ import { } from "../index.js"; import type { Instance } from "../entities/instance.js"; -const userInstanceCache = new Cache(1000 * 60 * 60 * 3); +const userInstanceCache = new Cache("userInstance", 1000 * 60 * 60 * 3); type IsUserDetailed = Detailed extends true ? Packed<"UserDetailed"> diff --git a/packages/backend/src/remote/activitypub/db-resolver.ts b/packages/backend/src/remote/activitypub/db-resolver.ts index 6e448d4b1..4b4ea9627 100644 --- a/packages/backend/src/remote/activitypub/db-resolver.ts +++ b/packages/backend/src/remote/activitypub/db-resolver.ts @@ -20,8 +20,8 @@ import type { IObject } from "./type.js"; import { getApId } from "./type.js"; import { resolvePerson } from "./models/person.js"; -const publicKeyCache = new Cache(Infinity); -const publicKeyByUserIdCache = new Cache(Infinity); +const publicKeyCache = new Cache("publicKey", Infinity); +const publicKeyByUserIdCache = new Cache("publicKeyByUserId", Infinity); export type UriParseResult = | { diff --git a/packages/backend/src/server/api/authenticate.ts b/packages/backend/src/server/api/authenticate.ts index 42274ad2a..b6fa973eb 100644 --- a/packages/backend/src/server/api/authenticate.ts +++ b/packages/backend/src/server/api/authenticate.ts @@ -9,7 +9,7 @@ import { localUserByNativeTokenCache, } from "@/services/user-cache.js"; -const appCache = new Cache(Infinity); +const appCache = new Cache("app", Infinity); export class AuthenticationError extends Error { constructor(message: string) { diff --git a/packages/backend/src/server/nodeinfo.ts b/packages/backend/src/server/nodeinfo.ts index dbfb28ff6..28cefd2cf 100644 --- a/packages/backend/src/server/nodeinfo.ts +++ b/packages/backend/src/server/nodeinfo.ts @@ -100,7 +100,7 @@ const nodeinfo2 = async () => { }; }; -const cache = new Cache>>(1000 * 60 * 10); +const cache = new Cache>>("nodeinfo", 1000 * 60 * 10); router.get(nodeinfo2_1path, async (ctx) => { const base = await cache.fetch(null, () => nodeinfo2()); diff --git a/packages/backend/src/services/instance-actor.ts b/packages/backend/src/services/instance-actor.ts index 9240f3107..1822cb3c7 100644 --- a/packages/backend/src/services/instance-actor.ts +++ b/packages/backend/src/services/instance-actor.ts @@ -6,7 +6,7 @@ import { IsNull } from "typeorm"; const ACTOR_USERNAME = "instance.actor" as const; -const cache = new Cache(Infinity); +const cache = new Cache("instanceActor", Infinity); export async function getInstanceActor(): Promise { const cached = await cache.get(null); diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index f00678ce2..ca0f05f2c 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -73,7 +73,7 @@ import { Mutex } from "redis-semaphore"; const mutedWordsCache = new Cache< { userId: UserProfile["userId"]; mutedWords: UserProfile["mutedWords"] }[] ->(1000 * 60 * 5); +>("mutedWords", 1000 * 60 * 5); type NotificationType = "reply" | "renote" | "quote" | "mention"; diff --git a/packages/backend/src/services/register-or-fetch-instance-doc.ts b/packages/backend/src/services/register-or-fetch-instance-doc.ts index ddb9ce241..e0c288037 100644 --- a/packages/backend/src/services/register-or-fetch-instance-doc.ts +++ b/packages/backend/src/services/register-or-fetch-instance-doc.ts @@ -4,7 +4,7 @@ import { genId } from "@/misc/gen-id.js"; import { toPuny } from "@/misc/convert-host.js"; import { Cache } from "@/misc/cache.js"; -const cache = new Cache(1000 * 60 * 60); +const cache = new Cache("registerOrFetchInstanceDoc", 1000 * 60 * 60); export async function registerOrFetchInstanceDoc( host: string, diff --git a/packages/backend/src/services/relay.ts b/packages/backend/src/services/relay.ts index 2325f76c6..1ec2891e8 100644 --- a/packages/backend/src/services/relay.ts +++ b/packages/backend/src/services/relay.ts @@ -15,7 +15,7 @@ import { createSystemUser } from "./create-system-user.js"; const ACTOR_USERNAME = "relay.actor" as const; -const relaysCache = new Cache(1000 * 60 * 10); +const relaysCache = new Cache("relay", 1000 * 60 * 10); export async function getRelayActor(): Promise { const user = await Users.findOneBy({ diff --git a/packages/backend/src/services/user-cache.ts b/packages/backend/src/services/user-cache.ts index 373fb8686..7fde21d87 100644 --- a/packages/backend/src/services/user-cache.ts +++ b/packages/backend/src/services/user-cache.ts @@ -3,17 +3,17 @@ import type { CacheableUser, ILocalUser, } from "@/models/entities/user.js"; -import { User } from "@/models/entities/user.js"; import { Users } from "@/models/index.js"; import { Cache } from "@/misc/cache.js"; import { redisClient, subscriber } from "@/db/redis.js"; -export const userByIdCache = new Cache(Infinity); +export const userByIdCache = new Cache("userById", Infinity); export const localUserByNativeTokenCache = new Cache( + "localUserByNativeToken", Infinity, ); -export const localUserByIdCache = new Cache(Infinity); -export const uriPersonCache = new Cache(Infinity); +export const localUserByIdCache = new Cache("localUserByIdCache", Infinity); +export const uriPersonCache = new Cache("uriPerson", Infinity); subscriber.on("message", async (_, data) => { const obj = JSON.parse(data); From 284c0db1fde488ea5b0259701666790b599845d3 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Sun, 2 Jul 2023 22:10:33 -0400 Subject: [PATCH 3/7] no more infinity caches --- packages/backend/src/misc/cache.ts | 36 ++++++++++++------ .../backend/src/misc/check-hit-antenna.ts | 2 +- packages/backend/src/misc/emoji-meta.ts | 2 +- packages/backend/src/misc/keypair-store.ts | 8 ++-- packages/backend/src/misc/populate-emojis.ts | 2 +- .../backend/src/models/repositories/user.ts | 6 ++- .../src/remote/activitypub/db-resolver.ts | 38 ++++++++++++------- .../src/remote/activitypub/models/person.ts | 2 +- .../backend/src/server/api/authenticate.ts | 10 +++-- packages/backend/src/server/nodeinfo.ts | 5 ++- .../backend/src/services/instance-actor.ts | 4 +- packages/backend/src/services/note/create.ts | 7 +--- .../register-or-fetch-instance-doc.ts | 2 +- packages/backend/src/services/relay.ts | 2 +- packages/backend/src/services/user-cache.ts | 14 +++++-- 15 files changed, 89 insertions(+), 51 deletions(-) diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index c2695aa78..588931ff1 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -6,8 +6,8 @@ export class Cache { private ttl: number; private prefix: string; - constructor(prefix: string, ttl: number) { - this.ttl = ttl; + constructor(prefix: string, ttlSeconds: number) { + this.ttl = ttlSeconds; this.prefix = `cache:${prefix}`; } @@ -15,26 +15,28 @@ export class Cache { return key ? `${this.prefix}:${key}` : this.prefix; } - public async set(key: string | null, value: T, transaction?: ChainableCommander): Promise { + public async set( + key: string | null, + value: T, + transaction?: ChainableCommander, + ): Promise { const _key = this.prefixedKey(key); const _value = Buffer.from(encode(value)); const commander = transaction ?? redisClient; - if (this.ttl === Infinity) { - await commander.set(_key, _value); - } else { - await commander.set(_key, _value, "PX", this.ttl); - } + await commander.set(_key, _value, "EX", this.ttl); } - public async get(key: string | null): Promise { + public async get(key: string | null, renew = false): Promise { const _key = this.prefixedKey(key); const cached = await redisClient.getBuffer(_key); if (cached === null) return undefined; + if (renew) await redisClient.expire(_key, this.ttl); + return decode(cached) as T; } - public async getAll(): Promise> { + public async getAll(renew = false): Promise> { const keys = await redisClient.keys(`${this.prefix}*`); const map = new Map(); if (keys.length === 0) { @@ -49,6 +51,14 @@ export class Cache { } } + if (renew) { + const trans = redisClient.multi(); + for (const key of map.keys()) { + trans.expire(key, this.ttl); + } + await trans.exec(); + } + return map; } @@ -66,9 +76,10 @@ export class Cache { public async fetch( key: string | null, fetcher: () => Promise, + renew = false, validator?: (cachedValue: T) => boolean, ): Promise { - const cachedValue = await this.get(key); + const cachedValue = await this.get(key, renew); if (cachedValue !== undefined) { if (validator) { if (validator(cachedValue)) { @@ -94,9 +105,10 @@ export class Cache { public async fetchMaybe( key: string | null, fetcher: () => Promise, + renew = false, validator?: (cachedValue: T) => boolean, ): Promise { - const cachedValue = await this.get(key); + const cachedValue = await this.get(key, renew); if (cachedValue !== undefined) { if (validator) { if (validator(cachedValue)) { diff --git a/packages/backend/src/misc/check-hit-antenna.ts b/packages/backend/src/misc/check-hit-antenna.ts index c422cca94..1ff09d629 100644 --- a/packages/backend/src/misc/check-hit-antenna.ts +++ b/packages/backend/src/misc/check-hit-antenna.ts @@ -11,7 +11,7 @@ import * as Acct from "@/misc/acct.js"; import type { Packed } from "./schema.js"; import { Cache } from "./cache.js"; -const blockingCache = new Cache("blocking", 1000 * 60 * 5); +const blockingCache = new Cache("blocking", 60 * 5); // NOTE: フォローしているユーザーのノート、リストのユーザーのノート、グループのユーザーのノート指定はパフォーマンス上の理由で無効になっている diff --git a/packages/backend/src/misc/emoji-meta.ts b/packages/backend/src/misc/emoji-meta.ts index d2d15411f..2b9365b82 100644 --- a/packages/backend/src/misc/emoji-meta.ts +++ b/packages/backend/src/misc/emoji-meta.ts @@ -11,7 +11,7 @@ export type Size = { height: number; }; -const cache = new Cache("emojiMeta",1000 * 60 * 10); // once every 10 minutes for the same url +const cache = new Cache("emojiMeta", 60 * 10); // once every 10 minutes for the same url const logger = new Logger("emoji"); export async function getEmojiSize(url: string): Promise { diff --git a/packages/backend/src/misc/keypair-store.ts b/packages/backend/src/misc/keypair-store.ts index b0e07c4ab..625577359 100644 --- a/packages/backend/src/misc/keypair-store.ts +++ b/packages/backend/src/misc/keypair-store.ts @@ -3,10 +3,12 @@ import type { User } from "@/models/entities/user.js"; import type { UserKeypair } from "@/models/entities/user-keypair.js"; import { Cache } from "./cache.js"; -const cache = new Cache("keypairStore", Infinity); +const cache = new Cache("keypairStore", 60 * 30); export async function getUserKeypair(userId: User["id"]): Promise { - return await cache.fetch(userId, () => - UserKeypairs.findOneByOrFail({ userId: userId }), + return await cache.fetch( + userId, + () => UserKeypairs.findOneByOrFail({ userId: userId }), + true, ); } diff --git a/packages/backend/src/misc/populate-emojis.ts b/packages/backend/src/misc/populate-emojis.ts index e6e6c2fb9..795a267f9 100644 --- a/packages/backend/src/misc/populate-emojis.ts +++ b/packages/backend/src/misc/populate-emojis.ts @@ -9,7 +9,7 @@ import config from "@/config/index.js"; import { query } from "@/prelude/url.js"; import { redisClient } from "@/db/redis.js"; -const cache = new Cache("populateEmojis", 1000 * 60 * 60 * 12); +const cache = new Cache("populateEmojis", 60 * 60 * 12); /** * 添付用絵文字情報 diff --git a/packages/backend/src/models/repositories/user.ts b/packages/backend/src/models/repositories/user.ts index 2bd8d4fba..5ca36e3d3 100644 --- a/packages/backend/src/models/repositories/user.ts +++ b/packages/backend/src/models/repositories/user.ts @@ -1,4 +1,3 @@ -import { URL } from "url"; import { In, Not } from "typeorm"; import Ajv from "ajv"; import type { ILocalUser, IRemoteUser } from "@/models/entities/user.js"; @@ -40,7 +39,10 @@ import { } from "../index.js"; import type { Instance } from "../entities/instance.js"; -const userInstanceCache = new Cache("userInstance", 1000 * 60 * 60 * 3); +const userInstanceCache = new Cache( + "userInstance", + 60 * 60 * 3, +); type IsUserDetailed = Detailed extends true ? Packed<"UserDetailed"> diff --git a/packages/backend/src/remote/activitypub/db-resolver.ts b/packages/backend/src/remote/activitypub/db-resolver.ts index 4b4ea9627..a710b9f11 100644 --- a/packages/backend/src/remote/activitypub/db-resolver.ts +++ b/packages/backend/src/remote/activitypub/db-resolver.ts @@ -5,7 +5,6 @@ import type { CacheableRemoteUser, CacheableUser, } from "@/models/entities/user.js"; -import { User, IRemoteUser } from "@/models/entities/user.js"; import type { UserPublickey } from "@/models/entities/user-publickey.js"; import type { MessagingMessage } from "@/models/entities/messaging-message.js"; import { @@ -20,8 +19,11 @@ import type { IObject } from "./type.js"; import { getApId } from "./type.js"; import { resolvePerson } from "./models/person.js"; -const publicKeyCache = new Cache("publicKey", Infinity); -const publicKeyByUserIdCache = new Cache("publicKeyByUserId", Infinity); +const publicKeyCache = new Cache("publicKey", 60 * 30); +const publicKeyByUserIdCache = new Cache( + "publicKeyByUserId", + 60 * 30, +); export type UriParseResult = | { @@ -123,17 +125,23 @@ export default class DbResolver { if (parsed.type !== "users") return null; return ( - (await userByIdCache.fetchMaybe(parsed.id, () => - Users.findOneBy({ - id: parsed.id, - }).then((x) => x ?? undefined), + (await userByIdCache.fetchMaybe( + parsed.id, + () => + Users.findOneBy({ + id: parsed.id, + }).then((x) => x ?? undefined), + true, )) ?? null ); } else { - return await uriPersonCache.fetch(parsed.uri, () => - Users.findOneBy({ - uri: parsed.uri, - }), + return await uriPersonCache.fetch( + parsed.uri, + () => + Users.findOneBy({ + uri: parsed.uri, + }), + true, ); } } @@ -156,14 +164,17 @@ export default class DbResolver { return key; }, + true, (key) => key != null, ); if (key == null) return null; return { - user: (await userByIdCache.fetch(key.userId, () => - Users.findOneByOrFail({ id: key.userId }), + user: (await userByIdCache.fetch( + key.userId, + () => Users.findOneByOrFail({ id: key.userId }), + true, )) as CacheableRemoteUser, key, }; @@ -183,6 +194,7 @@ export default class DbResolver { const key = await publicKeyByUserIdCache.fetch( user.id, () => UserPublickeys.findOneBy({ userId: user.id }), + true, (v) => v != null, ); diff --git a/packages/backend/src/remote/activitypub/models/person.ts b/packages/backend/src/remote/activitypub/models/person.ts index c541e9ae5..c5519ba03 100644 --- a/packages/backend/src/remote/activitypub/models/person.ts +++ b/packages/backend/src/remote/activitypub/models/person.ts @@ -135,7 +135,7 @@ export async function fetchPerson( ): Promise { if (typeof uri !== "string") throw new Error("uri is not string"); - const cached = await uriPersonCache.get(uri); + const cached = await uriPersonCache.get(uri, true); if (cached) return cached; // Fetch from the database if the URI points to this server diff --git a/packages/backend/src/server/api/authenticate.ts b/packages/backend/src/server/api/authenticate.ts index b6fa973eb..460a0ce84 100644 --- a/packages/backend/src/server/api/authenticate.ts +++ b/packages/backend/src/server/api/authenticate.ts @@ -9,7 +9,7 @@ import { localUserByNativeTokenCache, } from "@/services/user-cache.js"; -const appCache = new Cache("app", Infinity); +const appCache = new Cache("app", 60 * 30); export class AuthenticationError extends Error { constructor(message: string) { @@ -49,6 +49,7 @@ export default async ( const user = await localUserByNativeTokenCache.fetch( token, () => Users.findOneBy({ token }) as Promise, + true, ); if (user == null) { @@ -82,11 +83,14 @@ export default async ( Users.findOneBy({ id: accessToken.userId, }) as Promise, + true, ); if (accessToken.appId) { - const app = await appCache.fetch(accessToken.appId, () => - Apps.findOneByOrFail({ id: accessToken.appId! }), + const app = await appCache.fetch( + accessToken.appId, + () => Apps.findOneByOrFail({ id: accessToken.appId! }), + true, ); return [ diff --git a/packages/backend/src/server/nodeinfo.ts b/packages/backend/src/server/nodeinfo.ts index 28cefd2cf..940ca2e13 100644 --- a/packages/backend/src/server/nodeinfo.ts +++ b/packages/backend/src/server/nodeinfo.ts @@ -100,7 +100,10 @@ const nodeinfo2 = async () => { }; }; -const cache = new Cache>>("nodeinfo", 1000 * 60 * 10); +const cache = new Cache>>( + "nodeinfo", + 60 * 10, +); router.get(nodeinfo2_1path, async (ctx) => { const base = await cache.fetch(null, () => nodeinfo2()); diff --git a/packages/backend/src/services/instance-actor.ts b/packages/backend/src/services/instance-actor.ts index 1822cb3c7..a8b34ea57 100644 --- a/packages/backend/src/services/instance-actor.ts +++ b/packages/backend/src/services/instance-actor.ts @@ -6,10 +6,10 @@ import { IsNull } from "typeorm"; const ACTOR_USERNAME = "instance.actor" as const; -const cache = new Cache("instanceActor", Infinity); +const cache = new Cache("instanceActor", 60 * 30); export async function getInstanceActor(): Promise { - const cached = await cache.get(null); + const cached = await cache.get(null, true); if (cached) return cached; const user = (await Users.findOneBy({ diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index ca0f05f2c..095c75f42 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -29,17 +29,14 @@ import { Notes, Instances, UserProfiles, - Antennas, - Followings, MutedNotes, Channels, ChannelFollowings, - Blockings, NoteThreadMutings, } from "@/models/index.js"; import type { DriveFile } from "@/models/entities/drive-file.js"; import type { App } from "@/models/entities/app.js"; -import { Not, In, IsNull } from "typeorm"; +import { Not, In } from "typeorm"; import type { User, ILocalUser, IRemoteUser } from "@/models/entities/user.js"; import { genId } from "@/misc/gen-id.js"; import { @@ -73,7 +70,7 @@ import { Mutex } from "redis-semaphore"; const mutedWordsCache = new Cache< { userId: UserProfile["userId"]; mutedWords: UserProfile["mutedWords"] }[] ->("mutedWords", 1000 * 60 * 5); +>("mutedWords", 60 * 5); type NotificationType = "reply" | "renote" | "quote" | "mention"; diff --git a/packages/backend/src/services/register-or-fetch-instance-doc.ts b/packages/backend/src/services/register-or-fetch-instance-doc.ts index e0c288037..c0ead0819 100644 --- a/packages/backend/src/services/register-or-fetch-instance-doc.ts +++ b/packages/backend/src/services/register-or-fetch-instance-doc.ts @@ -4,7 +4,7 @@ import { genId } from "@/misc/gen-id.js"; import { toPuny } from "@/misc/convert-host.js"; import { Cache } from "@/misc/cache.js"; -const cache = new Cache("registerOrFetchInstanceDoc", 1000 * 60 * 60); +const cache = new Cache("registerOrFetchInstanceDoc", 60 * 60); export async function registerOrFetchInstanceDoc( host: string, diff --git a/packages/backend/src/services/relay.ts b/packages/backend/src/services/relay.ts index 1ec2891e8..6f7829c21 100644 --- a/packages/backend/src/services/relay.ts +++ b/packages/backend/src/services/relay.ts @@ -15,7 +15,7 @@ import { createSystemUser } from "./create-system-user.js"; const ACTOR_USERNAME = "relay.actor" as const; -const relaysCache = new Cache("relay", 1000 * 60 * 10); +const relaysCache = new Cache("relay", 60 * 10); export async function getRelayActor(): Promise { const user = await Users.findOneBy({ diff --git a/packages/backend/src/services/user-cache.ts b/packages/backend/src/services/user-cache.ts index 7fde21d87..ed700185d 100644 --- a/packages/backend/src/services/user-cache.ts +++ b/packages/backend/src/services/user-cache.ts @@ -7,13 +7,19 @@ import { Users } from "@/models/index.js"; import { Cache } from "@/misc/cache.js"; import { redisClient, subscriber } from "@/db/redis.js"; -export const userByIdCache = new Cache("userById", Infinity); +export const userByIdCache = new Cache("userById", 60 * 30); export const localUserByNativeTokenCache = new Cache( "localUserByNativeToken", - Infinity, + 60 * 30, +); +export const localUserByIdCache = new Cache( + "localUserByIdCache", + 60 * 30, +); +export const uriPersonCache = new Cache( + "uriPerson", + 60 * 30, ); -export const localUserByIdCache = new Cache("localUserByIdCache", Infinity); -export const uriPersonCache = new Cache("uriPerson", Infinity); subscriber.on("message", async (_, data) => { const obj = JSON.parse(data); From 6919477820f6d5f7e5e1a780018a3ae3d45107ef Mon Sep 17 00:00:00 2001 From: Syuilo Date: Sun, 2 Jul 2023 16:20:40 -0700 Subject: [PATCH 4/7] =?UTF-8?q?refactor:=20=E2=9A=A1=20make=20identicons?= =?UTF-8?q?=20and=20server=20metrics=20optional?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Kainoa Kanter --- locales/en-US.yml | 2 ++ locales/ja-JP.yml | 2 ++ packages/backend/assets/avatar.png | 3 +++ .../1688280713783-add-meta-options.js | 21 +++++++++++++++++++ packages/backend/src/daemons/server-stats.ts | 4 ++++ packages/backend/src/models/entities/meta.ts | 10 +++++++++ .../src/server/api/endpoints/admin/meta.ts | 12 +++++++++++ .../src/server/api/endpoints/server-info.ts | 21 ++++++++++++++++++- packages/backend/src/server/index.ts | 15 +++++++++---- packages/client/src/pages/admin/settings.vue | 17 +++++++++++++++ .../src/widgets/server-metric/index.vue | 2 +- 11 files changed, 103 insertions(+), 6 deletions(-) create mode 100644 packages/backend/assets/avatar.png create mode 100644 packages/backend/migration/1688280713783-add-meta-options.js diff --git a/locales/en-US.yml b/locales/en-US.yml index d493f332f..c6c442df7 100644 --- a/locales/en-US.yml +++ b/locales/en-US.yml @@ -1112,6 +1112,8 @@ isModerator: "Moderator" isAdmin: "Administrator" isPatron: "Calckey Patron" reactionPickerSkinTone: "Preferred emoji skin tone" +enableServerMachineStats: "Enable server hardware statistics" +enableIdenticonGeneration: "Enable Identicon generation" _sensitiveMediaDetection: description: "Reduces the effort of server moderation through automatically recognizing diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index fa3c3f1cc..9f2f825e5 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -978,6 +978,8 @@ enableCustomKaTeXMacro: "カスタムKaTeXマクロを有効にする" preventAiLearning: "AIによる学習を防止" preventAiLearningDescription: "投稿したノート、添付した画像などのコンテンツを学習の対象にしないようAIに要求します。これはnoaiフラグをHTMLレスポンスに含めることによって実現されます。" noGraze: "ブラウザの拡張機能「Graze for Mastodon」は、Calckeyの動作を妨げるため、無効にしてください。" +enableServerMachineStats: "サーバーのマシン情報を公開する" +enableIdenticonGeneration: "ユーザーごとのIdenticon生成を有効にする" _sensitiveMediaDetection: description: "機械学習を使って自動でセンシティブなメディアを検出し、モデレーションに役立てられます。サーバーの負荷が少し増えます。" diff --git a/packages/backend/assets/avatar.png b/packages/backend/assets/avatar.png new file mode 100644 index 000000000..c1a3501e7 --- /dev/null +++ b/packages/backend/assets/avatar.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e770a13738887f9fbb0b62f9881c7035a36c36832676ae10de531cd5c4c2cc8 +size 14059 diff --git a/packages/backend/migration/1688280713783-add-meta-options.js b/packages/backend/migration/1688280713783-add-meta-options.js new file mode 100644 index 000000000..e97a95c42 --- /dev/null +++ b/packages/backend/migration/1688280713783-add-meta-options.js @@ -0,0 +1,21 @@ +export class AddMetaOptions1688280713783 { + name = "AddMetaOptions1688280713783"; + + async up(queryRunner) { + await queryRunner.query( + `ALTER TABLE "meta" ADD "enableServerMachineStats" boolean NOT NULL DEFAULT false`, + ); + await queryRunner.query( + `ALTER TABLE "meta" ADD "enableIdenticonGeneration" boolean NOT NULL DEFAULT true`, + ); + } + + async down(queryRunner) { + await queryRunner.query( + `ALTER TABLE "meta" DROP COLUMN "enableIdenticonGeneration"`, + ); + await queryRunner.query( + `ALTER TABLE "meta" DROP COLUMN "enableServerMachineStats"`, + ); + } +} diff --git a/packages/backend/src/daemons/server-stats.ts b/packages/backend/src/daemons/server-stats.ts index c936d619a..ba7427876 100644 --- a/packages/backend/src/daemons/server-stats.ts +++ b/packages/backend/src/daemons/server-stats.ts @@ -1,6 +1,7 @@ import si from "systeminformation"; import Xev from "xev"; import * as osUtils from "os-utils"; +import { fetchMeta } from "@/misc/fetch-meta.js"; import meilisearch from "../db/meilisearch.js"; const ev = new Xev(); @@ -20,6 +21,9 @@ export default function () { ev.emit(`serverStatsLog:${x.id}`, log.slice(0, x.length || 50)); }); + const meta = fetchMeta(); + if (!meta.enableServerMachineStats) return; + async function tick() { const cpu = await cpuUsage(); const memStats = await mem(); diff --git a/packages/backend/src/models/entities/meta.ts b/packages/backend/src/models/entities/meta.ts index dd3c5b3b7..200ef5055 100644 --- a/packages/backend/src/models/entities/meta.ts +++ b/packages/backend/src/models/entities/meta.ts @@ -546,4 +546,14 @@ export class Meta { default: {}, }) public experimentalFeatures: Record; + + @Column("boolean", { + default: false, + }) + public enableServerMachineStats: boolean; + + @Column("boolean", { + default: true, + }) + public enableIdenticonGeneration: boolean; } diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index 319330127..50317d4a8 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -481,6 +481,16 @@ export const meta = { }, }, }, + enableServerMachineStats: { + type: "boolean", + optional: false, + nullable: false, + }, + enableIdenticonGeneration: { + type: "boolean", + optional: false, + nullable: false, + }, }, }, } as const; @@ -592,5 +602,7 @@ export default define(meta, paramDef, async (ps, me) => { enableIpLogging: instance.enableIpLogging, enableActiveEmailValidation: instance.enableActiveEmailValidation, experimentalFeatures: instance.experimentalFeatures, + enableServerMachineStats: instance.enableServerMachineStats, + enableIdenticonGeneration: instance.enableIdenticonGeneration, }; }); diff --git a/packages/backend/src/server/api/endpoints/server-info.ts b/packages/backend/src/server/api/endpoints/server-info.ts index 81bb053db..746eae662 100644 --- a/packages/backend/src/server/api/endpoints/server-info.ts +++ b/packages/backend/src/server/api/endpoints/server-info.ts @@ -2,11 +2,13 @@ import * as os from "node:os"; import si from "systeminformation"; import define from "../define.js"; import meilisearch from "@/db/meilisearch.js"; +import { fetchMeta } from "@/misc/fetch-meta.js"; export const meta = { requireCredential: false, requireCredentialPrivateMode: true, - + allowGet: true, + cacheSec: 30, tags: ["meta"], } as const; @@ -29,6 +31,23 @@ export default define(meta, paramDef, async () => { } } + const instanceMeta = await fetchMeta(); + if (!instanceMeta.enableServerMachineStats) { + return { + machine: 'Not specified', + cpu: { + model: 'Not specified', + cores: 0, + }, + mem: { + total: 0, + }, + fs: { + total: 0, + used: 0, + }, + }; + } return { machine: os.hostname(), cpu: { diff --git a/packages/backend/src/server/index.ts b/packages/backend/src/server/index.ts index 95d570eb3..7f8d0ed71 100644 --- a/packages/backend/src/server/index.ts +++ b/packages/backend/src/server/index.ts @@ -16,6 +16,7 @@ import { IsNull } from "typeorm"; import config from "@/config/index.js"; import Logger from "@/services/logger.js"; import { UserProfiles, Users } from "@/models/index.js"; +import { fetchMeta } from "@/misc/fetch-meta.js"; import { genIdenticon } from "@/misc/gen-identicon.js"; import { createTemp } from "@/misc/create-temp.js"; import { publishMainStream } from "@/services/stream.js"; @@ -125,10 +126,16 @@ router.get("/avatar/@:acct", async (ctx) => { }); router.get("/identicon/:x", async (ctx) => { - const [temp, cleanup] = await createTemp(); - await genIdenticon(ctx.params.x, fs.createWriteStream(temp)); - ctx.set("Content-Type", "image/png"); - ctx.body = fs.createReadStream(temp).on("close", () => cleanup()); + const meta = await fetchMeta(); + if (meta.enableIdenticonGeneration) { + const [temp, cleanup] = await createTemp(); + await genIdenticon(ctx.params.x, fs.createWriteStream(temp)); + ctx.set("Content-Type", "image/png"); + ctx.body = fs.createReadStream(temp).on("close", () => cleanup()); + } + else { + ctx.redirect("/static-assets/avatar.png") + } }); mastoRouter.get("/oauth/authorize", async (ctx) => { diff --git a/packages/client/src/pages/admin/settings.vue b/packages/client/src/pages/admin/settings.vue index e67c44b0a..0e83250ba 100644 --- a/packages/client/src/pages/admin/settings.vue +++ b/packages/client/src/pages/admin/settings.vue @@ -343,6 +343,17 @@ + + + + + + + + + + + @@ -442,6 +453,8 @@ let libreTranslateApiUrl: string = $ref(""); let libreTranslateApiKey: string = $ref(""); let defaultReaction: string = $ref(""); let defaultReactionCustom: string = $ref(""); +let enableServerMachineStats: boolean = $ref(false); +let enableIdenticonGeneration: boolean = $ref(false); async function init() { const meta = await os.api("admin/meta"); @@ -482,6 +495,8 @@ async function init() { defaultReactionCustom = ["⭐", "👍", "❤️"].includes(meta.defaultReaction) ? "" : meta.defaultReaction; + enableServerMachineStats = meta.enableServerMachineStats; + enableIdenticonGeneration = meta.enableIdenticonGeneration; } function save() { @@ -521,6 +536,8 @@ function save() { libreTranslateApiUrl, libreTranslateApiKey, defaultReaction, + enableServerMachineStats, + enableIdenticonGeneration, }).then(() => { fetchInstance(); }); diff --git a/packages/client/src/widgets/server-metric/index.vue b/packages/client/src/widgets/server-metric/index.vue index cf84212b1..49dc59ea2 100644 --- a/packages/client/src/widgets/server-metric/index.vue +++ b/packages/client/src/widgets/server-metric/index.vue @@ -106,7 +106,7 @@ const { widgetProps, configure, save } = useWidgetPropsManager( const meta = ref(null); -os.api("server-info", {}).then((res) => { +os.apiGet("server-info", {}).then((res) => { meta.value = res; }); From b36cc31e9b9858693bb36731aef4fd58a5facc12 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Sun, 2 Jul 2023 22:21:19 -0400 Subject: [PATCH 5/7] throw error if failed --- .../backend/src/server/api/endpoints/admin/emoji/add.ts | 9 ++------- .../backend/src/server/api/endpoints/admin/emoji/copy.ts | 9 ++------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/add.ts b/packages/backend/src/server/api/endpoints/admin/emoji/add.ts index 7d4081613..4366406ec 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/add.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/add.ts @@ -6,7 +6,7 @@ import { ApiError } from "../../../error.js"; import rndstr from "rndstr"; import { publishBroadcastStream } from "@/services/stream.js"; import { db } from "@/db/postgre.js"; -import { type Size, getEmojiSize } from "@/misc/emoji-meta.js"; +import { getEmojiSize } from "@/misc/emoji-meta.js"; export const meta = { tags: ["admin"], @@ -40,12 +40,7 @@ export default define(meta, paramDef, async (ps, me) => { ? file.name.split(".")[0] : `_${rndstr("a-z0-9", 8)}_`; - let size: Size = { width: 0, height: 0 }; - try { - size = await getEmojiSize(file.url); - } catch { - /* skip if any error happens */ - } + const size = await getEmojiSize(file.url); const emoji = await Emojis.insert({ id: genId(), diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts b/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts index 45cb9464d..c90e60633 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts @@ -6,7 +6,7 @@ import type { DriveFile } from "@/models/entities/drive-file.js"; import { uploadFromUrl } from "@/services/drive/upload-from-url.js"; import { publishBroadcastStream } from "@/services/stream.js"; import { db } from "@/db/postgre.js"; -import { type Size, getEmojiSize } from "@/misc/emoji-meta.js"; +import { getEmojiSize } from "@/misc/emoji-meta.js"; export const meta = { tags: ["admin"], @@ -65,12 +65,7 @@ export default define(meta, paramDef, async (ps, me) => { throw new ApiError(); } - let size: Size = { width: 0, height: 0 }; - try { - size = await getEmojiSize(driveFile.url); - } catch { - /* skip if any error happens */ - } + const size = await getEmojiSize(driveFile.url); const copied = await Emojis.insert({ id: genId(), From 04c166b12c341b840570266155be38298fd41d51 Mon Sep 17 00:00:00 2001 From: Poesty Li Date: Mon, 3 Jul 2023 01:25:50 +0000 Subject: [PATCH 6/7] chore: Translated using Weblate (Chinese (Simplified)) Currently translated at 97.4% (1769 of 1816 strings) Translation: Calckey/locales Translate-URL: https://hosted.weblate.org/projects/calckey/locales/zh_Hans/ --- locales/zh-CN.yml | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/locales/zh-CN.yml b/locales/zh-CN.yml index 6903f447d..110149d73 100644 --- a/locales/zh-CN.yml +++ b/locales/zh-CN.yml @@ -10,7 +10,7 @@ password: "密码" forgotPassword: "忘记密码" fetchingAsApObject: "正在联邦宇宙查询中" ok: "OK" -gotIt: "我明白了" +gotIt: "知道了！" cancel: "取消" enterUsername: "输入用户名" renotedBy: "转发自 {user}" @@ -78,7 +78,7 @@ followsYou: "正在关注你" createList: "创建列表" manageLists: "管理列表" error: "错误" -somethingHappened: "出现了一些问题！" +somethingHappened: "发生了一个错误" retry: "重试" pageLoadError: "页面加载失败。" pageLoadErrorDescription: "这通常是由于网络或浏览器缓存的原因。请清除缓存或等待片刻后重试。" @@ -202,7 +202,7 @@ noUsers: "无用户" editProfile: "编辑资料" noteDeleteConfirm: "要删除该帖子吗？" pinLimitExceeded: "无法置顶更多帖子了" -intro: "Misskey的部署结束啦！填写管理员账号吧！" +intro: "Calckey安装完成！请创建一个管理员用户。" done: "完成" processing: "正在处理" preview: "预览" @@ -222,10 +222,10 @@ instanceFollowers: "服务器的关注者" instanceUsers: "此服务器的用户" changePassword: "修改密码" security: "安全" -retypedNotMatch: "两次输入不一致！" +retypedNotMatch: "两次输入不匹配。" currentPassword: "现在的密码" newPassword: "新密码" -newPasswordRetype: "重新输入密码：" +newPasswordRetype: "重新输入新密码" attachFile: "插入附件" more: "更多！" featured: "热门" @@ -391,7 +391,7 @@ nUsersMentioned: "{n} 被提到" securityKey: "安全密钥" securityKeyName: "密钥名称" registerSecurityKey: "注册硬件安全密钥" -lastUsed: "最后使用：" +lastUsed: "上次使用" unregister: "删除账户" passwordLessLogin: "无密码登录" resetPassword: "重置密码" @@ -639,7 +639,7 @@ openInNewTab: "在新标签页中打开" openInSideView: "在侧边栏中打开" defaultNavigationBehaviour: "默认导航" editTheseSettingsMayBreakAccount: "编辑这些设置可以会损坏您的账号" -instanceTicker: "帖子的实例信息" +instanceTicker: "帖子的服务器信息" waitingFor: "等待{x}" random: "随机" system: "系统" @@ -759,7 +759,7 @@ instanceBlocking: "联邦管理" selectAccount: "选择账户" switchAccount: "切换账户" enabled: "已启用" -disabled: "已禁用 " +disabled: "已禁用" quickAction: "快捷操作" user: "用户" administration: "管理" @@ -875,7 +875,7 @@ statusbar: "状态栏" pleaseSelect: "请选择" reverse: "翻转" colored: "彩色" -refreshInterval: "刷新间隔" +refreshInterval: "更新间隔 " label: "标签" type: "类型" speed: "速度" @@ -1220,17 +1220,17 @@ _tutorial: step3_2: "你的主页和社交馈送是基于你所关注的人，所以试着先关注几个账户。{n点击个人资料右上角的加号圈就可以关注它。" step4_1: "让我们出去找你。" step4_2: "对于他们的第一条信息，有些人喜欢做{introduction}或一个简单的 \"hello world!\"" - step5_1: "时间限制，到处是时间限制！" - step5_2: "您的实例已启用各种时间线的{timelines}。" - step5_3: "主{icon}时间线是你可以看到你的订阅者的帖子的时间线。" - step5_4: "本地{icon}时间线是你可以看到实例中所有其他用户的信息的时间线。" - step5_5: "推荐的{icon}时间线 - 是时间轴，你可以看到管理员推荐的实例的信息" - step5_6: "社交{icon}时间线显示来自你的订阅者朋友的信息。" - step5_7: "全球{icon}时间线是你可以看到来自所有其他连接的实例的消息。" + step5_1: "时间线，无处不在的时间线！" + step5_2: "您的服务器已启用{timelines}种不同的时间线。" + step5_3: "主页{icon}时间线是你可以看到你关注账户的帖子的时间线。" + step5_4: "本地{icon}时间线是你可以看到此服务器上其它用户的帖子的时间线。" + step5_5: "社交{icon}时间线是主页和本地时间线的结合。" + step5_6: "推荐{icon}时间线是你可以看到管理员推荐服务器的帖子的时间线。" + step5_7: "全球{icon}时间线是你可以看到来自其它所有互联服务器的帖子的时间线。" step6_1: "那么，这里是什么地方？" - step6_2: "好吧，你不只是加入卡尔基。你已经加入了Fediverse的一个门户，这是一个由成千上万台服务器组成的互联网络，被称为 \"实例\"" + step6_2: "好吧，你不只是加入Calckey。你已经加入了Fediverse的一个门户，这是一个由成千上万台服务器组成的互联网络。" step6_3: "每个服务器的工作方式不同，并不是所有的服务器都运行Calckey。但这个人确实如此! 这有点复杂，但你很快就会明白的。" - step6_4: "现在去学习并享受乐趣！" + step6_4: "现在，去吧，去探索，去享受乐趣吧!" _2fa: alreadyRegistered: "此设备已被注册" registerTOTP: "注册设备" @@ -1292,7 +1292,7 @@ _permissions: _auth: shareAccess: "您要授权允许“{name}”访问您的帐户吗？" shareAccessAsk: "您确定要授权此应用程序访问您的帐户吗？" - permissionAsk: "这个应用程序需要以下权限" + permissionAsk: "此应用程序请求以下权限：" pleaseGoBack: "请返回到应用程序" callback: "回到应用程序" denied: "拒绝访问" @@ -1433,7 +1433,7 @@ _instanceCharts: usersTotal: "用户总计" notes: "帖子：增加/减少" notesTotal: "帖子总计" - ff: "关注/被关注：数量变化" + ff: "被关注用户/关注者的数量差异 " ffTotal: "关注/被关注者总计" cacheSize: "缓存大小：增加/减少" cacheSizeTotal: "缓存大小总计" @@ -1939,3 +1939,5 @@ isPatron: Calckey 赞助 _dialog: charactersExceeded: 超出了最大字符数！当前：{current} / 限制：{max} charactersBelow: 没有足够的字符！当前：{current} / 限制：{min} +enableIdenticonGeneration: 启用Identicon生成 +enableServerMachineStats: 启用服务器硬件统计 From 6f5e07de5d6a5f976969313b90c9108e84fc1258 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Sun, 2 Jul 2023 23:14:28 -0400 Subject: [PATCH 7/7] rename arg --- packages/backend/src/misc/cache.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index 588931ff1..fe68908e5 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -6,9 +6,9 @@ export class Cache { private ttl: number; private prefix: string; - constructor(prefix: string, ttlSeconds: number) { + constructor(name: string, ttlSeconds: number) { this.ttl = ttlSeconds; - this.prefix = `cache:${prefix}`; + this.prefix = `cache:${name}`; } private prefixedKey(key: string | null): string {