From 63d28adeff9ed2b5e5d84c58e88a443c2637dc83 Mon Sep 17 00:00:00 2001
From: ThatOneCalculator <kainoa@t1c.dev>
Date: Sat, 15 Jul 2023 00:19:09 -0700
Subject: [PATCH 1/5] =?UTF-8?q?fix:=20=F0=9F=90=9B=20allow=20up=20to=20102?=
=?UTF-8?q?4=20chars=20for=20SMTP=20login?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Closes #10472
---
.../migration/1678426061773-tweak-varchar-length.js | 10 ++++++++++
packages/backend/src/models/entities/meta.ts | 4 ++--
2 files changed, 12 insertions(+), 2 deletions(-)
create mode 100644 packages/backend/migration/1678426061773-tweak-varchar-length.js
diff --git a/packages/backend/migration/1678426061773-tweak-varchar-length.js b/packages/backend/migration/1678426061773-tweak-varchar-length.js
new file mode 100644
index 000000000..883374599
--- /dev/null
+++ b/packages/backend/migration/1678426061773-tweak-varchar-length.js
@@ -0,0 +1,10 @@
+export class tweakVarcharLength1678426061773 {
+ name = 'tweakVarcharLength1678426061773'
+
+ async up(queryRunner) {
+ await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "smtpUser" TYPE character varying(1024)`, undefined);
+ await queryRunner.query(`ALTER TABLE "meta" ALTER COLUMN "smtpPass" TYPE character varying(1024)`, undefined);
+ }
+
+ async down(queryRunner) {}
+}
diff --git a/packages/backend/src/models/entities/meta.ts b/packages/backend/src/models/entities/meta.ts
index d3de7ba53..e90a5269a 100644
--- a/packages/backend/src/models/entities/meta.ts
+++ b/packages/backend/src/models/entities/meta.ts
@@ -326,13 +326,13 @@ export class Meta {
public smtpPort: number | null;
@Column("varchar", {
- length: 128,
+ length: 1024,
nullable: true,
})
public smtpUser: string | null;
@Column("varchar", {
- length: 128,
+ length: 1024,
nullable: true,
})
public smtpPass: string | null;
From 7308d9f90f5dd2735bfee48db173cc7dee987b29 Mon Sep 17 00:00:00 2001
From: ThatOneCalculator <kainoa@t1c.dev>
Date: Sat, 15 Jul 2023 14:28:44 -0700
Subject: [PATCH 2/5] dev78
---
package.json | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/package.json b/package.json
index b38da39fa..a30d85b3d 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "calckey",
- "version": "14.0.0-dev77",
+ "version": "14.0.0-dev78",
"codename": "aqua",
"repository": {
"type": "git",
From 5324d1298fc2157adb9777e631a66814d6cc42c6 Mon Sep 17 00:00:00 2001
From: ThatOneCalculator <kainoa@t1c.dev>
Date: Sat, 15 Jul 2023 14:50:23 -0700
Subject: [PATCH 3/5] fix: :wrench: max post length 100000
Pleroma doesn't accept >100K (ref: https://blob.cat/objects/82f33d96-534f-45ee-902a-f77c2723db8b)
---
.config/example.yml | 2 +-
packages/backend/src/const.ts | 7 +++++--
packages/backend/src/misc/hard-limits.ts | 3 +--
3 files changed, 7 insertions(+), 5 deletions(-)
diff --git a/.config/example.yml b/.config/example.yml
index 9716915b6..f73f4f1d7 100644
--- a/.config/example.yml
+++ b/.config/example.yml
@@ -121,7 +121,7 @@ redis:
# ┌─────────────────────┐
#───┘ Other configuration └─────────────────────────────────────
-# Maximum length of a post (default 3000, max 250000000)
+# Maximum length of a post (default 3000, max 100000)
#maxNoteLength: 3000
# Maximum length of an image caption (default 1500, max 8192)
diff --git a/packages/backend/src/const.ts b/packages/backend/src/const.ts
index 1f1e63292..49f012c5e 100644
--- a/packages/backend/src/const.ts
+++ b/packages/backend/src/const.ts
@@ -1,7 +1,10 @@
import config from "@/config/index.js";
-import { DB_MAX_IMAGE_COMMENT_LENGTH } from "@/misc/hard-limits.js";
+import { DB_MAX_NOTE_TEXT_LENGTH, DB_MAX_IMAGE_COMMENT_LENGTH } from "@/misc/hard-limits.js";
-export const MAX_NOTE_TEXT_LENGTH = config.maxNoteLength ?? 3000;
+export const MAX_NOTE_TEXT_LENGTH = Math.min(
+ config.maxNoteLength ?? 3000,
+ DB_MAX_NOTE_TEXT_LENGTH,
+);
export const MAX_CAPTION_TEXT_LENGTH = Math.min(
config.maxCaptionLength ?? 1500,
DB_MAX_IMAGE_COMMENT_LENGTH,
diff --git a/packages/backend/src/misc/hard-limits.ts b/packages/backend/src/misc/hard-limits.ts
index f1a58a425..62671a706 100644
--- a/packages/backend/src/misc/hard-limits.ts
+++ b/packages/backend/src/misc/hard-limits.ts
@@ -3,9 +3,8 @@
/**
* Maximum note text length that can be stored in DB.
* Surrogate pairs count as one
- * DEPRECARTED: use const/MAX_NOTE_TEXT_LENGTH instead
*/
-// export const DB_MAX_NOTE_TEXT_LENGTH = 8192;
+export const DB_MAX_NOTE_TEXT_LENGTH = 100000;
/**
* Maximum image description length that can be stored in DB.
From 3ee2dc787b814517642e46d6c8b96aa9542ea42f Mon Sep 17 00:00:00 2001
From: ThatOneCalculator <kainoa@t1c.dev>
Date: Sat, 15 Jul 2023 15:32:39 -0700
Subject: [PATCH 4/5] docs: :bulb: going past DB_MAX_NOTE_TEXT_LENGTH
---
packages/backend/src/misc/hard-limits.ts | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/packages/backend/src/misc/hard-limits.ts b/packages/backend/src/misc/hard-limits.ts
index 62671a706..5ce3e0ac9 100644
--- a/packages/backend/src/misc/hard-limits.ts
+++ b/packages/backend/src/misc/hard-limits.ts
@@ -3,6 +3,11 @@
/**
* Maximum note text length that can be stored in DB.
* Surrogate pairs count as one
+ *
+ * NOTE: this can hypothetically be pushed further
+ * (up to 250000000), but will likely cause truncations
+ * and incompatibilities with other servers,
+ * as well as potential performance issues.
*/
export const DB_MAX_NOTE_TEXT_LENGTH = 100000;
From 17124f38344a484d92f3ee06dc4e320398476afd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E3=82=B3=E3=83=AB=E3=82=BB=E3=83=83=E3=83=88=E5=A7=AB=40?=
=?UTF-8?q?=E3=81=8C=E3=82=93=E3=81=B0=E3=82=89=E3=81=AA=E3=81=84?=
<cgsama@noreply.codeberg.org>
Date: Sat, 15 Jul 2023 22:56:09 +0000
Subject: [PATCH 5/5] =?UTF-8?q?feat:=20=E2=9C=A8=20Add=20media=20to=20Mast?=
=?UTF-8?q?odon=20and=20Calckey=20post=20imports=20(#10496)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
### What does this PR do?
Adding files fields in the export notes option, and corresponding import notes
Current the mastodon import does not import any attachments, this pr will use the "upload from url" feature to include medias if its a valid URL.
There are many way to convert the outbox.json file, can be simple as upload media_attachments to any web hosting and do string replace on the json file.
I also create a tool that upload the tar.gz file with auto convert and host the media as simplify the process at https://tempfile.moegirl.live
Detail example can be found at https://fedi.moegirl.live/notes/9h76gtqnp2gwl5dz
https://r2temp.moegirl.live/2023/7/15/15356683-050f-423a-b331-c9a05561f52a/shana-settings-_-meng-zhai-le-yuan-xyou-yu-ou-xiang-de-luo-ke-ke-wu-yan-moe-otaku-elysian-x-gloomily-idol-s-rococo-luncheon----mozilla-firefox-private-browsing-2023-07-15-18-36-37.mp4
Co-authored-by: CGsama <CGsama@outlook.com>
Reviewed-on: https://codeberg.org/calckey/calckey/pulls/10496
Co-authored-by: コルセット姫@がんばらない <cgsama@noreply.codeberg.org>
Co-committed-by: コルセット姫@がんばらない <cgsama@noreply.codeberg.org>
---
.../src/queue/processors/db/export-notes.ts | 9 +++++----
.../processors/db/import-calckey-post.ts | 19 +++++++++++++++++-
.../queue/processors/db/import-masto-post.ts | 20 ++++++++++++++++++-
3 files changed, 42 insertions(+), 6 deletions(-)
diff --git a/packages/backend/src/queue/processors/db/export-notes.ts b/packages/backend/src/queue/processors/db/export-notes.ts
index de8fac05b..bf53f8360 100644
--- a/packages/backend/src/queue/processors/db/export-notes.ts
+++ b/packages/backend/src/queue/processors/db/export-notes.ts
@@ -4,7 +4,7 @@ import * as fs from "node:fs";
import { queueLogger } from "../../logger.js";
import { addFile } from "@/services/drive/add-file.js";
import { format as dateFormat } from "date-fns";
-import { Users, Notes, Polls } from "@/models/index.js";
+import { Users, Notes, Polls, DriveFiles } from "@/models/index.js";
import { MoreThan } from "typeorm";
import type { Note } from "@/models/entities/note.js";
import type { Poll } from "@/models/entities/poll.js";
@@ -75,7 +75,7 @@ export async function exportNotes(
if (note.hasPoll) {
poll = await Polls.findOneByOrFail({ noteId: note.id });
}
- const content = JSON.stringify(serialize(note, poll));
+ const content = JSON.stringify(await serialize(note, poll));
const isFirst = exportedNotesCount === 0;
await write(isFirst ? content : ",\n" + content);
exportedNotesCount++;
@@ -112,15 +112,16 @@ export async function exportNotes(
done();
}
-function serialize(
+async function serialize(
note: Note,
poll: Poll | null = null,
-): Record<string, unknown> {
+): Promise<Record<string, unknown>> {
return {
id: note.id,
text: note.text,
createdAt: note.createdAt,
fileIds: note.fileIds,
+ files: await DriveFiles.packMany(note.fileIds),
replyId: note.replyId,
renoteId: note.renoteId,
poll: poll,
diff --git a/packages/backend/src/queue/processors/db/import-calckey-post.ts b/packages/backend/src/queue/processors/db/import-calckey-post.ts
index 28e794aa0..945c50d63 100644
--- a/packages/backend/src/queue/processors/db/import-calckey-post.ts
+++ b/packages/backend/src/queue/processors/db/import-calckey-post.ts
@@ -3,6 +3,8 @@ import create from "@/services/note/create.js";
import { Users } from "@/models/index.js";
import type { DbUserImportMastoPostJobData } from "@/queue/types.js";
import { queueLogger } from "../../logger.js";
+import { uploadFromUrl } from "@/services/drive/upload-from-url.js";
+import type { DriveFile } from "@/models/entities/drive-file.js";
import type Bull from "bull";
const logger = queueLogger.createSubLogger("import-calckey-post");
@@ -29,10 +31,25 @@ export async function importCkPost(
done();
return;
}
+ const urls = (post.files || [])
+ .map((x: any) => x.url)
+ .filter((x: String) => x.startsWith("http"));
+ const files: DriveFile[] = [];
+ for (const url of urls) {
+ try {
+ const file = await uploadFromUrl({
+ url: url,
+ user: user,
+ });
+ files.push(file);
+ } catch (e) {
+ logger.error(`Skipped adding file to drive: ${url}`);
+ }
+ }
const { text, cw, localOnly, createdAt } = Post.parse(post);
const note = await create(user, {
createdAt: createdAt,
- files: undefined,
+ files: files.length == 0 ? undefined : files,
poll: undefined,
text: text || undefined,
reply: null,
diff --git a/packages/backend/src/queue/processors/db/import-masto-post.ts b/packages/backend/src/queue/processors/db/import-masto-post.ts
index efa4adf3f..05166b085 100644
--- a/packages/backend/src/queue/processors/db/import-masto-post.ts
+++ b/packages/backend/src/queue/processors/db/import-masto-post.ts
@@ -6,6 +6,8 @@ import type Bull from "bull";
import { htmlToMfm } from "@/remote/activitypub/misc/html-to-mfm.js";
import { resolveNote } from "@/remote/activitypub/models/note.js";
import { Note } from "@/models/entities/note.js";
+import { uploadFromUrl } from "@/services/drive/upload-from-url.js";
+import type { DriveFile } from "@/models/entities/drive-file.js";
const logger = queueLogger.createSubLogger("import-masto-post");
@@ -43,9 +45,25 @@ export async function importMastoPost(
throw e;
}
job.progress(80);
+ const urls = post.object.attachment
+ .map((x: any) => x.url)
+ .filter((x: String) => x.startsWith("http"));
+ const files: DriveFile[] = [];
+ for (const url of urls) {
+ try {
+ const file = await uploadFromUrl({
+ url: url,
+ user: user,
+ });
+ files.push(file);
+ } catch (e) {
+ logger.error(`Skipped adding file to drive: ${url}`);
+ }
+ }
+
const note = await create(user, {
createdAt: new Date(post.object.published),
- files: undefined,
+ files: files.length == 0 ? undefined : files,
poll: undefined,
text: text || undefined,
reply,