From 624f305d9c293d46553c75c5eb1d39e03f520233 Mon Sep 17 00:00:00 2001
From: cutestnekoaqua <waterdev@galaxycrow.de>
Date: Tue, 28 Mar 2023 23:29:47 +0200
Subject: [PATCH 1/3] feat: experimental post import
---
packages/backend/src/misc/post.ts | 14 ++++++
packages/backend/src/queue/index.ts | 17 ++++++++
.../backend/src/queue/processors/db/index.ts | 2 +
packages/backend/src/server/api/endpoints.ts | 2 +
.../server/api/endpoints/i/import-posts.ts | 43 +++++++++++++++++++
.../src/pages/settings/import-export.vue | 10 +++++
6 files changed, 88 insertions(+)
create mode 100644 packages/backend/src/misc/post.ts
create mode 100644 packages/backend/src/server/api/endpoints/i/import-posts.ts
diff --git a/packages/backend/src/misc/post.ts b/packages/backend/src/misc/post.ts
new file mode 100644
index 000000000..e14aa3444
--- /dev/null
+++ b/packages/backend/src/misc/post.ts
@@ -0,0 +1,14 @@
+export type Post = {
+ text: string | null;
+ cw: string | null;
+ localOnly: boolean;
+ createdAt: Date;
+};
+
+export function parse(acct: any): Post {
+ return { text: acct.text, cw: acct.cw, localOnly: acct.localOnly, createdAt: new Date(acct.createdAt) };
+}
+
+export function toJson(acct: Post): string {
+ return { text: acct.text, cw: acct.cw, localOnly: acct.localOnly }.toString();
+}
diff --git a/packages/backend/src/queue/index.ts b/packages/backend/src/queue/index.ts
index c387efe92..d5b42d637 100644
--- a/packages/backend/src/queue/index.ts
+++ b/packages/backend/src/queue/index.ts
@@ -314,6 +314,23 @@ export function createImportFollowingJob(
);
}
+export function createImportPostsJob(
+ user: ThinUser,
+ fileId: DriveFile["id"],
+) {
+ return dbQueue.add(
+ "importPosts",
+ {
+ user: user,
+ fileId: fileId,
+ },
+ {
+ removeOnComplete: true,
+ removeOnFail: true,
+ },
+ );
+}
+
export function createImportMutingJob(user: ThinUser, fileId: DriveFile["id"]) {
return dbQueue.add(
"importMuting",
diff --git a/packages/backend/src/queue/processors/db/index.ts b/packages/backend/src/queue/processors/db/index.ts
index 90173053f..22b55a368 100644
--- a/packages/backend/src/queue/processors/db/index.ts
+++ b/packages/backend/src/queue/processors/db/index.ts
@@ -11,6 +11,7 @@ import { importFollowing } from "./import-following.js";
import { importUserLists } from "./import-user-lists.js";
import { deleteAccount } from "./delete-account.js";
import { importMuting } from "./import-muting.js";
+import { importPosts } from "./import-posts.js";
import { importBlocking } from "./import-blocking.js";
import { importCustomEmojis } from "./import-custom-emojis.js";
@@ -26,6 +27,7 @@ const jobs = {
importMuting,
importBlocking,
importUserLists,
+ importPosts,
importCustomEmojis,
deleteAccount,
} as Record<
diff --git a/packages/backend/src/server/api/endpoints.ts b/packages/backend/src/server/api/endpoints.ts
index ba0e721b9..920f87199 100644
--- a/packages/backend/src/server/api/endpoints.ts
+++ b/packages/backend/src/server/api/endpoints.ts
@@ -182,6 +182,7 @@ import * as ep___i_exportBlocking from "./endpoints/i/export-blocking.js";
import * as ep___i_exportFollowing from "./endpoints/i/export-following.js";
import * as ep___i_exportMute from "./endpoints/i/export-mute.js";
import * as ep___i_exportNotes from "./endpoints/i/export-notes.js";
+import * as ep___i_importPosts from "./endpoints/i/import-posts.js";
import * as ep___i_exportUserLists from "./endpoints/i/export-user-lists.js";
import * as ep___i_favorites from "./endpoints/i/favorites.js";
import * as ep___i_gallery_likes from "./endpoints/i/gallery/likes.js";
@@ -527,6 +528,7 @@ const eps = [
["i/export-following", ep___i_exportFollowing],
["i/export-mute", ep___i_exportMute],
["i/export-notes", ep___i_exportNotes],
+ ["i/import-posts", ep___i_importPosts],
["i/export-user-lists", ep___i_exportUserLists],
["i/favorites", ep___i_favorites],
["i/gallery/likes", ep___i_gallery_likes],
diff --git a/packages/backend/src/server/api/endpoints/i/import-posts.ts b/packages/backend/src/server/api/endpoints/i/import-posts.ts
new file mode 100644
index 000000000..254e37d8a
--- /dev/null
+++ b/packages/backend/src/server/api/endpoints/i/import-posts.ts
@@ -0,0 +1,43 @@
+import define from "../../define.js";
+import { createImportPostsJob } from "@/queue/index.js";
+import { ApiError } from "../../error.js";
+import { DriveFiles } from "@/models/index.js";
+import { DAY } from "@/const.js";
+
+export const meta = {
+ secure: true,
+ requireCredential: true,
+ limit: {
+ duration: DAY,
+ max: 1,
+ },
+ errors: {
+ noSuchFile: {
+ message: "No such file.",
+ code: "NO_SUCH_FILE",
+ id: "e674141e-bd2a-ba85-e616-aefb187c9c2a",
+ },
+
+ emptyFile: {
+ message: "That file is empty.",
+ code: "EMPTY_FILE",
+ id: "d2f12af1-e7b4-feac-86a3-519548f2728e",
+ },
+ },
+} as const;
+
+export const paramDef = {
+ type: "object",
+ properties: {
+ fileId: { type: "string", format: "misskey:id" },
+ },
+ required: ["fileId"],
+} as const;
+
+export default define(meta, paramDef, async (ps, user) => {
+ const file = await DriveFiles.findOneBy({ id: ps.fileId });
+
+ if (file == null) throw new ApiError(meta.errors.noSuchFile);
+ if (file.size === 0) throw new ApiError(meta.errors.emptyFile);
+ createImportPostsJob(user, file.id);
+});
diff --git a/packages/client/src/pages/settings/import-export.vue b/packages/client/src/pages/settings/import-export.vue
index 64c3cb4f9..4822ccff0 100644
--- a/packages/client/src/pages/settings/import-export.vue
+++ b/packages/client/src/pages/settings/import-export.vue
@@ -7,6 +7,11 @@
<template #icon><i class="ph-download-simple ph-bold ph-lg"></i></template>
<MkButton primary :class="$style.button" inline @click="exportNotes()"><i class="ph-download-simple ph-bold ph-lg"></i> {{ i18n.ts.export }}</MkButton>
</FormFolder>
+ <FormFolder class="_formBlock">
+ <template #label>{{ i18n.ts.import }}</template>
+ <template #icon><i class="ph-upload-simple ph-bold ph-lg"></i></template>
+ <MkButton primary :class="$style.button" inline @click="importPosts($event)"><i class="ph-upload-simple ph-bold ph-lg"></i> {{ i18n.ts.import }}</MkButton>
+ </FormFolder>
</FormSection>
<FormSection>
<template #label>{{ i18n.ts._exportOrImport.followingList }}</template>
@@ -108,6 +113,11 @@ const exportNotes = () => {
os.api('i/export-notes', {}).then(onExportSuccess).catch(onError);
};
+const importPosts = async (ev) => {
+ const file = await selectFile(ev.currentTarget ?? ev.target);
+ os.api('i/import-posts', { fileId: file.id }).then(onImportSuccess).catch(onError);
+};
+
const exportFollowing = () => {
os.api('i/export-following', {
excludeMuting: excludeMutingUsers.value,
From c05703001c9bc9808bbc7994ffb97e77a4b389c3 Mon Sep 17 00:00:00 2001
From: cutestnekoaqua <waterdev@galaxycrow.de>
Date: Tue, 28 Mar 2023 23:48:27 +0200
Subject: [PATCH 2/3] why is this git ignored?
---
.../src/queue/processors/db/import-posts.ts | 76 +++++++++++++++++++
1 file changed, 76 insertions(+)
create mode 100644 packages/backend/src/queue/processors/db/import-posts.ts
diff --git a/packages/backend/src/queue/processors/db/import-posts.ts b/packages/backend/src/queue/processors/db/import-posts.ts
new file mode 100644
index 000000000..a5c5c3692
--- /dev/null
+++ b/packages/backend/src/queue/processors/db/import-posts.ts
@@ -0,0 +1,76 @@
+import { IsNull } from "typeorm";
+import follow from "@/services/following/create.js";
+
+import * as Post from "@/misc/posts.js";
+import create from "@/services/note/create.js";
+import { downloadTextFile } from "@/misc/download-text-file.js";
+import { Users, DriveFiles } from "@/models/index.js";
+import type { DbUserImportJobData } from "@/queue/types.js";
+import { queueLogger } from "../../logger.js";
+import type Bull from "bull";
+
+const logger = queueLogger.createSubLogger("import-posts");
+
+export async function importPosts(
+ job: Bull.Job<DbUserImportJobData>,
+ done: any,
+): Promise<void> {
+ logger.info(`Importing following of ${job.data.user.id} ...`);
+
+ const user = await Users.findOneBy({ id: job.data.user.id });
+ if (user == null) {
+ done();
+ return;
+ }
+
+ const file = await DriveFiles.findOneBy({
+ id: job.data.fileId,
+ });
+ if (file == null) {
+ done();
+ return;
+ }
+
+ const csv = await downloadTextFile(file.url);
+
+ let linenum = 0;
+
+ if (file.type.endsWith("json")) {
+ for (const post of JSON.parse(csv)) {
+ try {
+ if (post.replyId != null) {
+ continue;
+ }
+ if (post.renoteId != null) {
+ continue;
+ }
+ if (post.visibility !== "public") continue;
+ const { text, cw, localOnly, createdAt } = Post.parse(post);
+
+ logger.info(`Posting[${linenum}] ...`);
+
+ const note = await create(user, {
+ createdAt: createdAt,
+ files: undefined,
+ poll: undefined,
+ text: text || undefined,
+ reply: null,
+ renote: null,
+ cw: cw,
+ localOnly,
+ visibility: "public",
+ visibleUsers: [],
+ channel: null,
+ apMentions: undefined,
+ apHashtags: undefined,
+ apEmojis: undefined,
+ });
+ } catch (e) {
+ logger.warn(`Error in line:${linenum} ${e}`);
+ }
+ }
+ }
+
+ logger.succ("Imported");
+ done();
+}
From 86e02468ce888ee34e6ebea94376a326f5c3b3a9 Mon Sep 17 00:00:00 2001
From: cutestnekoaqua <waterdev@galaxycrow.de>
Date: Tue, 28 Mar 2023 23:48:27 +0200
Subject: [PATCH 3/3] why is this git ignored?
---
packages/backend/src/queue/processors/db/import-posts.ts | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/packages/backend/src/queue/processors/db/import-posts.ts b/packages/backend/src/queue/processors/db/import-posts.ts
index a5c5c3692..a28350c75 100644
--- a/packages/backend/src/queue/processors/db/import-posts.ts
+++ b/packages/backend/src/queue/processors/db/import-posts.ts
@@ -1,7 +1,7 @@
import { IsNull } from "typeorm";
import follow from "@/services/following/create.js";
-import * as Post from "@/misc/posts.js";
+import * as Post from "@/misc/post.js";
import create from "@/services/note/create.js";
import { downloadTextFile } from "@/misc/download-text-file.js";
import { Users, DriveFiles } from "@/models/index.js";