From f55b5579ad1920c093c80cc98f3310bfe75f9cf0 Mon Sep 17 00:00:00 2001 From: fzzinchemical Date: Fri, 28 Mar 2025 16:14:49 +0100 Subject: [PATCH] feat: integrate xml-parser for API response handling and enhance data retrieval methods --- deno.json | 4 + deno.lock | 16 +++ src/plugins/rule34/api.ts | 238 ++++++++++++++++++++++++------------- src/plugins/rule34/test.ts | 14 +-- 4 files changed, 182 insertions(+), 90 deletions(-) diff --git a/deno.json b/deno.json index c621ca1..8bd7004 100755 --- a/deno.json +++ b/deno.json @@ -15,9 +15,13 @@ }, "imports": { "@discordeno": "npm:@discordeno@18.0.1", + "@melvdouc/xml-parser": "jsr:@melvdouc/xml-parser@^0.1.1", "@std/assert": "jsr:@std/assert@1", "@types/node": "npm:@types/node@^22.5.4", "@root/" : "./src/" + }, + "fmt": { + "useTabs": true } } diff --git a/deno.lock b/deno.lock index 16691fb..73c8246 100644 --- a/deno.lock +++ b/deno.lock @@ -1,6 +1,8 @@ { "version": "4", "specifiers": { + "jsr:@melvdouc/xml-parser@*": "0.1.1", + "jsr:@melvdouc/xml-parser@~0.1.1": "0.1.1", "jsr:@std/assert@*": "1.0.12", "jsr:@std/assert@1": "1.0.12", "jsr:@std/dotenv@*": "0.225.2", @@ -12,11 +14,15 @@ "jsr:@std/log@*": "0.224.7", "npm:@discordeno/bot@*": "19.0.0-next.b1bfe94", "npm:@types/node@*": "18.16.19", + "npm:@types/node@^22.5.4": "22.13.13", "npm:discordeno@*": "18.0.1", "npm:discordeno@18.0.1": "18.0.1", "npm:gson@*": "0.1.5" }, "jsr": { + "@melvdouc/xml-parser@0.1.1": { + "integrity": "5c79d37c6471cb74efb344988317270b57b4f181decb873e441453db42eb6e5f" + }, "@std/assert@1.0.12": { "integrity": "08009f0926dda9cbd8bef3a35d3b6a4b964b0ab5c3e140a4e0351fbf34af5b9a", "dependencies": [ @@ -103,6 +109,12 @@ "@types/node@18.16.19": { "integrity": "sha512-IXl7o+R9iti9eBW4Wg2hx1xQDig183jj7YLn8F7udNceyfkbn1ZxmzZXuak20gR40D7pIkIY1kYGx5VIGbaHKA==" }, + "@types/node@22.13.13": { + "integrity": "sha512-ClsL5nMwKaBRwPcCvH8E7+nU4GxHVx1axNvMZTFHMEfNI7oahimt26P5zjVCRrjiIWj6YFXfE1v3dEp94wLcGQ==", + "dependencies": [ + "undici-types" + ] + }, "circularjs@0.1.3": { "integrity": "sha512-RVp6t82JlYMz6CxtGJVoncK7StrDuAIaihiE3dC4T3gE/Pko3ZGxoDibOctjcJKxAIb4C2avHOFA9mDxCKGbbw==", "dependencies": [ @@ -130,6 +142,9 @@ "underscore@1.6.0": { "integrity": "sha512-z4o1fvKUojIWh9XuaVLUDdf86RQiq13AC1dmHbTpoyuu+bquHms76v16CjycCbec87J7z0k//SiQVk0sMdFmpQ==" }, + "undici-types@6.20.0": { + "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==" + }, "undici@5.28.4": { "integrity": "sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==", "dependencies": [ @@ -155,6 +170,7 @@ }, "workspace": { "dependencies": [ + "jsr:@melvdouc/xml-parser@~0.1.1", "jsr:@std/assert@1", "npm:@types/node@^22.5.4" ] diff --git a/src/plugins/rule34/api.ts b/src/plugins/rule34/api.ts index f0e5171..faedeb7 100755 --- a/src/plugins/rule34/api.ts +++ b/src/plugins/rule34/api.ts @@ -1,96 +1,168 @@ -import { assert } from "@std/assert/assert"; +import * as xml_parser from "jsr:@melvdouc/xml-parser"; -type APIResponse = Array<{ - preview_url: string; - sample_url: string; - file_url: string; - directory: number; - hash: string; - width: number; - height: number; - id: number; - image: string; - change: number; - owner: string; - parent_id: number; - rating: string; - sample: boolean; - sample_height: number; - sample_width: number; - score: number; - tags: string; - source: string; - status: string; - has_notes: boolean; - comment_count: number; -}>; +type ImageResponse = { + preview_url: string; + sample_url: string; + file_url: string; + directory: number; + hash: string; + width: number; + height: number; + id: number; + image: string; + change: number; + owner: string; + parent_id: number; + rating: string; + sample: boolean; + sample_height: number; + sample_width: number; + score: number; + tags: string; + source: string; + status: string; + has_notes: boolean; + comment_count: number; +}; + +type CommentResponse = { + created_at: string; + post_id: number; + body: string; + creator: string; + id: number; + creator_id: number; +}; + +type TagResponse = { + type: number; + count: number; + name: string; + ambiguous: boolean; + id: number; +}; // Define the API URL -const apiUrl = - "https://api.rule34.xxx/index.php?page=dapi&s=post&q=index&json=1"; +const baseUrl = "https://api.rule34.xxx"; +const postUrl = `${baseUrl}/index.php?page=dapi&s=post&q=index&json=1`; +const tagUrl = `${baseUrl}/index.php?page=dapi&s=tag&q=index`; +const commentsUrl = `${baseUrl}/index.php?page=dapi&s=comment&q=index`; -// Make a GET request -let baseResponse = await fetch(apiUrl, { headers: { "Accept": "application/json" } }) - .then(async (response) => { - if (!response.ok) { - throw new Error("Network response was not ok"); - } - return await response.json(); - }); -const hyperlinkarray: string[] = []; - -// --------------------------------- -// import * as stdHTML from "jsr:@std/html@1.0.2"; -// Deno.serve({ port: 6969 }, async (request) => { -// const html = hyperlinkarray.map((x) => ``) -// .join("\n"); -// return new Response(html, { -// headers: { "Content-Type": "text/html;charset=utf-8" }, -// }); -// }); -// --------------------------------- - -export async function refresh() { - await fetch(apiUrl, { headers: { "Accept": "application/json" } }) - .then(async (response) => { - if (!response.ok) { - throw new Error("Network response was not ok"); - } - baseResponse = await response.json(); - for (const k of baseResponse) { - if (!hyperlinkarray.includes(k.file_url)) { - hyperlinkarray.push(k.file_url); - } - } - }); +async function requestJSON(URL: string) { + const response = await requestRaw(URL); + return await response.json(); } -for (const k of baseResponse) { - hyperlinkarray.push(k.file_url); +async function requestRaw(URL: string) { + const response = await fetch(URL, { + headers: { "Accept": "application/json" }, + }); + if (!response.ok) { + throw new Error("Network response was not ok"); + } + return response; } -export async function dropRule() { - if (hyperlinkarray.length === 0) { - await refresh(); - return await dropRule(); - } - return hyperlinkarray.pop()!; +//List +async function getPosts(n: number) { + return await requestJSON(`${postUrl}&limit=${n}`); } -export async function dropRule5() { - let tmp = ""; - for (let i = 0; i < 5; i++) { - tmp += await dropRule() + "\n"; - } - return tmp; +async function getPostWithID(id: number) { + return await requestJSON( + `${postUrl}&id=${encodeURIComponent(id)}`, + ); } +async function getPID() { + return await requestJSON(`${postUrl}&pid`); +} -//test refresh here because local stack is not given to other functions -Deno.test("Test Image-Stack refresh", async() => { - await refresh() - const previousStack = [...hyperlinkarray] - await new Promise(r => setTimeout(r, 10000)); - await refresh() - assert(previousStack.length <= hyperlinkarray.length, "Stack-size did not increase as expected!") -}) \ No newline at end of file +async function getTags(tags: string[]) { + const list = tags.join("+"); + return await requestJSON( + `${postUrl}&tags=${encodeURIComponent(list)}`, + ); +} + +// Comments +async function getPostComments(postID: number) { + const response = await requestRaw(`${commentsUrl}&post_id=${postID}`); + return XMLtoGenericDatatypeParser(response, parseComment); +} + +async function getTagByID(id: number) { + const response = await requestRaw(`${tagUrl}&id=${id}`); + return XMLtoGenericDatatypeParser(response, parseTag); +} + +async function getTagList(n: number) { + const response = await requestRaw(`${tagUrl}&limit=${n}`); + return XMLtoGenericDatatypeParser(response, parseTag); +} + +function parseTag(attributes: Record) { + return Object.fromEntries( + Object.entries(attributes).map(([k, v]) => { + if (k === "ambiguous") { + return [k, v === "true"]; + } else if (k === "name") { + return [k, v]; + } else { + return [k, convertStringtoNumber(v)]; + } + }), + ); +} + +function parseComment(attributes: Record) { + return Object.fromEntries( + Object.entries(attributes).map(([k, v]) => { + if (k === "id" || k === "creator_id" || k === "post_id") { + return [k, convertStringtoNumber(v)]; + } else { + return [k, v]; + } + }), + ); +} + +function convertStringtoNumber(str: string) { + return parseFloat(str); +} + +async function XMLtoGenericDatatypeParser( + response: Response, + callback: (attributes: Record) => T, +) { + const parsedResponse = xml_parser.parse(await response.text()); + const stack: T[] = []; + + for (const v of parsedResponse) { + if (v.kind === "REGULAR_TAG_NODE") { + for (const entry of v.children) { + if (entry.kind === "ORPHAN_TAG_NODE") { + if (entry.attributes !== undefined) { + stack.push(callback(entry.attributes)); + } + } + } + } + } + return stack; +} + +Deno.test("Post Comment", async () => { + const response = await getPostComments(1213); + console.debug(response); +}); + +Deno.test("Get Tag by ID", async () => { + const response = await getTagByID(1); + console.debug(response); +}); + +Deno.test("Get Tag List", async () => { + const response = await getTagList(6); + console.debug(response); +}); diff --git a/src/plugins/rule34/test.ts b/src/plugins/rule34/test.ts index 5ec73b2..31203d0 100644 --- a/src/plugins/rule34/test.ts +++ b/src/plugins/rule34/test.ts @@ -1,8 +1,8 @@ -import { assert } from "jsr:@std/assert"; -import { dropRule, refresh } from "@root/plugins/rule34/api.ts"; +// import { assert } from "jsr:@std/assert"; +// import { dropRule, refresh } from "@root/plugins/rule34/api.ts"; -Deno.test("Test Drop", async () => { - await refresh(); - const link = await dropRule(); - assert(link !== "", "Empty String was dropped!"); -}); +// Deno.test("Test Drop", async () => { +// await refresh(); +// const link = await dropRule(); +// assert(link !== "", "Empty String was dropped!"); +// });