From baeac362de5ac7c92d63d67a56c74f4ccfdc74d3 Mon Sep 17 00:00:00 2001 From: Skylar Grant Date: Sun, 5 Jan 2025 18:23:54 -0500 Subject: [PATCH] Swap to localLLM --- package-lock.json | 35 ++++++++++++++++++++ package.json | 5 +-- src/data/strings.json | 4 ++- src/modules/VoidLLM.js | 67 ++++++++++++++++++++++++++++++++++++++ src/modules/functions.js | 22 +++++++++++++ src/slash-commands/chat.js | 21 +++++++----- 6 files changed, 142 insertions(+), 12 deletions(-) create mode 100644 src/modules/VoidLLM.js diff --git a/package-lock.json b/package-lock.json index 9781a8f..81606a8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "MIT", "dependencies": { + "axios": "^1.7.9", "discord.js": "^14.9.0", "dotenv": "^16.0.3", "openai": "^4.24.1", @@ -255,6 +256,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/axios": { + "version": "1.7.9", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.9.tgz", + "integrity": "sha512-LhLcE7Hbiryz8oMDdDptSrWowmB4Bl6RCt6sIJKpRB4XtVf0iEgewX3au/pJqm+Py1kCASkb/FFKjxQaLtxJvw==", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/base-64": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz", @@ -480,6 +491,25 @@ "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, + "node_modules/follow-redirects": { + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/for-each": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", @@ -1024,6 +1054,11 @@ "undici-types": "~5.26.4" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, "node_modules/regexp.prototype.flags": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.1.tgz", diff --git a/package.json b/package.json index fc27e07..679ab80 100644 --- a/package.json +++ b/package.json @@ -17,9 +17,10 @@ }, "homepage": "https://git.vfsh.dev/voidf1sh/nodgpt#readme", "dependencies": { + "axios": "^1.7.9", "discord.js": "^14.9.0", "dotenv": "^16.0.3", - "string.prototype.replaceall": "^1.0.7", - "openai": "^4.24.1" + "openai": "^4.24.1", + "string.prototype.replaceall": "^1.0.7" } } diff --git a/src/data/strings.json b/src/data/strings.json index 511d60e..eef58be 100644 --- a/src/data/strings.json +++ b/src/data/strings.json @@ -36,7 +36,9 @@ "chatResCentsPer": 0.2, "chatResUnits": 1000, "imgModel": "dall-e-3", - "imgNumber": 1 + "imgNumber": 1, + "sysPrompt": "You are a Discord bot named Nodbot aka NodGPT. You are a shitposting content serving bot in a server full of degenerates. Respond accordingly. Snark and sass is encouraged.", + "localModel": "llama3.2" }, "temp": {} } \ No newline at end of file diff --git a/src/modules/VoidLLM.js b/src/modules/VoidLLM.js new file mode 100644 index 0000000..260f618 --- /dev/null +++ b/src/modules/VoidLLM.js @@ -0,0 +1,67 @@ +const axios = require('axios'); +const strings = require('../data/strings.json'); +const llmHost = "https://llm.vfsh.me"; +const generateEndpoint = "/api/generate"; +const chatEndpoint = "/api/chat"; + +module.exports = { + Generate: class Generate { + constructor(model) { + this.model = model; + this.sysPrompt = strings.ai.sysPrompt; + } + send(prompt) { + return new Promise((resolve, reject) => { + axios.post(`${llmHost}${generateEndpoint}`, { + model: this.model, + system: this.sysPrompt, + prompt: prompt, + raw: false, + stream: false + }, { + headers: { + 'Content-Type': 'application/json' + } + }).then((res) => { + resolve(res.data); + }).catch((err) => { + reject(err); + }); + }); + } + }, + Chat: class Chat { + constructor(model) { + this.model = model; + this.messages = [ + { + role: "system", + content: strings.ai.sysPrompt + } + ]; + } + send(prompt) { + return new Promise((resolve, reject) => { + // Append the message to the messages array + this.messages.push({ + role: "user", + content: prompt + }); + axios.post(`${llmHost}${chatEndpoint}`, { + model: this.model, + messages: this.messages, + stream: false + }, { + headers: { + 'Content-Type': 'application/json', + } + }).then((res) => { + this.messages.push(res.data.message); + resolve(res.data.message); + }).catch((err) => { + reject(err); + }); + }); + } + } +} \ No newline at end of file diff --git a/src/modules/functions.js b/src/modules/functions.js index f0ed35d..f40162f 100644 --- a/src/modules/functions.js +++ b/src/modules/functions.js @@ -15,6 +15,9 @@ const { EmbedBuilder } = Discord; const OpenAI = require("openai"); const openai = new OpenAI(); +// LocalLLM +const {Chat, Generate} = require("./VoidLLM.js"); + // Various imports from other files const config = require('../data/config.json'); const strings = require('../data/strings.json'); @@ -92,6 +95,13 @@ const functions = { .setFooter({ text: `This prompt used ${usage.tokens} tokens for a cost of ${usage.usdc}ยข. Generated using ${strings.ai.chatModel}` }) return { embeds: [gptEmbed] }; }, + llm(prompt, response) { + const llmEmbed = new EmbedBuilder() + .setAuthor({ name: "NodBot powered by voidGPT", iconURL: "https://assets.vfsh.dev/openai-logos/PNGs/openai-logomark.png" }) + .setDescription(`**Prompt**\n${prompt}\n\n**Response**\n${response}`) + .setFooter({ text: `Generated using ${strings.ai.localModel}` }); + return { embeds: [llmEmbed] }; + }, generatingResponse() { const embed = new EmbedBuilder() .setAuthor({ name: "NodBot powered by OpenAI", iconURL: "https://assets.vfsh.dev/openai-logos/PNGs/openai-logomark.png" }) @@ -135,6 +145,18 @@ const functions = { }); } }, + localLLM: { + async chat(prompt) { + try { + const chat = new Chat(strings.ai.localModel); + const response = await chat.send(prompt); + return response; + } catch (e) { + console.error(e); + return; + } + } + }, async sendHeartbeat(url) { if (isDev) console.log("Heartbeat Sent: " + url); https.get(url, async (response) => { diff --git a/src/slash-commands/chat.js b/src/slash-commands/chat.js index 2b9edd5..18e703a 100644 --- a/src/slash-commands/chat.js +++ b/src/slash-commands/chat.js @@ -15,14 +15,17 @@ module.exports = { await interaction.deferReply(); await interaction.editReply(fn.builders.embeds.generatingResponse()); const userPrompt = interaction.options.getString("prompt"); - const response = await fn.openAI.chatPrompt(userPrompt).catch(e => console.error(e)); - const responseText = response.choices[0].message.content; - const usage = { - tokens: response.usage.total_tokens, - usdc: (response.usage.prompt_tokens * (strings.ai.chatPromptCentsPer / strings.ai.chatPromptUnits)) + - (response.usage.completion_tokens * (strings.ai.chatResCentsPer / strings.ai.chatResUnits)) - }; - const gptEmbed = fn.builders.embeds.gpt(userPrompt, responseText, usage); - await interaction.editReply(gptEmbed); + const response = await fn.localLLM.chat(userPrompt).catch(e => console.error(e)); + const llmEmbed = fn.builders.embeds.llm(userPrompt, response.content); + await interaction.editReply(llmEmbed); + // const response = await fn.openAI.chatPrompt(userPrompt).catch(e => console.error(e)); + // const responseText = response.choices[0].message.content; + // const usage = { + // tokens: response.usage.total_tokens, + // usdc: (response.usage.prompt_tokens * (strings.ai.chatPromptCentsPer / strings.ai.chatPromptUnits)) + + // (response.usage.completion_tokens * (strings.ai.chatResCentsPer / strings.ai.chatResUnits)) + // }; + // const gptEmbed = fn.builders.embeds.gpt(userPrompt, responseText, usage); + // await interaction.editReply(gptEmbed); }, }; \ No newline at end of file