From 0ba525c4a5802106d582c75f713728accf2f151a Mon Sep 17 00:00:00 2001 From: GramingFoxTeam Date: Sun, 25 Dec 2022 12:49:16 +0300 Subject: [PATCH] feat: GODEL AI --- bots/discord/events/discord/messageCreate.js | 3 +- bots/discord/events/helper/aiResponse.js | 18 +- bots/discord/package-lock.json | 194 +++++++++---------- bots/discord/package.json | 2 +- bots/reddit/events/helper/aiResponse.js | 24 +-- bots/telegram/events/helper/aiResponse.js | 23 +-- server/PROTOCOL.md | 27 +-- server/config.example.json | 14 +- server/events/addTrainData.js | 22 --- server/events/ai.js | 17 +- server/events/trainAI.js | 54 ------ server/index.js | 21 +- server/package-lock.json | 63 ++++-- server/package.json | 4 +- 14 files changed, 197 insertions(+), 289 deletions(-) delete mode 100644 server/events/addTrainData.js delete mode 100644 server/events/trainAI.js diff --git a/bots/discord/events/discord/messageCreate.js b/bots/discord/events/discord/messageCreate.js index c31c824..66094fc 100644 --- a/bots/discord/events/discord/messageCreate.js +++ b/bots/discord/events/discord/messageCreate.js @@ -5,6 +5,7 @@ export default { once: false, execute(helper, _, msg) { if (!msg.content || msg.author.bot) return; - helper.scanText(msg.content.toLowerCase(), `${msg.channelId}/${msg.id}`); + if (!msg.mentions.has(msg.client.user)) return; + helper.scanText(msg.content.toLowerCase().replace(/<.*?>/g, ''), `${msg.channelId}/${msg.id}`); } }; diff --git a/bots/discord/events/helper/aiResponse.js b/bots/discord/events/helper/aiResponse.js index 1d6355d..958052a 100644 --- a/bots/discord/events/helper/aiResponse.js +++ b/bots/discord/events/helper/aiResponse.js @@ -2,14 +2,9 @@ export default { name: 'aiResponse', once: false, async execute(client, config, aiRes) { - const response = config.responses.find( - (res) => res.label === aiRes.predictions[0].label - ); - if (!response) return; - - if (Number(aiRes.predictions[0].score) >= response.threshold) { - if (!response.responses[0]) return; + if (!aiRes.response) return; + try { const ids = aiRes.id.split('/'); let channel = client.channels.cache.get(ids[0]); @@ -24,12 +19,11 @@ export default { await channel.messages.fetch(ids[1]); message = channel.messages.cache.get(ids[1]); } - - const replyMsg = response.responses.find(res => res.p === 'discord').text; - - message.reply(replyMsg); + + message.reply(aiRes.response); return; - } + } catch (e) {} + } }; diff --git a/bots/discord/package-lock.json b/bots/discord/package-lock.json index 76b7a0b..de01699 100644 --- a/bots/discord/package-lock.json +++ b/bots/discord/package-lock.json @@ -9,46 +9,46 @@ "version": "1.0.0", "license": "GPL-3.0-or-later", "dependencies": { - "discord.js": "^14.6.0" + "discord.js": "^14.7.1" } }, "node_modules/@discordjs/builders": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@discordjs/builders/-/builders-1.3.0.tgz", - "integrity": "sha512-Pvca6Nw8Hp+n3N+Wp17xjygXmMvggbh5ywUsOYE2Et4xkwwVRwgzxDJiMUuYapPtnYt4w/8aKlf5khc8ipLvhg==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@discordjs/builders/-/builders-1.4.0.tgz", + "integrity": "sha512-nEeTCheTTDw5kO93faM1j8ZJPonAX86qpq/QVoznnSa8WWcCgJpjlu6GylfINTDW6o7zZY0my2SYdxx2mfNwGA==", "dependencies": { "@discordjs/util": "^0.1.0", - "@sapphire/shapeshift": "^3.7.0", - "discord-api-types": "^0.37.12", + "@sapphire/shapeshift": "^3.7.1", + "discord-api-types": "^0.37.20", "fast-deep-equal": "^3.1.3", - "ts-mixer": "^6.0.1", - "tslib": "^2.4.0" + "ts-mixer": "^6.0.2", + "tslib": "^2.4.1" }, "engines": { "node": ">=16.9.0" } }, "node_modules/@discordjs/collection": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@discordjs/collection/-/collection-1.2.0.tgz", - "integrity": "sha512-VvrrtGb7vbfPHzbhGq9qZB5o8FOB+kfazrxdt0OtxzSkoBuw9dURMkCwWizZ00+rDpiK2HmLHBZX+y6JsG9khw==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@discordjs/collection/-/collection-1.3.0.tgz", + "integrity": "sha512-ylt2NyZ77bJbRij4h9u/wVy7qYw/aDqQLWnadjvDqW/WoWCxrsX6M3CIw9GVP5xcGCDxsrKj5e0r5evuFYwrKg==", "engines": { "node": ">=16.9.0" } }, "node_modules/@discordjs/rest": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@discordjs/rest/-/rest-1.3.0.tgz", - "integrity": "sha512-U6X5J+r/MxYpPTlHFuPxXEf92aKsBaD2teBC7sWkKILIr30O8c9+XshfL7KFBCavnAqS/qE+PF9fgRilO3N44g==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@discordjs/rest/-/rest-1.5.0.tgz", + "integrity": "sha512-lXgNFqHnbmzp5u81W0+frdXN6Etf4EUi8FAPcWpSykKd8hmlWh1xy6BmE0bsJypU1pxohaA8lQCgp70NUI3uzA==", "dependencies": { - "@discordjs/collection": "^1.2.0", + "@discordjs/collection": "^1.3.0", "@discordjs/util": "^0.1.0", "@sapphire/async-queue": "^1.5.0", "@sapphire/snowflake": "^3.2.2", - "discord-api-types": "^0.37.12", + "discord-api-types": "^0.37.23", "file-type": "^18.0.0", - "tslib": "^2.4.0", - "undici": "^5.11.0" + "tslib": "^2.4.1", + "undici": "^5.13.0" }, "engines": { "node": ">=16.9.0" @@ -72,12 +72,12 @@ } }, "node_modules/@sapphire/shapeshift": { - "version": "3.7.0", - "resolved": "https://registry.npmjs.org/@sapphire/shapeshift/-/shapeshift-3.7.0.tgz", - "integrity": "sha512-A6vI1zJoxhjWo4grsxpBRBgk96SqSdjLX5WlzKp9H+bJbkM07mvwcbtbVAmUZHbi/OG3HLfiZ1rlw4BhH6tsBQ==", + "version": "3.8.1", + "resolved": "https://registry.npmjs.org/@sapphire/shapeshift/-/shapeshift-3.8.1.tgz", + "integrity": "sha512-xG1oXXBhCjPKbxrRTlox9ddaZTvVpOhYLmKmApD/vIWOV1xEYXnpoFs68zHIZBGbqztq6FrUPNPerIrO1Hqeaw==", "dependencies": { "fast-deep-equal": "^3.1.3", - "lodash.uniqwith": "^4.5.0" + "lodash": "^4.17.21" }, "engines": { "node": ">=v14.0.0", @@ -85,9 +85,9 @@ } }, "node_modules/@sapphire/snowflake": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/@sapphire/snowflake/-/snowflake-3.2.2.tgz", - "integrity": "sha512-ula2O0kpSZtX9rKXNeQMrHwNd7E4jPDJYUXmEGTFdMRfyfMw+FPyh04oKMjAiDuOi64bYgVkOV3MjK+loImFhQ==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@sapphire/snowflake/-/snowflake-3.3.0.tgz", + "integrity": "sha512-Hec5N6zEkZuZFLybVKyLFLlcSgYmR6C1/+9NkIhxPwOf6tgX52ndJCSz8ADejmbrNE0VuNCNkpzhRZzenEC9vA==", "engines": { "node": ">=v14.0.0", "npm": ">=7.0.0" @@ -99,9 +99,9 @@ "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==" }, "node_modules/@types/node": { - "version": "18.11.9", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.9.tgz", - "integrity": "sha512-CRpX21/kGdzjOpFsZSkcrXMGIBWMGNIHXXBVFSH+ggkftxg+XYP20TESbh+zFvFj3EQOl5byk0HTRn1IL6hbqg==" + "version": "18.11.17", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.17.tgz", + "integrity": "sha512-HJSUJmni4BeDHhfzn6nF0sVmd1SMezP7/4F0Lq+aXzmp2xm9O7WXrUtHW/CHlYVtZUbByEvWidHqRtcJXGF2Ng==" }, "node_modules/@types/ws": { "version": "8.5.3", @@ -123,27 +123,27 @@ } }, "node_modules/discord-api-types": { - "version": "0.37.18", - "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.37.18.tgz", - "integrity": "sha512-mJ+9C8gmG5csssVZPH06Y8IGiJykljFyZc6n6F+T3vKo6yNBI5TtLIbwt6t9hJzsR5f1ITzRZ6cuPrTvRCUxqA==" + "version": "0.37.24", + "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.37.24.tgz", + "integrity": "sha512-1+Fb4huJCihdbkJLcq2p7nBmtlmAryNwjefT8wwJnL8c7bc7WA87Oaa5mbLe96QvZyfwnwRCDX40H0HhcVV50g==" }, "node_modules/discord.js": { - "version": "14.6.0", - "resolved": "https://registry.npmjs.org/discord.js/-/discord.js-14.6.0.tgz", - "integrity": "sha512-On1K7xpJZRe0KsziIaDih2ksYPhgxym/ZqV45i1f3yig4vUotikqs7qp5oXiTzQ/UTiNRCixUWFTh7vA1YBCqw==", + "version": "14.7.1", + "resolved": "https://registry.npmjs.org/discord.js/-/discord.js-14.7.1.tgz", + "integrity": "sha512-1FECvqJJjjeYcjSm0IGMnPxLqja/pmG1B0W2l3lUY2Gi4KXiyTeQmU1IxWcbXHn2k+ytP587mMWqva2IA87EbA==", "dependencies": { - "@discordjs/builders": "^1.3.0", - "@discordjs/collection": "^1.2.0", - "@discordjs/rest": "^1.3.0", + "@discordjs/builders": "^1.4.0", + "@discordjs/collection": "^1.3.0", + "@discordjs/rest": "^1.4.0", "@discordjs/util": "^0.1.0", "@sapphire/snowflake": "^3.2.2", "@types/ws": "^8.5.3", - "discord-api-types": "^0.37.12", + "discord-api-types": "^0.37.20", "fast-deep-equal": "^3.1.3", "lodash.snakecase": "^4.1.1", - "tslib": "^2.4.0", - "undici": "^5.11.0", - "ws": "^8.9.0" + "tslib": "^2.4.1", + "undici": "^5.13.0", + "ws": "^8.11.0" }, "engines": { "node": ">=16.9.0" @@ -194,16 +194,16 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, "node_modules/lodash.snakecase": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/lodash.snakecase/-/lodash.snakecase-4.1.1.tgz", "integrity": "sha512-QZ1d4xoBHYUeuouhEq3lk3Uq7ldgyFXGBhg04+oRLnIz8o9T65Eh+8YdroUwn846zchkA9yDsDl5CVVaV2nqYw==" }, - "node_modules/lodash.uniqwith": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/lodash.uniqwith/-/lodash.uniqwith-4.5.0.tgz", - "integrity": "sha512-7lYL8bLopMoy4CTICbxygAUq6CdRJ36vFc80DucPueUee+d5NBRxz3FdT9Pes/HEx5mPoT9jwnsEJWz1N7uq7Q==" - }, "node_modules/peek-readable": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-5.0.0.tgz", @@ -322,9 +322,9 @@ "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" }, "node_modules/undici": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.12.0.tgz", - "integrity": "sha512-zMLamCG62PGjd9HHMpo05bSLvvwWOZgGeiWlN/vlqu3+lRo3elxktVGEyLMX+IO7c2eflLjcW74AlkhEZm15mg==", + "version": "5.14.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.14.0.tgz", + "integrity": "sha512-yJlHYw6yXPPsuOH0x2Ib1Km61vu4hLiRRQoafs+WUgX1vO64vgnxiCEN9dpIrhZyHFsai3F0AEj4P9zy19enEQ==", "dependencies": { "busboy": "^1.6.0" }, @@ -360,36 +360,36 @@ }, "dependencies": { "@discordjs/builders": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@discordjs/builders/-/builders-1.3.0.tgz", - "integrity": "sha512-Pvca6Nw8Hp+n3N+Wp17xjygXmMvggbh5ywUsOYE2Et4xkwwVRwgzxDJiMUuYapPtnYt4w/8aKlf5khc8ipLvhg==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@discordjs/builders/-/builders-1.4.0.tgz", + "integrity": "sha512-nEeTCheTTDw5kO93faM1j8ZJPonAX86qpq/QVoznnSa8WWcCgJpjlu6GylfINTDW6o7zZY0my2SYdxx2mfNwGA==", "requires": { "@discordjs/util": "^0.1.0", - "@sapphire/shapeshift": "^3.7.0", - "discord-api-types": "^0.37.12", + "@sapphire/shapeshift": "^3.7.1", + "discord-api-types": "^0.37.20", "fast-deep-equal": "^3.1.3", - "ts-mixer": "^6.0.1", - "tslib": "^2.4.0" + "ts-mixer": "^6.0.2", + "tslib": "^2.4.1" } }, "@discordjs/collection": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@discordjs/collection/-/collection-1.2.0.tgz", - "integrity": "sha512-VvrrtGb7vbfPHzbhGq9qZB5o8FOB+kfazrxdt0OtxzSkoBuw9dURMkCwWizZ00+rDpiK2HmLHBZX+y6JsG9khw==" + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@discordjs/collection/-/collection-1.3.0.tgz", + "integrity": "sha512-ylt2NyZ77bJbRij4h9u/wVy7qYw/aDqQLWnadjvDqW/WoWCxrsX6M3CIw9GVP5xcGCDxsrKj5e0r5evuFYwrKg==" }, "@discordjs/rest": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@discordjs/rest/-/rest-1.3.0.tgz", - "integrity": "sha512-U6X5J+r/MxYpPTlHFuPxXEf92aKsBaD2teBC7sWkKILIr30O8c9+XshfL7KFBCavnAqS/qE+PF9fgRilO3N44g==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@discordjs/rest/-/rest-1.5.0.tgz", + "integrity": "sha512-lXgNFqHnbmzp5u81W0+frdXN6Etf4EUi8FAPcWpSykKd8hmlWh1xy6BmE0bsJypU1pxohaA8lQCgp70NUI3uzA==", "requires": { - "@discordjs/collection": "^1.2.0", + "@discordjs/collection": "^1.3.0", "@discordjs/util": "^0.1.0", "@sapphire/async-queue": "^1.5.0", "@sapphire/snowflake": "^3.2.2", - "discord-api-types": "^0.37.12", + "discord-api-types": "^0.37.23", "file-type": "^18.0.0", - "tslib": "^2.4.0", - "undici": "^5.11.0" + "tslib": "^2.4.1", + "undici": "^5.13.0" } }, "@discordjs/util": { @@ -403,18 +403,18 @@ "integrity": "sha512-JkLdIsP8fPAdh9ZZjrbHWR/+mZj0wvKS5ICibcLrRI1j84UmLMshx5n9QmL8b95d4onJ2xxiyugTgSAX7AalmA==" }, "@sapphire/shapeshift": { - "version": "3.7.0", - "resolved": "https://registry.npmjs.org/@sapphire/shapeshift/-/shapeshift-3.7.0.tgz", - "integrity": "sha512-A6vI1zJoxhjWo4grsxpBRBgk96SqSdjLX5WlzKp9H+bJbkM07mvwcbtbVAmUZHbi/OG3HLfiZ1rlw4BhH6tsBQ==", + "version": "3.8.1", + "resolved": "https://registry.npmjs.org/@sapphire/shapeshift/-/shapeshift-3.8.1.tgz", + "integrity": "sha512-xG1oXXBhCjPKbxrRTlox9ddaZTvVpOhYLmKmApD/vIWOV1xEYXnpoFs68zHIZBGbqztq6FrUPNPerIrO1Hqeaw==", "requires": { "fast-deep-equal": "^3.1.3", - "lodash.uniqwith": "^4.5.0" + "lodash": "^4.17.21" } }, "@sapphire/snowflake": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/@sapphire/snowflake/-/snowflake-3.2.2.tgz", - "integrity": "sha512-ula2O0kpSZtX9rKXNeQMrHwNd7E4jPDJYUXmEGTFdMRfyfMw+FPyh04oKMjAiDuOi64bYgVkOV3MjK+loImFhQ==" + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@sapphire/snowflake/-/snowflake-3.3.0.tgz", + "integrity": "sha512-Hec5N6zEkZuZFLybVKyLFLlcSgYmR6C1/+9NkIhxPwOf6tgX52ndJCSz8ADejmbrNE0VuNCNkpzhRZzenEC9vA==" }, "@tokenizer/token": { "version": "0.3.0", @@ -422,9 +422,9 @@ "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==" }, "@types/node": { - "version": "18.11.9", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.9.tgz", - "integrity": "sha512-CRpX21/kGdzjOpFsZSkcrXMGIBWMGNIHXXBVFSH+ggkftxg+XYP20TESbh+zFvFj3EQOl5byk0HTRn1IL6hbqg==" + "version": "18.11.17", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.17.tgz", + "integrity": "sha512-HJSUJmni4BeDHhfzn6nF0sVmd1SMezP7/4F0Lq+aXzmp2xm9O7WXrUtHW/CHlYVtZUbByEvWidHqRtcJXGF2Ng==" }, "@types/ws": { "version": "8.5.3", @@ -443,27 +443,27 @@ } }, "discord-api-types": { - "version": "0.37.18", - "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.37.18.tgz", - "integrity": "sha512-mJ+9C8gmG5csssVZPH06Y8IGiJykljFyZc6n6F+T3vKo6yNBI5TtLIbwt6t9hJzsR5f1ITzRZ6cuPrTvRCUxqA==" + "version": "0.37.24", + "resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.37.24.tgz", + "integrity": "sha512-1+Fb4huJCihdbkJLcq2p7nBmtlmAryNwjefT8wwJnL8c7bc7WA87Oaa5mbLe96QvZyfwnwRCDX40H0HhcVV50g==" }, "discord.js": { - "version": "14.6.0", - "resolved": "https://registry.npmjs.org/discord.js/-/discord.js-14.6.0.tgz", - "integrity": "sha512-On1K7xpJZRe0KsziIaDih2ksYPhgxym/ZqV45i1f3yig4vUotikqs7qp5oXiTzQ/UTiNRCixUWFTh7vA1YBCqw==", + "version": "14.7.1", + "resolved": "https://registry.npmjs.org/discord.js/-/discord.js-14.7.1.tgz", + "integrity": "sha512-1FECvqJJjjeYcjSm0IGMnPxLqja/pmG1B0W2l3lUY2Gi4KXiyTeQmU1IxWcbXHn2k+ytP587mMWqva2IA87EbA==", "requires": { - "@discordjs/builders": "^1.3.0", - "@discordjs/collection": "^1.2.0", - "@discordjs/rest": "^1.3.0", + "@discordjs/builders": "^1.4.0", + "@discordjs/collection": "^1.3.0", + "@discordjs/rest": "^1.4.0", "@discordjs/util": "^0.1.0", "@sapphire/snowflake": "^3.2.2", "@types/ws": "^8.5.3", - "discord-api-types": "^0.37.12", + "discord-api-types": "^0.37.20", "fast-deep-equal": "^3.1.3", "lodash.snakecase": "^4.1.1", - "tslib": "^2.4.0", - "undici": "^5.11.0", - "ws": "^8.9.0" + "tslib": "^2.4.1", + "undici": "^5.13.0", + "ws": "^8.11.0" } }, "fast-deep-equal": { @@ -491,16 +491,16 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, "lodash.snakecase": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/lodash.snakecase/-/lodash.snakecase-4.1.1.tgz", "integrity": "sha512-QZ1d4xoBHYUeuouhEq3lk3Uq7ldgyFXGBhg04+oRLnIz8o9T65Eh+8YdroUwn846zchkA9yDsDl5CVVaV2nqYw==" }, - "lodash.uniqwith": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/lodash.uniqwith/-/lodash.uniqwith-4.5.0.tgz", - "integrity": "sha512-7lYL8bLopMoy4CTICbxygAUq6CdRJ36vFc80DucPueUee+d5NBRxz3FdT9Pes/HEx5mPoT9jwnsEJWz1N7uq7Q==" - }, "peek-readable": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-5.0.0.tgz", @@ -571,9 +571,9 @@ "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" }, "undici": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.12.0.tgz", - "integrity": "sha512-zMLamCG62PGjd9HHMpo05bSLvvwWOZgGeiWlN/vlqu3+lRo3elxktVGEyLMX+IO7c2eflLjcW74AlkhEZm15mg==", + "version": "5.14.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.14.0.tgz", + "integrity": "sha512-yJlHYw6yXPPsuOH0x2Ib1Km61vu4hLiRRQoafs+WUgX1vO64vgnxiCEN9dpIrhZyHFsai3F0AEj4P9zy19enEQ==", "requires": { "busboy": "^1.6.0" } diff --git a/bots/discord/package.json b/bots/discord/package.json index ccf4af1..1e6811f 100644 --- a/bots/discord/package.json +++ b/bots/discord/package.json @@ -11,6 +11,6 @@ "author": "Reis Can", "license": "GPL-3.0-or-later", "dependencies": { - "discord.js": "^14.6.0" + "discord.js": "^14.7.1" } } diff --git a/bots/reddit/events/helper/aiResponse.js b/bots/reddit/events/helper/aiResponse.js index b938c4d..eb6dd8b 100644 --- a/bots/reddit/events/helper/aiResponse.js +++ b/bots/reddit/events/helper/aiResponse.js @@ -2,31 +2,21 @@ export default { name: 'aiResponse', once: false, async execute(client, config, aiRes) { - const response = config.responses.find( - (res) => res.label === aiRes.predictions[0].label - ); - if (!response) return; + const ids = aiRes.id.split('/'); - if (Number(aiRes.predictions[0].score) >= response.threshold) { - const ids = aiRes.id.split('/'); - - if (!response.responses[0]) return; - - const replyMsg = response.responses.find(res => res.p === 'reddit').text; - - switch (ids[0]) { + switch (ids[0]) { case 'comment': { - client.getComment(ids[1]).reply(replyMsg); + client.getComment(ids[1]).reply(aiRes.response); break; } case 'post': { - client.getSubmission(ids[1]).reply(replyMsg); + client.getSubmission(ids[1]).reply(aiRes.response); break; } - } - - return; } + + return; + } }; diff --git a/bots/telegram/events/helper/aiResponse.js b/bots/telegram/events/helper/aiResponse.js index 07fa9e1..8aa49b2 100644 --- a/bots/telegram/events/helper/aiResponse.js +++ b/bots/telegram/events/helper/aiResponse.js @@ -2,24 +2,13 @@ export default { name: 'aiResponse', once: false, async execute(bot, config, aiRes) { - const response = config.responses.find( - (res) => res.label === aiRes.predictions[0].label - ); - if (!response) return; + const ids = aiRes.id.split('/'); - if (Number(aiRes.predictions[0].score) >= response.threshold) { - const ids = aiRes.id.split('/'); + bot.sendMessage(ids[0], aiRes.response, { + message_thread_id: ids[1], + reply_to_message_id: ids[2] + }); - if (!response.responses[0]) return; - - const replyMsg = response.responses.find(res => res.p === 'telegram').text; - - bot.sendMessage(ids[0], replyMsg, { - message_thread_id: ids[1], - reply_to_message_id: ids[2] - }); - - return; - } + return; } }; diff --git a/server/PROTOCOL.md b/server/PROTOCOL.md index c51d3c9..18a5d60 100644 --- a/server/PROTOCOL.md +++ b/server/PROTOCOL.md @@ -20,32 +20,7 @@ And the server would return something like this: { "op": 2, "id": "String", - "predictions": [ - { - "label": "DOWNLOAD", - "score": "1" - } - ] -} -``` - -# Training the AI - -To add data to the train data, send a BSON (JSON) like this: - -```json -{ - "op": 3, - "label": "FALSEPOSITIVE", - "text": "how" -} -``` - -To train the AI and to re-load it, send this BSON (JSON): - -```json -{ - "event": 4 + "response": "I think the term afn is just a generic slang term for the app that allows you to modify the behavior of Dalvik based android application..." } ``` diff --git a/server/config.example.json b/server/config.example.json index 56cc7df..d4509e6 100644 --- a/server/config.example.json +++ b/server/config.example.json @@ -2,11 +2,13 @@ "server": { "port": 3000 }, - - "fasttext": { - "bin": "./model/fastText/fasttext", - "loadModel": "./model/model.bin", - "trainFile": "./model/train.tsv", - "debug": true + + "transformers": { + "model": "./model.onnx", + "tokenizer": "./tokenizer.json", + "instruction": "Instruction: given a dialog context and related knowledge, you need to answer the question based on the knowledge.", + "knowledge": [ + "ReVanced is a generic patcher that allows you to modify the behavior of any Dalvik based Android application" + ] } } diff --git a/server/events/addTrainData.js b/server/events/addTrainData.js deleted file mode 100644 index f7c6f5e..0000000 --- a/server/events/addTrainData.js +++ /dev/null @@ -1,22 +0,0 @@ -import { readFileSync, writeFileSync } from 'node:fs'; -import { join } from 'node:path'; - -export default function addTrainData(eventData, __dirname, config) { - const file = readFileSync( - join(__dirname, config.fasttext.trainFile), - 'utf-8' - ); - const { label, text } = eventData; - - if (file.includes(text)) return; - - const data = file.split('\n'); - - const labelIndex = data.findIndex((data) => data.startsWith(label)); - - data.splice(labelIndex === -1 ? 0 : labelIndex, 0, `${label} ${text}`); - - writeFileSync(join(__dirname, config.fasttext.trainFile), data.join('\n')); - - return; -} diff --git a/server/events/ai.js b/server/events/ai.js index 671dff5..8d154f4 100644 --- a/server/events/ai.js +++ b/server/events/ai.js @@ -1,13 +1,24 @@ import { serialize } from 'bson'; -export default async function runAI(client, data, ft) { - const predictions = await ft.predict(data.text); +async function generateResponse(tokenizer, model, config, dialog) { + const knowledge = `[KNOWLEDGE] ${config.knowledge.join(' ')}`; + const context = `[CONTEXT] ${dialog.substring(0, 64)}`; + + const query = `${config.instruction} ${context} ${knowledge}`; + + const inputTokenIds = tokenizer.encode(query); + const outputTokenIds = await model.generate(inputTokenIds, { maxLength: 64, topK: 10 }); + return await tokenizer.decode(outputTokenIds, true); +} + +export default async function runAI(client, data, tokenizer, model, config) { + const response = await generateResponse(tokenizer, model, config, data.text); client.write( serialize({ op: 2, id: data.id, - predictions + response }) ); diff --git a/server/events/trainAI.js b/server/events/trainAI.js deleted file mode 100644 index 05e23e4..0000000 --- a/server/events/trainAI.js +++ /dev/null @@ -1,54 +0,0 @@ -import FastText from 'fasttext.js'; -import { join } from 'node:path'; - -export default async function trainAI(ftext, __dirname, config) { - const ft = new FastText({ - train: { - // number of concurrent threads - thread: 8, - // verbosity level [2] - verbose: 4, - // number of negatives sampled [5] - neg: 7, - // loss function {ns, hs, softmax} [ns] - loss: 'ns', - // learning rate [0.05] - lr: 1, - // change the rate of updates for the learning rate [100] - lrUpdateRate: 1000, - // max length of word ngram [1] - wordNgrams: 5, - // minimal number of word occurences - minCount: 1, - // minimal number of word occurences - minCountLabel: 1, - // size of word vectors [100] - dim: 100, - // size of the context window [5] - ws: 5, - // number of epochs [5] - epoch: 20, - // number of buckets [2000000] - bucket: 2000000, - // min length of char ngram [3] - minn: process.env.TRAIN_MINN || 3, - // max length of char ngram [6] - maxn: process.env.TRAIN_MAXN || 6, - // sampling threshold [0.0001] - t: 0.0001, - // load pre trained word vectors from unsupervised model - pretrainedVectors: '' - }, - serializeTo: join(__dirname, config.fasttext.loadModel).replace('.bin', ''), - trainFile: join(__dirname, config.fasttext.trainFile), - bin: join(__dirname, config.fasttext.bin) - }); - - ftext.unload(); - - await ft.train(); - - ftext.load(); - - return; -} diff --git a/server/index.js b/server/index.js index a50ff93..bdb02d2 100644 --- a/server/index.js +++ b/server/index.js @@ -10,12 +10,11 @@ const config = JSON.parse(readFileSync('./config.json', 'utf-8')); import { createServer } from 'node:net'; import { deserialize } from 'bson'; -import FastText from 'fasttext.js'; -import { runAI, trainAI, runOCR, addTrainData } from './events/index.js'; +import transformers from 'transformers-nodejs'; +import { runAI, runOCR } from './events/index.js'; -const ft = new FastText(config.fasttext); - -ft.load(); +const tokenizer = await transformers.AutoTokenizer.fromPretrained(config.transformers.tokenizer); +const model = await transformers.AutoModelForSeq2SeqLM.fromPretrained(config.transformers.model); const server = createServer(async (client) => { client.on('data', async (data) => { @@ -25,17 +24,7 @@ const server = createServer(async (client) => { switch (eventData.op) { case 1: { - runAI(client, eventData, ft); - break; - } - - case 3: { - addTrainData(eventData, __dirname, config); - break; - } - - case 4: { - trainAI(ft, __dirname, config); + runAI(client, eventData, tokenizer, model, config.transformers); break; } diff --git a/server/package-lock.json b/server/package-lock.json index 435d9b9..d0fc590 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -10,8 +10,8 @@ "license": "GPL-3.0-or-later", "dependencies": { "bson": "^4.7.0", - "fasttext.js": "^1.1.2", - "node-tesseract-ocr": "^2.2.1" + "node-tesseract-ocr": "^2.2.1", + "transformers-nodejs": "github:reisxd/transformers-nodejs" } }, "node_modules/base64-js": { @@ -67,14 +67,6 @@ "ieee754": "^1.1.13" } }, - "node_modules/fasttext.js": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/fasttext.js/-/fasttext.js-1.1.2.tgz", - "integrity": "sha512-ZdH7lJzOlHd2KfT5sr3Z88EQDiEKQzxt+v2J7t1ZVZZhmtM5XZtCQNVDD/wWRNl5zEjv8LBrBoRGTIpcMX52Vw==", - "engines": { - "node": "*" - } - }, "node_modules/ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -101,6 +93,32 @@ "engines": { "node": ">=10" } + }, + "node_modules/onnxruntime-common": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.13.1.tgz", + "integrity": "sha512-HtfNeuZ8p47avtZR2mGldXA728keJj46w/6yy5DQTLA9zmpjXF/iJdS8f9yAtyWbSGw3s95DfBPWw2myhA1r0A==" + }, + "node_modules/onnxruntime-node": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.13.1.tgz", + "integrity": "sha512-HHwxs13dSCJX+iqr90p8WGlG54XTxNG6lKXhUC4xu4r95GFjLaA6JxNUjeuGQ6T6E06hCy61lZMdH5MTcBNLTA==", + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "onnxruntime-common": "~1.13.1" + } + }, + "node_modules/transformers-nodejs": { + "version": "1.0.0", + "resolved": "git+ssh://git@github.com/reisxd/transformers-nodejs.git#28e591496a7a231b0ad8dc62081365d4077638f9", + "license": "GPL-3.0-or-later", + "dependencies": { + "onnxruntime-node": "^1.13.1" + } } }, "dependencies": { @@ -126,11 +144,6 @@ "ieee754": "^1.1.13" } }, - "fasttext.js": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/fasttext.js/-/fasttext.js-1.1.2.tgz", - "integrity": "sha512-ZdH7lJzOlHd2KfT5sr3Z88EQDiEKQzxt+v2J7t1ZVZZhmtM5XZtCQNVDD/wWRNl5zEjv8LBrBoRGTIpcMX52Vw==" - }, "ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -140,6 +153,26 @@ "version": "2.2.1", "resolved": "https://registry.npmjs.org/node-tesseract-ocr/-/node-tesseract-ocr-2.2.1.tgz", "integrity": "sha512-Q9cD79JGpPNQBxbi1fV+OAsTxYKLpx22sagsxSyKbu1u+t6UarApf5m32uVc8a5QAP1Wk7fIPN0aJFGGEE9DyQ==" + }, + "onnxruntime-common": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.13.1.tgz", + "integrity": "sha512-HtfNeuZ8p47avtZR2mGldXA728keJj46w/6yy5DQTLA9zmpjXF/iJdS8f9yAtyWbSGw3s95DfBPWw2myhA1r0A==" + }, + "onnxruntime-node": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.13.1.tgz", + "integrity": "sha512-HHwxs13dSCJX+iqr90p8WGlG54XTxNG6lKXhUC4xu4r95GFjLaA6JxNUjeuGQ6T6E06hCy61lZMdH5MTcBNLTA==", + "requires": { + "onnxruntime-common": "~1.13.1" + } + }, + "transformers-nodejs": { + "version": "git+ssh://git@github.com/reisxd/transformers-nodejs.git#28e591496a7a231b0ad8dc62081365d4077638f9", + "from": "transformers-nodejs@https://github.com/reisxd/transformers-nodejs", + "requires": { + "onnxruntime-node": "^1.13.1" + } } } } diff --git a/server/package.json b/server/package.json index d2095d5..eff9968 100644 --- a/server/package.json +++ b/server/package.json @@ -8,7 +8,7 @@ "license": "GPL-3.0-or-later", "dependencies": { "bson": "^4.7.0", - "fasttext.js": "^1.1.2", - "node-tesseract-ocr": "^2.2.1" + "node-tesseract-ocr": "^2.2.1", + "transformers-nodejs": "github:reisxd/transformers-nodejs" } }