From 9962e4fe68ca49c87ab35cfad3259a5e807fc7d4 Mon Sep 17 00:00:00 2001 From: reis Date: Thu, 10 Nov 2022 13:55:33 +0000 Subject: [PATCH] Add Server --- .gitignore | 3 + package.json | 15 ++++ server/PROTOCOL.md | 54 +++++++++++++++ server/events/ai.js | 16 +++++ server/events/index.js | 7 ++ server/events/trainAI.js | 51 ++++++++++++++ server/index.js | 31 +++++++++ server/package-lock.json | 145 +++++++++++++++++++++++++++++++++++++++ server/package.json | 14 ++++ 9 files changed, 336 insertions(+) create mode 100644 .gitignore create mode 100644 package.json create mode 100644 server/PROTOCOL.md create mode 100644 server/events/ai.js create mode 100644 server/events/index.js create mode 100644 server/events/trainAI.js create mode 100644 server/index.js create mode 100644 server/package-lock.json create mode 100644 server/package.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9d4c99f --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +node_modules +model +test.js \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..418d51a --- /dev/null +++ b/package.json @@ -0,0 +1,15 @@ +{ + "name": "revanced-helper", + "version": "0.0.1", + "description": "The community bots (and the server) to help the community and make moderators job easy.", + "repository": { + "type": "git", + "url": "git+https://github.com/reisxd/revanced-helper.git" + }, + "author": "Reis Can", + "license": "GPL-3.0-or-later", + "bugs": { + "url": "https://github.com/reisxd/revanced-helper/issues" + }, + "homepage": "https://github.com/reisxd/revanced-helper#readme" +} diff --git a/server/PROTOCOL.md b/server/PROTOCOL.md new file mode 100644 index 0000000..07394f5 --- /dev/null +++ b/server/PROTOCOL.md @@ -0,0 +1,54 @@ +# Server Protocol + +The server uses TCP for connection and BSON for messages, so you need to serialize and deserialize the messages. + +# AI + +Sending the server this JSON (BSON) will send you back the AI predictions. + +```json +{ + "event": "ai", + "id": "String", + "text": "How do i download ReVanced?" +} +``` + +And the server would return something like this: + +```json +{ + "event": "ai_response", + "id": "String", + "predictions": [ + { + "label": "DOWNLOAD", + "score": "1" + } + ] +} +``` + +# OCR + +Soon:tm: + +# Training the AI + +To add data to the train data, send a BSON (JSON) like this: + +```json +{ + "event": "add_train_data", + "label": "FALSEPOSITIVE", + "text": "how" +} +``` + +To train the AI and to re-load it, send this BSON (JSON): + +```json +{ + "event": "train_ai" +} +``` \ No newline at end of file diff --git a/server/events/ai.js b/server/events/ai.js new file mode 100644 index 0000000..c9a3ce8 --- /dev/null +++ b/server/events/ai.js @@ -0,0 +1,16 @@ +import { serialize } from 'bson'; + +export default async function runAI(client, data, predict) { + const predictions = await predict(data.text); + const jsonData = { + event: 'ai_response', + id: data.id, + predictions + }; + + const bsonData = serialize(jsonData); + + client.pipe(bsonData); + + return; +} \ No newline at end of file diff --git a/server/events/index.js b/server/events/index.js new file mode 100644 index 0000000..5e16c07 --- /dev/null +++ b/server/events/index.js @@ -0,0 +1,7 @@ +import runAI from './ai.js'; +import trainAI from './trainAI.js'; + +export { + runAI, + trainAI +} \ No newline at end of file diff --git a/server/events/trainAI.js b/server/events/trainAI.js new file mode 100644 index 0000000..279e76e --- /dev/null +++ b/server/events/trainAI.js @@ -0,0 +1,51 @@ +import FastText from 'fasttext.js'; +const ft = new FastText({ + train: { + // number of concurrent threads + thread: 8, + // verbosity level [2] + verbose: 4, + // number of negatives sampled [5] + neg: 7, + // loss function {ns, hs, softmax} [ns] + loss: 'ns', + // learning rate [0.05] + lr: 1, + // change the rate of updates for the learning rate [100] + lrUpdateRate: 1000, + // max length of word ngram [1] + wordNgrams: 5, + // minimal number of word occurences + minCount: 1, + // minimal number of word occurences + minCountLabel: 1, + // size of word vectors [100] + dim: 100, + // size of the context window [5] + ws: 5, + // number of epochs [5] + epoch: 20, + // number of buckets [2000000] + bucket: 2000000, + // min length of char ngram [3] + minn: process.env.TRAIN_MINN || 3, + // max length of char ngram [6] + maxn: process.env.TRAIN_MAXN || 6, + // sampling threshold [0.0001] + t: 0.0001, + // load pre trained word vectors from unsupervised model + pretrainedVectors: '' + }, + serializeTo: '/workspaces/revanced-helper/server/model/model', + trainFile: '/workspaces/revanced-helper/server/model/train.tsv', +}); + +export default async function trainAI(unload, load) { + //unload(); + + await ft.train() + + // load(); + + return; +} \ No newline at end of file diff --git a/server/index.js b/server/index.js new file mode 100644 index 0000000..b17d373 --- /dev/null +++ b/server/index.js @@ -0,0 +1,31 @@ +import { createServer } from 'node:net'; +import { deserialize } from 'bson'; +import FastText from 'fasttext.js'; +import { runAI, trainAI } from './events/index.js'; + +const ft = new FastText({ + loadModel: './model/model.bin' +}); + +ft.load(); + +const server = createServer(async (client) => { + client.on('data', async (data) => { + const eventData = deserialize(data); + + switch(eventData.event) { + case 'ai': { + runAI(client, eventData, ft.predict); + break; + } + + case 'train_ai': { + trainAI(ft.unload, ft.load); + break; + } + } + + }); +}); + +server.listen(process.env.PORT || 3000); \ No newline at end of file diff --git a/server/package-lock.json b/server/package-lock.json new file mode 100644 index 0000000..435d9b9 --- /dev/null +++ b/server/package-lock.json @@ -0,0 +1,145 @@ +{ + "name": "server", + "version": "0.0.1", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "server", + "version": "0.0.1", + "license": "GPL-3.0-or-later", + "dependencies": { + "bson": "^4.7.0", + "fasttext.js": "^1.1.2", + "node-tesseract-ocr": "^2.2.1" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/bson": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz", + "integrity": "sha512-VrlEE4vuiO1WTpfof4VmaVolCVYkYTgB9iWgYNOrVlnifpME/06fhFRmONgBhClD5pFC1t9ZWqFUQEQAzY43bA==", + "dependencies": { + "buffer": "^5.6.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/fasttext.js": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/fasttext.js/-/fasttext.js-1.1.2.tgz", + "integrity": "sha512-ZdH7lJzOlHd2KfT5sr3Z88EQDiEKQzxt+v2J7t1ZVZZhmtM5XZtCQNVDD/wWRNl5zEjv8LBrBoRGTIpcMX52Vw==", + "engines": { + "node": "*" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/node-tesseract-ocr": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/node-tesseract-ocr/-/node-tesseract-ocr-2.2.1.tgz", + "integrity": "sha512-Q9cD79JGpPNQBxbi1fV+OAsTxYKLpx22sagsxSyKbu1u+t6UarApf5m32uVc8a5QAP1Wk7fIPN0aJFGGEE9DyQ==", + "engines": { + "node": ">=10" + } + } + }, + "dependencies": { + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, + "bson": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz", + "integrity": "sha512-VrlEE4vuiO1WTpfof4VmaVolCVYkYTgB9iWgYNOrVlnifpME/06fhFRmONgBhClD5pFC1t9ZWqFUQEQAzY43bA==", + "requires": { + "buffer": "^5.6.0" + } + }, + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "fasttext.js": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/fasttext.js/-/fasttext.js-1.1.2.tgz", + "integrity": "sha512-ZdH7lJzOlHd2KfT5sr3Z88EQDiEKQzxt+v2J7t1ZVZZhmtM5XZtCQNVDD/wWRNl5zEjv8LBrBoRGTIpcMX52Vw==" + }, + "ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + }, + "node-tesseract-ocr": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/node-tesseract-ocr/-/node-tesseract-ocr-2.2.1.tgz", + "integrity": "sha512-Q9cD79JGpPNQBxbi1fV+OAsTxYKLpx22sagsxSyKbu1u+t6UarApf5m32uVc8a5QAP1Wk7fIPN0aJFGGEE9DyQ==" + } + } +} diff --git a/server/package.json b/server/package.json new file mode 100644 index 0000000..d2095d5 --- /dev/null +++ b/server/package.json @@ -0,0 +1,14 @@ +{ + "name": "server", + "version": "0.0.1", + "description": "The AI and OCR server for ReVanced Helper.", + "type": "module", + "main": "index.js", + "author": "Reis Can", + "license": "GPL-3.0-or-later", + "dependencies": { + "bson": "^4.7.0", + "fasttext.js": "^1.1.2", + "node-tesseract-ocr": "^2.2.1" + } +}