Add Server

2026-01-11 13:56:15 +00:00 · 2022-11-10 13:55:33 +00:00
parent c196886d06
commit 9962e4fe68
9 changed files with 336 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -0,0 +1,3 @@
+node_modules
+model
+test.js
--- a/package.json
+++ b/package.json
@@ -0,0 +1,15 @@
+{
+  "name": "revanced-helper",
+  "version": "0.0.1",
+  "description": "The community bots (and the server) to help the community and make moderators job easy.",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/reisxd/revanced-helper.git"
+  },
+  "author": "Reis Can",
+  "license": "GPL-3.0-or-later",
+  "bugs": {
+    "url": "https://github.com/reisxd/revanced-helper/issues"
+  },
+  "homepage": "https://github.com/reisxd/revanced-helper#readme"
+}
--- a/server/PROTOCOL.md
+++ b/server/PROTOCOL.md
@@ -0,0 +1,54 @@
+# Server Protocol
+
+The server uses TCP for connection and BSON for messages, so you need to serialize and deserialize the messages.
+
+# AI
+
+Sending the server this JSON (BSON) will send you back the AI predictions.
+
+```json
+{
+    "event": "ai",
+    "id": "String", 
+    "text": "How do i download ReVanced?"
+}
+```
+
+And the server would return something like this:
+
+```json
+{
+    "event": "ai_response",
+    "id": "String",
+    "predictions": [
+        {
+            "label": "DOWNLOAD",
+            "score": "1"
+        }
+    ]
+}
+```
+
+# OCR
+
+Soon:tm:
+
+# Training the AI
+
+To add data to the train data, send a BSON (JSON) like this:
+
+```json
+{
+    "event": "add_train_data",
+    "label": "FALSEPOSITIVE",
+    "text": "how"
+}
+```
+
+To train the AI and to re-load it, send this BSON (JSON):
+
+```json
+{
+    "event": "train_ai"
+}
+```
--- a/server/events/ai.js
+++ b/server/events/ai.js
@@ -0,0 +1,16 @@
+import { serialize } from 'bson';
+
+export default async function runAI(client, data, predict) {
+    const predictions = await predict(data.text);
+    const jsonData = {
+        event: 'ai_response',
+        id: data.id,
+        predictions
+    };
+
+    const bsonData = serialize(jsonData);
+
+    client.pipe(bsonData);
+    
+    return;
+}
--- a/server/events/index.js
+++ b/server/events/index.js
@@ -0,0 +1,7 @@
+import runAI from './ai.js';
+import trainAI from './trainAI.js';
+
+export {
+    runAI,
+    trainAI
+}
--- a/server/events/trainAI.js
+++ b/server/events/trainAI.js
@@ -0,0 +1,51 @@
+import FastText from 'fasttext.js';
+const ft = new FastText({
+    train: {
+        // number of concurrent threads
+        thread: 8,
+        // verbosity level [2]
+        verbose: 4,
+        // number of negatives sampled [5]
+        neg: 7,
+        // loss function {ns, hs, softmax} [ns]
+        loss: 'ns',
+        // learning rate [0.05]
+        lr: 1,
+        // change the rate of updates for the learning rate [100]
+        lrUpdateRate: 1000,
+        // max length of word ngram [1]
+        wordNgrams: 5,
+        // minimal number of word occurences
+        minCount: 1,
+        // minimal number of word occurences
+        minCountLabel: 1,
+        // size of word vectors [100]
+        dim: 100,
+        // size of the context window [5]
+        ws: 5,
+        //  number of epochs [5]
+        epoch: 20,
+        // number of buckets [2000000]
+        bucket: 2000000,
+        // min length of char ngram [3]
+        minn: process.env.TRAIN_MINN || 3,
+        // max length of char ngram [6]
+        maxn: process.env.TRAIN_MAXN || 6,
+        // sampling threshold [0.0001]
+        t: 0.0001,
+        // load pre trained word vectors from unsupervised model
+        pretrainedVectors: ''
+    },
+    serializeTo: '/workspaces/revanced-helper/server/model/model',
+    trainFile: '/workspaces/revanced-helper/server/model/train.tsv',
+});
+
+export default async function trainAI(unload, load) {
+    //unload();
+
+    await ft.train()
+
+  //  load();
+
+    return;
+}
--- a/server/index.js
+++ b/server/index.js
@@ -0,0 +1,31 @@
+import { createServer } from 'node:net';
+import { deserialize } from 'bson';
+import FastText from 'fasttext.js';
+import { runAI, trainAI } from './events/index.js';
+
+const ft = new FastText({
+    loadModel: './model/model.bin'
+});
+
+ft.load();
+
+const server = createServer(async (client) => {
+    client.on('data', async (data) => {
+        const eventData = deserialize(data);
+
+        switch(eventData.event) {
+            case 'ai': {
+                runAI(client, eventData, ft.predict);
+                break;
+            }
+
+            case 'train_ai': {
+                trainAI(ft.unload, ft.load);
+                break;
+            }
+        }
+
+    });
+});
+
+server.listen(process.env.PORT || 3000);
--- a/server/package-lock.json
+++ b/server/package-lock.json
@@ -0,0 +1,145 @@
+{
+  "name": "server",
+  "version": "0.0.1",
+  "lockfileVersion": 2,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "server",
+      "version": "0.0.1",
+      "license": "GPL-3.0-or-later",
+      "dependencies": {
+        "bson": "^4.7.0",
+        "fasttext.js": "^1.1.2",
+        "node-tesseract-ocr": "^2.2.1"
+      }
+    },
+    "node_modules/base64-js": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
+      "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ]
+    },
+    "node_modules/bson": {
+      "version": "4.7.0",
+      "resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz",
+      "integrity": "sha512-VrlEE4vuiO1WTpfof4VmaVolCVYkYTgB9iWgYNOrVlnifpME/06fhFRmONgBhClD5pFC1t9ZWqFUQEQAzY43bA==",
+      "dependencies": {
+        "buffer": "^5.6.0"
+      },
+      "engines": {
+        "node": ">=6.9.0"
+      }
+    },
+    "node_modules/buffer": {
+      "version": "5.7.1",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
+      "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "dependencies": {
+        "base64-js": "^1.3.1",
+        "ieee754": "^1.1.13"
+      }
+    },
+    "node_modules/fasttext.js": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/fasttext.js/-/fasttext.js-1.1.2.tgz",
+      "integrity": "sha512-ZdH7lJzOlHd2KfT5sr3Z88EQDiEKQzxt+v2J7t1ZVZZhmtM5XZtCQNVDD/wWRNl5zEjv8LBrBoRGTIpcMX52Vw==",
+      "engines": {
+        "node": "*"
+      }
+    },
+    "node_modules/ieee754": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
+      "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ]
+    },
+    "node_modules/node-tesseract-ocr": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/node-tesseract-ocr/-/node-tesseract-ocr-2.2.1.tgz",
+      "integrity": "sha512-Q9cD79JGpPNQBxbi1fV+OAsTxYKLpx22sagsxSyKbu1u+t6UarApf5m32uVc8a5QAP1Wk7fIPN0aJFGGEE9DyQ==",
+      "engines": {
+        "node": ">=10"
+      }
+    }
+  },
+  "dependencies": {
+    "base64-js": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
+      "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="
+    },
+    "bson": {
+      "version": "4.7.0",
+      "resolved": "https://registry.npmjs.org/bson/-/bson-4.7.0.tgz",
+      "integrity": "sha512-VrlEE4vuiO1WTpfof4VmaVolCVYkYTgB9iWgYNOrVlnifpME/06fhFRmONgBhClD5pFC1t9ZWqFUQEQAzY43bA==",
+      "requires": {
+        "buffer": "^5.6.0"
+      }
+    },
+    "buffer": {
+      "version": "5.7.1",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
+      "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
+      "requires": {
+        "base64-js": "^1.3.1",
+        "ieee754": "^1.1.13"
+      }
+    },
+    "fasttext.js": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/fasttext.js/-/fasttext.js-1.1.2.tgz",
+      "integrity": "sha512-ZdH7lJzOlHd2KfT5sr3Z88EQDiEKQzxt+v2J7t1ZVZZhmtM5XZtCQNVDD/wWRNl5zEjv8LBrBoRGTIpcMX52Vw=="
+    },
+    "ieee754": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
+      "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="
+    },
+    "node-tesseract-ocr": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/node-tesseract-ocr/-/node-tesseract-ocr-2.2.1.tgz",
+      "integrity": "sha512-Q9cD79JGpPNQBxbi1fV+OAsTxYKLpx22sagsxSyKbu1u+t6UarApf5m32uVc8a5QAP1Wk7fIPN0aJFGGEE9DyQ=="
+    }
+  }
+}
--- a/server/package.json
+++ b/server/package.json
@@ -0,0 +1,14 @@
+{
+  "name": "server",
+  "version": "0.0.1",
+  "description": "The AI and OCR server for ReVanced Helper.",
+  "type": "module",
+  "main": "index.js",
+  "author": "Reis Can",
+  "license": "GPL-3.0-or-later",
+  "dependencies": {
+    "bson": "^4.7.0",
+    "fasttext.js": "^1.1.2",
+    "node-tesseract-ocr": "^2.2.1"
+  }
+}