Compare commits

...

48 Commits

Author SHA1 Message Date
Bas950
5a90f95e58 chore: release v0.0.37 2024-09-24 17:07:06 +02:00
Bas950
64547dc0ef feat: ip data 2024-09-24 17:06:48 +02:00
Bas950
9cf3f93889 chore: release v0.0.36 2024-09-24 13:47:54 +02:00
Bas950
0e30a0d250 chore: scan keys instead 2024-09-24 13:47:39 +02:00
Bas950
4dc941bb91 chore: release v0.0.35 2024-09-24 11:08:50 +02:00
Bas950
3a78c6529e feat: use prom-client 2024-09-24 10:50:21 +02:00
Bas950
d4673720a0 chore: release v1.0.4 2024-09-20 19:03:37 +02:00
Bas950
dc859448bd feat: schema v1.11 2024-09-20 18:57:28 +02:00
Bas950
9cbb88beda chore: release v0.0.34 2024-09-17 10:54:54 +02:00
Bas950
09bcfe703f chore: scan count config 2024-09-17 10:54:48 +02:00
Bas950
d24eda8957 chore: release v0.0.33 2024-09-17 10:45:42 +02:00
Bas950
bfffcb94ee chore: some testing 2024-09-17 10:45:38 +02:00
Bas950
4db6a78816 chore: release v0.0.32 2024-09-17 10:35:04 +02:00
Bas950
666838874f chore: forgot to save 2024-09-17 10:34:58 +02:00
Bas950
697f3660c2 chore: some improvements 2024-09-17 10:34:46 +02:00
Florian Metz
a668add973 chore: release v0.0.31 2024-09-17 10:00:06 +02:00
Florian Metz
42b70b1259 chore: optimize active presence gauge update with concurrency limit 2024-09-17 09:59:41 +02:00
Bas950
253b680d3e chore: release v0.0.30 2024-09-17 09:36:49 +02:00
Bas950
e9a40dc553 chore: small updates 2024-09-17 09:36:43 +02:00
Bas950
b25880d4cd chore: release v0.0.29 2024-09-17 09:11:12 +02:00
Bas950
fb06227aeb chore: release v0.0.29 2024-09-17 09:07:58 +02:00
Bas950
ff3d00497b chore: release v0.0.29 2024-09-17 09:07:21 +02:00
Bas950
a06780f85a chore: reduce memory 2024-09-17 09:06:10 +02:00
Bas950
5b1969c7ab chore: release v0.0.28 2024-09-16 23:22:15 +02:00
Bas950
bedd34594c chore: disable ip stuff for now 2024-09-16 23:22:11 +02:00
Bas950
47feaa5c70 chore: release v0.0.27 2024-09-16 22:56:14 +02:00
Bas950
9fb32f53ae chore: reduce batch size 2024-09-16 22:56:10 +02:00
Bas950
bfb84bb080 chore: release v0.0.26 2024-09-16 22:32:52 +02:00
Bas950
f545b174bd chore: lint 2024-09-16 22:32:42 +02:00
Bas950
4a492cf275 fix: store ip data in postgres 2024-09-16 22:30:29 +02:00
Bas950
3f65f678b1 chore: release v0.0.25 2024-09-16 20:58:24 +02:00
Bas950
a71b66540b chore: release v0.0.14 2024-09-16 20:58:07 +02:00
Bas van Zanten
e675f74983 feat: update tracing (#1067) 2024-09-16 20:18:35 +02:00
Florian Metz
e9e6639492 chore: release v0.0.13 2024-09-15 03:09:23 +02:00
Florian Metz
3258179040 chore: release v0.0.24 2024-09-15 03:09:13 +02:00
Florian Metz
086d476af2 chore: update hash 2024-09-15 03:09:04 +02:00
Florian Metz
146bf9e270 chore: release v0.0.23 2024-09-15 02:48:56 +02:00
Florian Metz
a02f25ba29 chore: test 2024-09-15 02:48:41 +02:00
Florian Metz
416b65f0d4 chore: release v0.0.12 2024-09-15 02:41:31 +02:00
Florian Metz
f8e9fc832d chore: test 2024-09-15 02:41:16 +02:00
Florian Metz
86b0f07216 chore: test 2024-09-15 02:31:38 +02:00
Florian Metz
9eb5c03877 chore: release v0.0.11 2024-09-15 02:25:50 +02:00
Florian Metz
e63e1270aa chore: release v0.0.22 2024-09-15 02:25:38 +02:00
Florian Metz
f730e71bbf chore: test 2024-09-15 02:25:10 +02:00
Bas950
8b68bf85c8 chore: release v0.0.10 2024-09-13 17:27:44 +02:00
Bas950
e4c794a9ad chore: 202 on disabled flag 2024-09-13 17:27:38 +02:00
Bas950
6e8258d76f chore: release v0.0.21 2024-09-13 15:08:16 +02:00
Bas950
56b796c621 chore: use ky 2024-09-13 15:08:08 +02:00
39 changed files with 2165 additions and 338 deletions

4
.gitignore vendored
View File

@@ -3,6 +3,7 @@ out
dist
tmp
lib
data
.vscode
.env
@@ -22,4 +23,5 @@ src/update.ini
!eslint.config.js
coverage
*.tsbuildinfo
*.tsbuildinfo
.DS_Store

View File

@@ -1,2 +1,3 @@
*.js
*.ts
*.ts
*.json

View File

@@ -0,0 +1,10 @@
import { defineConfig } from "drizzle-kit";
export default defineConfig({
dbCredentials: {
url: "postgresql://metrics:metrics@localhost:5432/metrics",
},
dialect: "postgresql",
schema: "./src/db.ts",
out: "./drizzle",
});

View File

@@ -0,0 +1,9 @@
CREATE TABLE IF NOT EXISTS "online_users_ip_data" (
"uuid" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
"ip" varchar(45) NOT NULL,
"country" varchar(2) NOT NULL,
"latitude" numeric(10, 8) NOT NULL,
"longitude" numeric(11, 8) NOT NULL,
"name" varchar(255),
"timestamp" timestamp DEFAULT now()
);

View File

@@ -0,0 +1,2 @@
CREATE INDEX IF NOT EXISTS "idx_online_users_uuid" ON "online_users_ip_data" USING btree ("uuid");--> statement-breakpoint
CREATE INDEX IF NOT EXISTS "idx_online_users_timestamp" ON "online_users_ip_data" USING btree ("timestamp");

View File

@@ -0,0 +1 @@
ALTER TABLE "online_users_ip_data" ALTER COLUMN "timestamp" SET DATA TYPE timestamp with time zone;

View File

@@ -0,0 +1,2 @@
ALTER TABLE "online_users_ip_data" ADD COLUMN "presences" jsonb DEFAULT '[]' NOT NULL;--> statement-breakpoint
ALTER TABLE "online_users_ip_data" DROP COLUMN IF EXISTS "name";

View File

@@ -0,0 +1 @@
ALTER TABLE "online_users_ip_data" ADD COLUMN "sessions" integer DEFAULT 0 NOT NULL;

View File

@@ -0,0 +1,70 @@
{
"id": "e29a6708-01f1-455a-b345-63dac1e124dc",
"prevId": "00000000-0000-0000-0000-000000000000",
"version": "7",
"dialect": "postgresql",
"tables": {
"public.online_users_ip_data": {
"name": "online_users_ip_data",
"schema": "",
"columns": {
"uuid": {
"name": "uuid",
"type": "uuid",
"primaryKey": true,
"notNull": true,
"default": "gen_random_uuid()"
},
"ip": {
"name": "ip",
"type": "varchar(45)",
"primaryKey": false,
"notNull": true
},
"country": {
"name": "country",
"type": "varchar(2)",
"primaryKey": false,
"notNull": true
},
"latitude": {
"name": "latitude",
"type": "numeric(10, 8)",
"primaryKey": false,
"notNull": true
},
"longitude": {
"name": "longitude",
"type": "numeric(11, 8)",
"primaryKey": false,
"notNull": true
},
"name": {
"name": "name",
"type": "varchar(255)",
"primaryKey": false,
"notNull": false
},
"timestamp": {
"name": "timestamp",
"type": "timestamp",
"primaryKey": false,
"notNull": false,
"default": "now()"
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {}
}
},
"enums": {},
"schemas": {},
"sequences": {},
"_meta": {
"columns": {},
"schemas": {},
"tables": {}
}
}

View File

@@ -0,0 +1,101 @@
{
"id": "4aa32a8e-f573-43b9-976a-2d078a0df0ea",
"prevId": "e29a6708-01f1-455a-b345-63dac1e124dc",
"version": "7",
"dialect": "postgresql",
"tables": {
"public.online_users_ip_data": {
"name": "online_users_ip_data",
"schema": "",
"columns": {
"uuid": {
"name": "uuid",
"type": "uuid",
"primaryKey": true,
"notNull": true,
"default": "gen_random_uuid()"
},
"ip": {
"name": "ip",
"type": "varchar(45)",
"primaryKey": false,
"notNull": true
},
"country": {
"name": "country",
"type": "varchar(2)",
"primaryKey": false,
"notNull": true
},
"latitude": {
"name": "latitude",
"type": "numeric(10, 8)",
"primaryKey": false,
"notNull": true
},
"longitude": {
"name": "longitude",
"type": "numeric(11, 8)",
"primaryKey": false,
"notNull": true
},
"name": {
"name": "name",
"type": "varchar(255)",
"primaryKey": false,
"notNull": false
},
"timestamp": {
"name": "timestamp",
"type": "timestamp",
"primaryKey": false,
"notNull": false,
"default": "now()"
}
},
"indexes": {
"idx_online_users_uuid": {
"name": "idx_online_users_uuid",
"columns": [
{
"expression": "uuid",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
},
"idx_online_users_timestamp": {
"name": "idx_online_users_timestamp",
"columns": [
{
"expression": "timestamp",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {}
}
},
"enums": {},
"schemas": {},
"sequences": {},
"_meta": {
"columns": {},
"schemas": {},
"tables": {}
}
}

View File

@@ -0,0 +1,101 @@
{
"id": "c1b8dbed-b232-4d66-9e74-b9af333095bc",
"prevId": "4aa32a8e-f573-43b9-976a-2d078a0df0ea",
"version": "7",
"dialect": "postgresql",
"tables": {
"public.online_users_ip_data": {
"name": "online_users_ip_data",
"schema": "",
"columns": {
"uuid": {
"name": "uuid",
"type": "uuid",
"primaryKey": true,
"notNull": true,
"default": "gen_random_uuid()"
},
"ip": {
"name": "ip",
"type": "varchar(45)",
"primaryKey": false,
"notNull": true
},
"country": {
"name": "country",
"type": "varchar(2)",
"primaryKey": false,
"notNull": true
},
"latitude": {
"name": "latitude",
"type": "numeric(10, 8)",
"primaryKey": false,
"notNull": true
},
"longitude": {
"name": "longitude",
"type": "numeric(11, 8)",
"primaryKey": false,
"notNull": true
},
"name": {
"name": "name",
"type": "varchar(255)",
"primaryKey": false,
"notNull": false
},
"timestamp": {
"name": "timestamp",
"type": "timestamp with time zone",
"primaryKey": false,
"notNull": false,
"default": "now()"
}
},
"indexes": {
"idx_online_users_uuid": {
"name": "idx_online_users_uuid",
"columns": [
{
"expression": "uuid",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
},
"idx_online_users_timestamp": {
"name": "idx_online_users_timestamp",
"columns": [
{
"expression": "timestamp",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {}
}
},
"enums": {},
"schemas": {},
"sequences": {},
"_meta": {
"columns": {},
"schemas": {},
"tables": {}
}
}

View File

@@ -0,0 +1,102 @@
{
"id": "e409a4d0-f698-484a-b412-38966a7b3a19",
"prevId": "c1b8dbed-b232-4d66-9e74-b9af333095bc",
"version": "7",
"dialect": "postgresql",
"tables": {
"public.online_users_ip_data": {
"name": "online_users_ip_data",
"schema": "",
"columns": {
"uuid": {
"name": "uuid",
"type": "uuid",
"primaryKey": true,
"notNull": true,
"default": "gen_random_uuid()"
},
"ip": {
"name": "ip",
"type": "varchar(45)",
"primaryKey": false,
"notNull": true
},
"country": {
"name": "country",
"type": "varchar(2)",
"primaryKey": false,
"notNull": true
},
"latitude": {
"name": "latitude",
"type": "numeric(10, 8)",
"primaryKey": false,
"notNull": true
},
"longitude": {
"name": "longitude",
"type": "numeric(11, 8)",
"primaryKey": false,
"notNull": true
},
"presences": {
"name": "presences",
"type": "jsonb",
"primaryKey": false,
"notNull": true,
"default": "'[]'"
},
"timestamp": {
"name": "timestamp",
"type": "timestamp with time zone",
"primaryKey": false,
"notNull": false,
"default": "now()"
}
},
"indexes": {
"idx_online_users_uuid": {
"name": "idx_online_users_uuid",
"columns": [
{
"expression": "uuid",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
},
"idx_online_users_timestamp": {
"name": "idx_online_users_timestamp",
"columns": [
{
"expression": "timestamp",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {}
}
},
"enums": {},
"schemas": {},
"sequences": {},
"_meta": {
"columns": {},
"schemas": {},
"tables": {}
}
}

View File

@@ -0,0 +1,109 @@
{
"id": "179435b5-dc15-4a42-9539-c3f336699d63",
"prevId": "e409a4d0-f698-484a-b412-38966a7b3a19",
"version": "7",
"dialect": "postgresql",
"tables": {
"public.online_users_ip_data": {
"name": "online_users_ip_data",
"schema": "",
"columns": {
"uuid": {
"name": "uuid",
"type": "uuid",
"primaryKey": true,
"notNull": true,
"default": "gen_random_uuid()"
},
"ip": {
"name": "ip",
"type": "varchar(45)",
"primaryKey": false,
"notNull": true
},
"country": {
"name": "country",
"type": "varchar(2)",
"primaryKey": false,
"notNull": true
},
"latitude": {
"name": "latitude",
"type": "numeric(10, 8)",
"primaryKey": false,
"notNull": true
},
"longitude": {
"name": "longitude",
"type": "numeric(11, 8)",
"primaryKey": false,
"notNull": true
},
"sessions": {
"name": "sessions",
"type": "integer",
"primaryKey": false,
"notNull": true,
"default": 0
},
"presences": {
"name": "presences",
"type": "jsonb",
"primaryKey": false,
"notNull": true,
"default": "'[]'"
},
"timestamp": {
"name": "timestamp",
"type": "timestamp with time zone",
"primaryKey": false,
"notNull": false,
"default": "now()"
}
},
"indexes": {
"idx_online_users_uuid": {
"name": "idx_online_users_uuid",
"columns": [
{
"expression": "uuid",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
},
"idx_online_users_timestamp": {
"name": "idx_online_users_timestamp",
"columns": [
{
"expression": "timestamp",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {}
}
},
"enums": {},
"schemas": {},
"sequences": {},
"_meta": {
"columns": {},
"schemas": {},
"tables": {}
}
}

View File

@@ -0,0 +1,41 @@
{
"version": "7",
"dialect": "postgresql",
"entries": [
{
"idx": 0,
"version": "7",
"when": 1726516195146,
"tag": "0000_flippant_marrow",
"breakpoints": true
},
{
"idx": 1,
"version": "7",
"when": 1726516348344,
"tag": "0001_white_lifeguard",
"breakpoints": true
},
{
"idx": 2,
"version": "7",
"when": 1726516660134,
"tag": "0002_new_darkhawk",
"breakpoints": true
},
{
"idx": 3,
"version": "7",
"when": 1726517073510,
"tag": "0003_narrow_mastermind",
"breakpoints": true
},
{
"idx": 4,
"version": "7",
"when": 1726517405363,
"tag": "0004_tiresome_puff_adder",
"breakpoints": true
}
]
}

16
apps/api-master/environment.d.ts vendored Normal file
View File

@@ -0,0 +1,16 @@
declare module "ip-location-api" {
export function lookup(ip: string): Promise<{
latitude: number;
longitude: number;
country: string;
} | null>;
export function updateDb(options: { fields?: string[]; dataDir?: string; tmpDataDir?: string; smallMemory?: boolean; autoUpdate?: string }): Promise<void>;
export function reload(options: { fields?: string[]; dataDir?: string; tmpDataDir?: string; smallMemory?: boolean; autoUpdate?: string }): Promise<void>;
}
declare namespace NodeJS {
export interface ProcessEnv {
METRICS_DATABASE_URL?: string;
}
}

View File

@@ -1,7 +1,7 @@
{
"name": "@premid/api-master",
"type": "module",
"version": "0.0.20",
"version": "0.0.37",
"private": true,
"description": "PreMiD's api master",
"license": "MPL-2.0",
@@ -11,21 +11,27 @@
],
"scripts": {
"start": "node --enable-source-maps .",
"dev": "node --watch --env-file .env --enable-source-maps ."
"dev": "node --watch --env-file .env --enable-source-maps .",
"db:generate": "drizzle-kit generate",
"db:migrate": "drizzle-kit migrate",
"db:update": "pnpm db:generate && pnpm db:migrate",
"db:studio": "drizzle-kit studio"
},
"dependencies": {
"@discordjs/rest": "^2.3.0",
"@envelop/sentry": "^9.0.0",
"@opentelemetry/api": "^1.9.0",
"@opentelemetry/exporter-prometheus": "^0.52.1",
"@opentelemetry/node": "^0.24.0",
"@sentry/node": "^8.17.0",
"cron": "^3.1.7",
"debug": "^4.3.6",
"drizzle-orm": "^0.33.0",
"ioredis": "^5.3.2",
"p-limit": "^6.1.0"
"ip-location-api": "^2.0.1",
"ky": "^1.7.2",
"p-limit": "^6.1.0",
"postgres": "^3.4.4",
"prom-client": "^15.1.3"
},
"devDependencies": {
"@types/debug": "^4.1.12"
"@types/debug": "^4.1.12",
"drizzle-kit": "^0.24.2"
}
}

27
apps/api-master/src/db.ts Normal file
View File

@@ -0,0 +1,27 @@
import process from "node:process";
import { decimal, index, integer, jsonb, pgTable, timestamp, uuid, varchar } from "drizzle-orm/pg-core";
import { drizzle } from "drizzle-orm/postgres-js";
import postgres from "postgres";
// Define the schema
export const onlineUsersIpData = pgTable("online_users_ip_data", {
uuid: uuid("uuid").primaryKey().defaultRandom(),
ip: varchar("ip", { length: 45 }).notNull(),
country: varchar("country", { length: 2 }).notNull(),
latitude: decimal("latitude", { precision: 10, scale: 8 }).notNull(),
longitude: decimal("longitude", { precision: 11, scale: 8 }).notNull(),
sessions: integer("sessions").notNull().default(0),
presences: jsonb("presences").notNull().default("[]").$type<string[]>(),
timestamp: timestamp("timestamp", { withTimezone: true }).defaultNow(),
}, table => ({
idxOnlineUsersUuid: index("idx_online_users_uuid").on(table.uuid),
idxOnlineUsersTimestamp: index("idx_online_users_timestamp").on(table.timestamp),
}));
if (!process.env.METRICS_DATABASE_URL) {
throw new Error("METRICS_DATABASE_URL is not set");
}
export const sql = postgres(process.env.METRICS_DATABASE_URL);
export const db = drizzle(sql);

View File

@@ -0,0 +1,10 @@
import { lt, sql } from "drizzle-orm";
import { db, onlineUsersIpData } from "../db.js";
import { mainLog } from "../index.js";
export async function cleanupOldUserData(retentionDays: number) {
mainLog("Cleaning up old user ip data");
const interval = `'${retentionDays} days'`;
await db.delete(onlineUsersIpData)
.where(lt(onlineUsersIpData.timestamp, sql`now() - interval ${sql.raw(interval)}`));
}

View File

@@ -1,5 +1,5 @@
import { REST } from "@discordjs/rest";
import pLimit from "p-limit";
import ky, { HTTPError, TimeoutError } from "ky";
import { mainLog, redis } from "../index.js";
let inProgress = false;
@@ -11,6 +11,7 @@ export async function clearOldSessions() {
inProgress = true;
const now = Date.now();
const pattern = "pmd-api.sessions.*";
let cursor = "0";
let totalSessions = 0;
let cleared = 0;
@@ -22,21 +23,15 @@ export async function clearOldSessions() {
const limit = pLimit(100); // Create a limit of 100 concurrent operations
do {
const [nextCursor, result] = await redis.hscan("pmd-api.sessions", cursor, "COUNT", batchSize);
cursor = nextCursor;
totalSessions += result.length / 2;
const [newCursor, keys] = await redis.scan(cursor, "MATCH", pattern, "COUNT", 1000); //* Use SCAN with COUNT for memory efficiency
const deletePromises = [];
cursor = newCursor;
totalSessions += keys.length;
for (let i = 0; i < result.length; i += 2) {
const key = result[i];
const value = result[i + 1];
const deletePromises: Promise<string>[] = [];
if (!key || !value) {
continue;
}
const session = JSON.parse(value) as {
for (const key of keys) {
const session = await redis.hgetall(key) as unknown as {
token: string;
session: string;
lastUpdated: number;
@@ -57,13 +52,13 @@ export async function clearOldSessions() {
});
if (keysToDelete.length >= batchSize) {
await redis.hdel("pmd-api.sessions", ...keysToDelete);
await redis.del(...keysToDelete);
keysToDelete = [];
}
} while (cursor !== "0");
if (keysToDelete.length > 0) {
await redis.hdel("pmd-api.sessions", ...keysToDelete);
await redis.del(...keysToDelete);
}
if (totalSessions === 0) {
@@ -77,35 +72,29 @@ export async function clearOldSessions() {
}
async function deleteSession(session: { token: string; session: string }, key: string): Promise<string> {
const abortController = new AbortController();
const timeoutId = setTimeout(() => abortController.abort("Timeout"), 5000); //* 5 second timeout
try {
const discord = new REST({ version: "10", authPrefix: "Bearer" });
discord.setToken(session.token);
await discord.post("/users/@me/headless-sessions/delete", {
signal: abortController.signal,
body: {
await ky.post("https://discord.com/api/v10/users/@me/headless-sessions/delete", {
json: {
token: session.session,
},
headers: {
Authorization: `Bearer ${session.token}`,
},
retry: 3,
timeout: 5000,
});
clearTimeout(timeoutId);
return key;
}
catch (error) {
clearTimeout(timeoutId);
if (error instanceof Error && error.name === "AbortError") {
if (error instanceof TimeoutError) {
mainLog(`Session deletion aborted due to timeout for key ${key}`);
}
else if (error instanceof Error) {
mainLog(`Failed to delete session for key ${key}: [${error.name}] ${error.message}`);
else if (error instanceof HTTPError) {
mainLog(`Failed to delete session for key ${key}: [${error.name}] ${error.message} ${JSON.stringify(await error.response.json())}`);
}
else {
mainLog(`Failed to delete session for key ${key}: Unknown error`);
}
return key;
}
return key;
}

View File

@@ -0,0 +1,40 @@
import type { InferInsertModel } from "drizzle-orm";
import { db, onlineUsersIpData } from "../db.js";
import { lookupIp } from "./lookupIp.js";
const batchSize = 1000;
export async function insertIpData(
data: Map<string, {
presences: string[];
sessions: number;
}>,
) {
const timestamp = new Date();
const list = [...data.keys()];
//* Split into batches of batchSize
for (let i = 0; i < list.length; i += batchSize) {
const batch = list.slice(i, i + batchSize);
const mapped = await Promise.all(batch.map(async (ip) => {
const parsed = await lookupIp(ip);
if (parsed) {
const { presences, sessions } = data.get(ip)!;
return {
ip,
country: parsed.country,
latitude: parsed.latitude.toString(),
longitude: parsed.longitude.toString(),
presences,
sessions,
timestamp,
} satisfies InferInsertModel<typeof onlineUsersIpData>;
}
}));
const toInsert = mapped.filter(Boolean) as InferInsertModel<typeof onlineUsersIpData>[];
if (toInsert.length > 0) {
await db.insert(onlineUsersIpData).values(toInsert);
}
}
}

View File

@@ -0,0 +1,46 @@
import { join } from "node:path";
import process from "node:process";
import { lookup, reload } from "ip-location-api";
import { mainLog } from "../index.js";
const fields = ["latitude", "longitude", "country"];
const dataDir = join(process.cwd(), "data");
const tmpDataDir = join(process.cwd(), "tmp");
const smallMemory = true;
let initialized = false;
export async function lookupIp(ip: string): Promise<{ latitude: number; longitude: number; country: string } | undefined> {
if (!initialized) {
reloadIpLocationApi();
return undefined;
}
try {
return await lookup(ip) ?? undefined;
}
catch {
return undefined;
}
}
let reloading: Promise<void> | undefined;
let log: debug.Debugger | undefined;
export async function reloadIpLocationApi() {
log ??= mainLog.extend("IP-Location-API");
if (reloading)
return reloading;
reloading = new Promise((resolve, reject) => {
log?.("Reloading IP location API");
reload({ fields, dataDir, tmpDataDir, smallMemory, autoUpdate: "0 9 * * *" }).then(() => {
log?.("IP location API reloaded");
initialized = true;
reloading = undefined;
resolve();
}).catch(reject);
});
return reloading;
}

View File

@@ -1,13 +0,0 @@
import { redis } from "../index.js";
import { counter } from "../tracing.js";
let activeActivities = 0;
counter.add(0);
export async function setCounter() {
const length = await redis.hlen("pmd-api.sessions");
if (length === activeActivities)
return;
const diff = length - activeActivities;
activeActivities = length;
counter.add(diff);
}

View File

@@ -0,0 +1,25 @@
import http from "node:http";
import { mainLog } from "../index.js";
import { register } from "../tracing.js";
export function setupServer() {
const server = http.createServer(async (req, res) => {
//* If it's a head request, just return 200
if (req.method === "HEAD")
return res.writeHead(200).end();
//* If it's a favicon request, just return 404
if (req.url === "/favicon.ico")
return res.writeHead(404).end();
//* Basic routing logic
res.writeHead(200, { "Content-Type": "text/plain" });
res.end(await register.metrics());
});
server.listen(9464, () => {
mainLog("Server running");
});
return server;
}

View File

@@ -0,0 +1,77 @@
import process from "node:process";
import pLimit from "p-limit";
import type { Gauge } from "prom-client";
import { mainLog, redis } from "../index.js";
import { insertIpData } from "./insertIpData.js";
export const updateActivePresenceGaugeLimit = pLimit(1);
let log: debug.Debugger | undefined;
const scanCount = Number.parseInt(process.env.SCAN_COUNT || "1000", 10);
export async function updateActivePresenceGauge(gauge: Gauge) {
await updateActivePresenceGaugeLimit(async () => {
log ??= mainLog.extend("Heartbeat-Updates");
log?.("Starting active presence gauge update");
const pattern = "pmd-api.heartbeatUpdates.*";
let cursor: string = "0";
const serviceCounts = new Map<string, number>();
const ips = new Map<string, {
presences: Set<string>;
sessions: number;
}>();
do {
const [newCursor, keys] = await redis.scan(cursor, "MATCH", pattern, "COUNT", scanCount);
cursor = newCursor;
//* Use pipelining for batch Redis operations
const pipeline = redis.pipeline();
keys.forEach(key => pipeline.hmget(key, "service", "version", "ip_address"));
const hashes = await pipeline.exec();
if (!hashes) {
log?.("No hashes found");
return;
}
hashes.forEach(([err, hash]) => {
if (err || !Array.isArray(hash))
return;
const [service, version, ip] = hash;
const serviceVersion = service && version ? `${service}:${version}` : "none";
serviceCounts.set(serviceVersion, (serviceCounts.get(serviceVersion) || 0) + 1);
if (ip) {
const ipData = ips.get(ip) || { presences: new Set(), sessions: 0 };
if (serviceVersion !== "none")
ipData.presences.add(serviceVersion);
ipData.sessions++;
ips.set(ip, ipData);
}
});
} while (cursor !== "0");
log?.("Updating active presence gauge");
//* Batch update the gauge
gauge.reset();
for (const [serviceVersion, count] of serviceCounts) {
const [presence_name, version] = serviceVersion.split(":");
gauge.set({ presence_name, version }, count);
}
//* Convert IP data for insertion
const ipDataForInsertion = new Map(
Array.from(ips, ([ip, data]) => [ip, {
presences: Array.from(data.presences),
sessions: data.sessions,
}]),
);
await insertIpData(ipDataForInsertion);
log?.("Active presence gauge update completed");
});
}

View File

@@ -1,33 +1,44 @@
import process from "node:process";
import { CronJob } from "cron";
import debug from "debug";
import { clearOldSessions } from "./functions/clearOldSessions.js";
import createRedis from "./functions/createRedis.js";
import { setCounter } from "./functions/setCounter.js";
import "./tracing.js";
import { reloadIpLocationApi } from "./functions/lookupIp.js";
import { cleanupOldUserData } from "./functions/cleanupOldUserData.js";
import { setupServer } from "./functions/setupServer.js";
export const redis = createRedis();
export const server = setupServer();
export const mainLog = debug("api-master");
debug("Starting cron job to clear old sessions");
debug("Starting cron jobs");
void reloadIpLocationApi();
void new CronJob(
// Every 5 seconds
"*/5 * * * * *",
() => {
clearOldSessions();
if (process.env.DISABLE_CLEAR_OLD_SESSIONS !== "true") {
clearOldSessions();
}
},
undefined,
true,
);
void new CronJob(
// Every second
"* * * * * *",
// Every day at 1am
"0 1 * * *",
() => {
setCounter();
cleanupOldUserData(14); // Keep 14 days of data
},
undefined,
true,
undefined,
undefined,
true,
);

View File

@@ -1,18 +1,41 @@
import { ValueType } from "@opentelemetry/api";
import { PrometheusExporter } from "@opentelemetry/exporter-prometheus";
import { MeterProvider } from "@opentelemetry/sdk-metrics";
import process from "node:process";
import { Counter, Gauge, Registry, collectDefaultMetrics } from "prom-client";
import { updateActivePresenceGauge, updateActivePresenceGaugeLimit } from "./functions/updateActivePresenceGauge.js";
import { redis } from "./index.js";
const prometheusExporter = new PrometheusExporter();
const scanCount = Number.parseInt(process.env.SCAN_COUNT || "1000", 10);
const provider = new MeterProvider({
readers: [prometheusExporter],
export const register = new Registry();
collectDefaultMetrics({ register });
export const activeSessionsCounter = new Counter({
name: "active_sessions",
help: "Number of active sessions",
async collect() {
this.reset();
let length = 0;
let cursor = "0";
do {
const reply = await redis.scan(cursor, "MATCH", "pmd-api.sessions.*", "COUNT", scanCount);
cursor = reply[0];
length += reply[1].length;
} while (cursor !== "0");
this.inc(length);
},
});
const meter = provider.getMeter("nice");
export const counter = meter.createUpDownCounter("active_activites", {
description: "Number of active activities",
valueType: ValueType.INT,
export const activePresencesCounter = new Gauge({
name: "active_presences",
help: "Number of active presences",
labelNames: ["presence_name", "version"],
async collect() {
if (process.env.DISABLE_ACTIVE_PRESENCE_GAUGE !== "true") {
this.reset();
updateActivePresenceGaugeLimit.clearQueue();
await updateActivePresenceGauge(this);
}
},
});
prometheusExporter.startServer();
register.registerMetric(activeSessionsCounter);
register.registerMetric(activePresencesCounter);

View File

@@ -3,6 +3,7 @@
"compilerOptions": {
"composite": true,
"rootDir": "src",
"types": ["./environment.d.ts"],
"outDir": "dist"
},
"include": ["src/**/*"]

View File

@@ -1,7 +1,7 @@
{
"name": "@premid/api-worker",
"type": "module",
"version": "0.0.9",
"version": "0.0.14",
"private": true,
"description": "PreMiD's api",
"license": "MPL-2.0",

View File

@@ -0,0 +1,10 @@
import process from "node:process";
import { defu } from "defu";
const disabledFlags = process.env.DISABLED_FEATURE_FLAGS?.split(",") ?? [];
const flags = Object.fromEntries(disabledFlags.map(flag => [flag, false]));
export const featureFlags = defu(flags, {
WebSocketManager: true,
SessionKeepAlive: true,
});

View File

@@ -1,13 +1,10 @@
import { readFile } from "node:fs/promises";
import { resolve } from "node:path";
import process from "node:process";
import { useSentry } from "@envelop/sentry";
import { maxAliasesPlugin } from "@escape.tech/graphql-armor-max-aliases";
import { maxDepthPlugin } from "@escape.tech/graphql-armor-max-depth";
import { maxDirectivesPlugin } from "@escape.tech/graphql-armor-max-directives";
import { maxTokensPlugin } from "@escape.tech/graphql-armor-max-tokens";
import fastifyWebsocket from "@fastify/websocket";
import { defu } from "defu";
import fastify from "fastify";
import { createSchema, createYoga } from "graphql-yoga";
@@ -15,6 +12,7 @@ import type { FastifyReply, FastifyRequest } from "fastify";
import { Socket } from "../classes/Socket.js";
import { resolvers } from "../graphql/resolvers/v5/index.js";
import { sessionKeepAlive } from "../routes/sessionKeepAlive.js";
import { featureFlags } from "../constants.js";
import createRedis from "./createRedis.js";
export interface FastifyContext {
@@ -48,7 +46,7 @@ export default async function createServer() {
maxDepthPlugin(),
maxDirectivesPlugin(),
maxTokensPlugin(),
useSentry(),
/* useSentry(), */
],
schema: createSchema<FastifyContext>({
resolvers,
@@ -87,15 +85,7 @@ export default async function createServer() {
});
app.get("/v5/feature-flags", async (request, reply) => {
const disabledFlags = process.env.DISABLED_FEATURE_FLAGS?.split(",") ?? [];
const flags = Object.fromEntries(disabledFlags.map(flag => [flag, false]));
const test = defu(flags, {
WebSocketManager: true,
SessionKeepAlive: true,
});
void reply.send(test);
void reply.send(featureFlags);
});
app.post("/v5/session-keep-alive", sessionKeepAlive);

View File

@@ -1,4 +1,5 @@
import { type } from "arktype";
import { GraphQLError } from "graphql";
import { redis } from "../../../../functions/createServer.js";
import type { MutationResolvers } from "../../../../generated/graphql-v5.js";
@@ -15,7 +16,7 @@ const mutation: MutationResolvers["addScience"] = async (_parent, input) => {
const out = addScienceSchema(input);
if (out instanceof type.errors)
throw new Error(out.summary);
throw new GraphQLError(out.summary);
await redis.hset(
"pmd-api.scienceUpdates",

View File

@@ -1,15 +1,17 @@
import { type } from "arktype";
import { GraphQLError } from "graphql";
import type { MutationResolvers } from "../../../../generated/graphql-v5.js";
import { redis } from "../../../../functions/createServer.js";
const heartbeatSchema = type({
identifier: "string.uuid & string.lower",
presences: {
"identifier": "string.uuid & string.lower",
"presence?": {
service: "string.trim",
version: "string.semver",
language: "string.trim",
since: "number.epoch",
},
extension: {
"extension": {
"version": "string.semver",
"language": "string.trim",
"connected?": {
@@ -19,19 +21,29 @@ const heartbeatSchema = type({
},
});
const mutation: MutationResolvers["heartbeat"] = async (_parent, input) => {
const mutation: MutationResolvers["heartbeat"] = async (_parent, input, context) => {
const out = heartbeatSchema(input);
if (out instanceof type.errors)
throw new Error(out.summary);
throw new GraphQLError(out.summary);
// ! Disabled for now
/* await redis.setex(
`pmd-api.heartbeatUpdates.${data.identifier}`,
// 5 minutes
300,
JSON.stringify(data)
); */
//* Get the user's IP address from Cloudflare headers or fallback to the request IP
const userIp = context.request.headers.get("cf-connecting-ip") || context.request.ip;
// * Use Redis Hash with 'service' in the key to store heartbeat data
const redisKey = `pmd-api.heartbeatUpdates.${out.identifier}`;
await redis.hset(redisKey, {
service: out.presence?.service,
version: out.presence?.version,
language: out.presence?.language,
since: out.presence?.since.toString(),
extension_version: out.extension.version,
extension_language: out.extension.language,
extension_connected_app: out.extension.connected?.app?.toString(),
extension_connected_discord: out.extension.connected?.discord?.toString(),
ip_address: userIp,
});
await redis.expire(redisKey, 300);
return {
__typename: "HeartbeatResult",

View File

@@ -3,8 +3,6 @@ import process from "node:process";
import * as Sentry from "@sentry/node";
import { connect } from "mongoose";
import "./tracing.js";
// eslint-disable-next-line perfectionist/sort-imports
import createServer from "./functions/createServer.js";
// TODO SETUP SENTRY

View File

@@ -4,6 +4,7 @@ import { type } from "arktype";
import { Routes } from "discord-api-types/v10";
import type { FastifyReply, FastifyRequest } from "fastify";
import { redis } from "../functions/createServer.js";
import { featureFlags } from "../constants.js";
const schema = type({
token: "string.trim",
@@ -13,6 +14,9 @@ const schema = type({
});
export async function sessionKeepAlive(request: FastifyRequest, reply: FastifyReply) {
if (!featureFlags.SessionKeepAlive)
return reply.status(202).send();
//* Get the headers
const out = schema({
token: request.headers["x-token"],
@@ -27,17 +31,15 @@ export async function sessionKeepAlive(request: FastifyRequest, reply: FastifyRe
if (!await isTokenValid(out.token))
return reply.status(400).send({ code: "INVALID_TOKEN", message: "The token is invalid" });
await redis.hset(
"pmd-api.sessions",
out.scienceId,
JSON.stringify({
session: out.session,
token: out.token,
lastUpdated: Date.now(),
}),
);
const redisKey = `pmd-api.sessions.${out.scienceId}`;
await redis.hset(redisKey, {
session: out.session,
token: out.token,
lastUpdated: Date.now(),
});
await redis.expire(redisKey, 300); // 5 minutes
const interval = Number.parseInt(process.env.SESSION_KEEP_ALIVE_INTERVAL ?? "5000");
const interval = Number.parseInt(process.env.SESSION_KEEP_ALIVE_INTERVAL ?? "5000"); // 5 seconds
return reply.status(200).send({
code: "OK",

View File

@@ -1,7 +1,7 @@
{
"name": "@premid/schema-server",
"type": "module",
"version": "1.0.3",
"version": "1.0.4",
"private": true,
"description": "A small service to serve the JSON schemas for PreMiD",
"license": "MPL-2.0",

View File

@@ -0,0 +1,260 @@
{
"$schema": "http://json-schema.org/draft-07/schema",
"$id": "https://schemas.premid.app/metadata/1.11",
"title": "Metadata",
"type": "object",
"description": "Metadata that describes a presence.",
"definitions": {
"user": {
"type": "object",
"description": "User information.",
"properties": {
"name": {
"type": "string",
"description": "The name of the user."
},
"id": {
"type": "string",
"description": "The Discord snowflake of the user.",
"pattern": "^\\d+$"
}
},
"additionalProperties": false,
"required": [
"name",
"id"
]
}
},
"properties": {
"$schema": {
"$comment": "This is required otherwise the schema will fail itself when it is applied to a document via $schema. This is optional so that validators that use this schema don't fail if the metadata doesn't have the $schema property.",
"type": "string",
"description": "The metadata schema URL."
},
"author": {
"$ref": "#/definitions/user",
"description": "The author of this presence."
},
"contributors": {
"type": "array",
"description": "Any extra contributors to this presence.",
"items": {
"$ref": "#/definitions/user"
}
},
"service": {
"type": "string",
"description": "The service this presence is for."
},
"altnames": {
"type": "array",
"description": "Alternative names for the service.",
"items": {
"type": "string",
"description": "An alternative name."
},
"minItems": 1
},
"description": {
"type": "object",
"description": "A description of the presence in multiple languages.",
"propertyNames": {
"type": "string",
"description": "The language key. The key must be languagecode(_REGIONCODE).",
"pattern": "^[a-z]{2}(?:_(?:[A-Z]{2}|[0-9]{1,3}))?$"
},
"patternProperties": {
"^[a-z]{2}(?:_(?:[A-Z]{2}|[0-9]{1,3}))?$": {
"type": "string",
"description": "The description of the presence in the key's language."
}
},
"additionalProperties": false,
"required": [
"en"
]
},
"url": {
"type": [
"string",
"array"
],
"description": "The service's website URL, or an array of URLs. Protocols should not be added.",
"pattern": "^(([a-z0-9-]+\\.)*[0-9a-z_-]+(\\.[a-z]+)+|(\\d{1,3}\\.){3}\\d{1,3}|localhost)$",
"items": {
"type": "string",
"description": "One of the service's website URLs.",
"pattern": "^(([a-z0-9-]+\\.)*[0-9a-z_-]+(\\.[a-z]+)+|(\\d{1,3}\\.){3}\\d{1,3}|localhost)$"
},
"minItems": 2
},
"version": {
"type": "string",
"description": "The SemVer version of the presence. Must just be major.minor.patch.",
"pattern": "^\\d+\\.\\d+\\.\\d+$"
},
"apiVersion": {
"type": "integer",
"description": "The Presence System version this Presence supports.",
"minimum": 1,
"maximum": 2
},
"logo": {
"type": "string",
"description": "The logo of the service this presence is for.",
"pattern": "^https?://.+\\.(png|jpe?g|gif|webp)$"
},
"thumbnail": {
"type": "string",
"description": "A thumbnail of the service this presence is for.",
"pattern": "^https?://.+\\.(png|jpe?g|gif|webp)$"
},
"color": {
"type": "string",
"description": "The theme color of the service this presence is for. Must be either a 6 digit or a 3 digit hex code.",
"pattern": "^#([A-Fa-f0-9]{3}){1,2}$"
},
"tags": {
"type": [
"array"
],
"description": "The tags for the presence.",
"items": {
"type": "string",
"description": "A tag.",
"pattern": "^[^A-Z\\s!\"#$%&'()*+,./:;<=>?@\\[\\\\\\]^_`{|}~]+$"
},
"minItems": 1
},
"category": {
"type": "string",
"description": "The category the presence falls under.",
"enum": [
"anime",
"games",
"music",
"socials",
"videos",
"other"
]
},
"iframe": {
"type": "boolean",
"description": "Whether or not the presence should run in IFrames."
},
"readLogs": {
"type": "boolean",
"description": "Whether or not the extension should be reading logs."
},
"regExp": {
"type": "string",
"description": "A regular expression used to match URLs for the presence to inject into."
},
"iFrameRegExp": {
"type": "string",
"description": "A regular expression used to match IFrames for the presence to inject into."
},
"button": {
"type": "boolean",
"description": "Controls whether the presence is automatically added when the extension is installed. For partner presences only."
},
"warning": {
"type": "boolean",
"description": "Shows a warning saying that it requires additional steps for the presence to function correctly."
},
"settings": {
"type": "array",
"description": "An array of settings the user can change in the presence.",
"items": {
"type": "object",
"description": "A setting.",
"properties": {
"id": {
"type": "string",
"description": "The ID of the setting."
},
"title": {
"type": "string",
"description": "The title of the setting. Required only if `multiLanguage` is disabled."
},
"icon": {
"type": "string",
"description": "The icon of the setting. Required only if `multiLanguage` is disabled.",
"pattern": "^fa([bsdrlt]|([-](brands|solid|duotone|regular|light|thin))) fa-[0-9a-z-]+$"
},
"if": {
"type": "object",
"description": "Restrict showing this setting if another setting is the defined value.",
"propertyNames": {
"type": "string",
"description": "The ID of the setting."
},
"patternProperties": {
"": {
"type": [
"string",
"number",
"boolean"
],
"description": "The value of the setting."
}
},
"additionalProperties": false
},
"placeholder": {
"type": "string",
"description": "The placeholder for settings that require input. Shown when the input is empty."
},
"value": {
"type": [
"string",
"number",
"boolean"
],
"description": "The default value of the setting. Not compatible with `values`."
},
"values": {
"type": "array",
"description": "The default values of the setting. Not compatible with `value`.",
"items": {
"type": [
"string",
"number",
"boolean"
],
"description": "The value of the setting."
}
},
"multiLanguage": {
"type": [
"string",
"boolean",
"array"
],
"description": "When false, multi-localization is disabled. When true, strings from the `general.json` file are available for use. When a string, it is the name of a file (excluding .json) of a used language from the localization GitHub repo. When an array of strings, it is all of the file names (excluding .json) of used languages from the localization GitHub repo.",
"items": {
"type": "string",
"description": "The name of a file from the localization GitHub repository."
}
}
},
"additionalProperties": false
}
}
},
"additionalProperties": false,
"required": [
"author",
"service",
"description",
"url",
"version",
"apiVersion",
"logo",
"thumbnail",
"color",
"tags",
"category"
]
}

View File

@@ -22,7 +22,6 @@ export const useExtensionStore = defineStore("extension", () => {
}
}
// eslint-disable-next-line unicorn/consistent-function-scoping
function fetchPresences() {
window.dispatchEvent(new CustomEvent("PreMiD_GetPresenceList"));
}

1187
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

View File

@@ -29,5 +29,5 @@
"isolatedModules": true,
"skipLibCheck": true
},
"exclude": ["**/*/node_modules", "**/*/dist"]
"exclude": ["**/*/node_modules", "**/*/dist", "**/drizzle.config.ts"]
}