chore: release v0.0.30

.gitignore

@@ -3,6 +3,7 @@ out
 dist
 tmp
 lib
+data
 
 .vscode
 .env
@@ -22,4 +23,5 @@ src/update.ini
 !eslint.config.js
 
 coverage
-*.tsbuildinfo
+*.tsbuildinfo
+.DS_Store

.prettierignore

@@ -1,2 +1,3 @@
 *.js
-*.ts
+*.ts
+*.json

apps/api-master/drizzle.config.ts

@@ -0,0 +1,10 @@
+import { defineConfig } from "drizzle-kit";
+
+export default defineConfig({
+	dbCredentials: {
+		url: "postgresql://metrics:metrics@localhost:5432/metrics",
+	},
+	dialect: "postgresql",
+	schema: "./src/db.ts",
+	out: "./drizzle",
+});

apps/api-master/drizzle/0000_flippant_marrow.sql

@@ -0,0 +1,9 @@
+CREATE TABLE IF NOT EXISTS "online_users_ip_data" (
+	"uuid" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"ip" varchar(45) NOT NULL,
+	"country" varchar(2) NOT NULL,
+	"latitude" numeric(10, 8) NOT NULL,
+	"longitude" numeric(11, 8) NOT NULL,
+	"name" varchar(255),
+	"timestamp" timestamp DEFAULT now()
+);

apps/api-master/drizzle/0001_white_lifeguard.sql

@@ -0,0 +1,2 @@
+CREATE INDEX IF NOT EXISTS "idx_online_users_uuid" ON "online_users_ip_data" USING btree ("uuid");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "idx_online_users_timestamp" ON "online_users_ip_data" USING btree ("timestamp");

apps/api-master/drizzle/0002_new_darkhawk.sql

@@ -0,0 +1 @@
+ALTER TABLE "online_users_ip_data" ALTER COLUMN "timestamp" SET DATA TYPE timestamp with time zone;

apps/api-master/drizzle/0003_narrow_mastermind.sql

@@ -0,0 +1,2 @@
+ALTER TABLE "online_users_ip_data" ADD COLUMN "presences" jsonb DEFAULT '[]' NOT NULL;--> statement-breakpoint
+ALTER TABLE "online_users_ip_data" DROP COLUMN IF EXISTS "name";

apps/api-master/drizzle/0004_tiresome_puff_adder.sql

@@ -0,0 +1 @@
+ALTER TABLE "online_users_ip_data" ADD COLUMN "sessions" integer DEFAULT 0 NOT NULL;

apps/api-master/drizzle/meta/0000_snapshot.json

@@ -0,0 +1,70 @@
+{
+	"id": "e29a6708-01f1-455a-b345-63dac1e124dc",
+	"prevId": "00000000-0000-0000-0000-000000000000",
+	"version": "7",
+	"dialect": "postgresql",
+	"tables": {
+		"public.online_users_ip_data": {
+			"name": "online_users_ip_data",
+			"schema": "",
+			"columns": {
+				"uuid": {
+					"name": "uuid",
+					"type": "uuid",
+					"primaryKey": true,
+					"notNull": true,
+					"default": "gen_random_uuid()"
+				},
+				"ip": {
+					"name": "ip",
+					"type": "varchar(45)",
+					"primaryKey": false,
+					"notNull": true
+				},
+				"country": {
+					"name": "country",
+					"type": "varchar(2)",
+					"primaryKey": false,
+					"notNull": true
+				},
+				"latitude": {
+					"name": "latitude",
+					"type": "numeric(10, 8)",
+					"primaryKey": false,
+					"notNull": true
+				},
+				"longitude": {
+					"name": "longitude",
+					"type": "numeric(11, 8)",
+					"primaryKey": false,
+					"notNull": true
+				},
+				"name": {
+					"name": "name",
+					"type": "varchar(255)",
+					"primaryKey": false,
+					"notNull": false
+				},
+				"timestamp": {
+					"name": "timestamp",
+					"type": "timestamp",
+					"primaryKey": false,
+					"notNull": false,
+					"default": "now()"
+				}
+			},
+			"indexes": {},
+			"foreignKeys": {},
+			"compositePrimaryKeys": {},
+			"uniqueConstraints": {}
+		}
+	},
+	"enums": {},
+	"schemas": {},
+	"sequences": {},
+	"_meta": {
+		"columns": {},
+		"schemas": {},
+		"tables": {}
+	}
+}

apps/api-master/drizzle/meta/0001_snapshot.json

@@ -0,0 +1,101 @@
+{
+	"id": "4aa32a8e-f573-43b9-976a-2d078a0df0ea",
+	"prevId": "e29a6708-01f1-455a-b345-63dac1e124dc",
+	"version": "7",
+	"dialect": "postgresql",
+	"tables": {
+		"public.online_users_ip_data": {
+			"name": "online_users_ip_data",
+			"schema": "",
+			"columns": {
+				"uuid": {
+					"name": "uuid",
+					"type": "uuid",
+					"primaryKey": true,
+					"notNull": true,
+					"default": "gen_random_uuid()"
+				},
+				"ip": {
+					"name": "ip",
+					"type": "varchar(45)",
+					"primaryKey": false,
+					"notNull": true
+				},
+				"country": {
+					"name": "country",
+					"type": "varchar(2)",
+					"primaryKey": false,
+					"notNull": true
+				},
+				"latitude": {
+					"name": "latitude",
+					"type": "numeric(10, 8)",
+					"primaryKey": false,
+					"notNull": true
+				},
+				"longitude": {
+					"name": "longitude",
+					"type": "numeric(11, 8)",
+					"primaryKey": false,
+					"notNull": true
+				},
+				"name": {
+					"name": "name",
+					"type": "varchar(255)",
+					"primaryKey": false,
+					"notNull": false
+				},
+				"timestamp": {
+					"name": "timestamp",
+					"type": "timestamp",
+					"primaryKey": false,
+					"notNull": false,
+					"default": "now()"
+				}
+			},
+			"indexes": {
+				"idx_online_users_uuid": {
+					"name": "idx_online_users_uuid",
+					"columns": [
+						{
+							"expression": "uuid",
+							"isExpression": false,
+							"asc": true,
+							"nulls": "last"
+						}
+					],
+					"isUnique": false,
+					"concurrently": false,
+					"method": "btree",
+					"with": {}
+				},
+				"idx_online_users_timestamp": {
+					"name": "idx_online_users_timestamp",
+					"columns": [
+						{
+							"expression": "timestamp",
+							"isExpression": false,
+							"asc": true,
+							"nulls": "last"
+						}
+					],
+					"isUnique": false,
+					"concurrently": false,
+					"method": "btree",
+					"with": {}
+				}
+			},
+			"foreignKeys": {},
+			"compositePrimaryKeys": {},
+			"uniqueConstraints": {}
+		}
+	},
+	"enums": {},
+	"schemas": {},
+	"sequences": {},
+	"_meta": {
+		"columns": {},
+		"schemas": {},
+		"tables": {}
+	}
+}

apps/api-master/drizzle/meta/0002_snapshot.json

@@ -0,0 +1,101 @@
+{
+	"id": "c1b8dbed-b232-4d66-9e74-b9af333095bc",
+	"prevId": "4aa32a8e-f573-43b9-976a-2d078a0df0ea",
+	"version": "7",
+	"dialect": "postgresql",
+	"tables": {
+		"public.online_users_ip_data": {
+			"name": "online_users_ip_data",
+			"schema": "",
+			"columns": {
+				"uuid": {
+					"name": "uuid",
+					"type": "uuid",
+					"primaryKey": true,
+					"notNull": true,
+					"default": "gen_random_uuid()"
+				},
+				"ip": {
+					"name": "ip",
+					"type": "varchar(45)",
+					"primaryKey": false,
+					"notNull": true
+				},
+				"country": {
+					"name": "country",
+					"type": "varchar(2)",
+					"primaryKey": false,
+					"notNull": true
+				},
+				"latitude": {
+					"name": "latitude",
+					"type": "numeric(10, 8)",
+					"primaryKey": false,
+					"notNull": true
+				},
+				"longitude": {
+					"name": "longitude",
+					"type": "numeric(11, 8)",
+					"primaryKey": false,
+					"notNull": true
+				},
+				"name": {
+					"name": "name",
+					"type": "varchar(255)",
+					"primaryKey": false,
+					"notNull": false
+				},
+				"timestamp": {
+					"name": "timestamp",
+					"type": "timestamp with time zone",
+					"primaryKey": false,
+					"notNull": false,
+					"default": "now()"
+				}
+			},
+			"indexes": {
+				"idx_online_users_uuid": {
+					"name": "idx_online_users_uuid",
+					"columns": [
+						{
+							"expression": "uuid",
+							"isExpression": false,
+							"asc": true,
+							"nulls": "last"
+						}
+					],
+					"isUnique": false,
+					"concurrently": false,
+					"method": "btree",
+					"with": {}
+				},
+				"idx_online_users_timestamp": {
+					"name": "idx_online_users_timestamp",
+					"columns": [
+						{
+							"expression": "timestamp",
+							"isExpression": false,
+							"asc": true,
+							"nulls": "last"
+						}
+					],
+					"isUnique": false,
+					"concurrently": false,
+					"method": "btree",
+					"with": {}
+				}
+			},
+			"foreignKeys": {},
+			"compositePrimaryKeys": {},
+			"uniqueConstraints": {}
+		}
+	},
+	"enums": {},
+	"schemas": {},
+	"sequences": {},
+	"_meta": {
+		"columns": {},
+		"schemas": {},
+		"tables": {}
+	}
+}

apps/api-master/drizzle/meta/0003_snapshot.json

@@ -0,0 +1,102 @@
+{
+	"id": "e409a4d0-f698-484a-b412-38966a7b3a19",
+	"prevId": "c1b8dbed-b232-4d66-9e74-b9af333095bc",
+	"version": "7",
+	"dialect": "postgresql",
+	"tables": {
+		"public.online_users_ip_data": {
+			"name": "online_users_ip_data",
+			"schema": "",
+			"columns": {
+				"uuid": {
+					"name": "uuid",
+					"type": "uuid",
+					"primaryKey": true,
+					"notNull": true,
+					"default": "gen_random_uuid()"
+				},
+				"ip": {
+					"name": "ip",
+					"type": "varchar(45)",
+					"primaryKey": false,
+					"notNull": true
+				},
+				"country": {
+					"name": "country",
+					"type": "varchar(2)",
+					"primaryKey": false,
+					"notNull": true
+				},
+				"latitude": {
+					"name": "latitude",
+					"type": "numeric(10, 8)",
+					"primaryKey": false,
+					"notNull": true
+				},
+				"longitude": {
+					"name": "longitude",
+					"type": "numeric(11, 8)",
+					"primaryKey": false,
+					"notNull": true
+				},
+				"presences": {
+					"name": "presences",
+					"type": "jsonb",
+					"primaryKey": false,
+					"notNull": true,
+					"default": "'[]'"
+				},
+				"timestamp": {
+					"name": "timestamp",
+					"type": "timestamp with time zone",
+					"primaryKey": false,
+					"notNull": false,
+					"default": "now()"
+				}
+			},
+			"indexes": {
+				"idx_online_users_uuid": {
+					"name": "idx_online_users_uuid",
+					"columns": [
+						{
+							"expression": "uuid",
+							"isExpression": false,
+							"asc": true,
+							"nulls": "last"
+						}
+					],
+					"isUnique": false,
+					"concurrently": false,
+					"method": "btree",
+					"with": {}
+				},
+				"idx_online_users_timestamp": {
+					"name": "idx_online_users_timestamp",
+					"columns": [
+						{
+							"expression": "timestamp",
+							"isExpression": false,
+							"asc": true,
+							"nulls": "last"
+						}
+					],
+					"isUnique": false,
+					"concurrently": false,
+					"method": "btree",
+					"with": {}
+				}
+			},
+			"foreignKeys": {},
+			"compositePrimaryKeys": {},
+			"uniqueConstraints": {}
+		}
+	},
+	"enums": {},
+	"schemas": {},
+	"sequences": {},
+	"_meta": {
+		"columns": {},
+		"schemas": {},
+		"tables": {}
+	}
+}

apps/api-master/drizzle/meta/0004_snapshot.json

@@ -0,0 +1,109 @@
+{
+	"id": "179435b5-dc15-4a42-9539-c3f336699d63",
+	"prevId": "e409a4d0-f698-484a-b412-38966a7b3a19",
+	"version": "7",
+	"dialect": "postgresql",
+	"tables": {
+		"public.online_users_ip_data": {
+			"name": "online_users_ip_data",
+			"schema": "",
+			"columns": {
+				"uuid": {
+					"name": "uuid",
+					"type": "uuid",
+					"primaryKey": true,
+					"notNull": true,
+					"default": "gen_random_uuid()"
+				},
+				"ip": {
+					"name": "ip",
+					"type": "varchar(45)",
+					"primaryKey": false,
+					"notNull": true
+				},
+				"country": {
+					"name": "country",
+					"type": "varchar(2)",
+					"primaryKey": false,
+					"notNull": true
+				},
+				"latitude": {
+					"name": "latitude",
+					"type": "numeric(10, 8)",
+					"primaryKey": false,
+					"notNull": true
+				},
+				"longitude": {
+					"name": "longitude",
+					"type": "numeric(11, 8)",
+					"primaryKey": false,
+					"notNull": true
+				},
+				"sessions": {
+					"name": "sessions",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": true,
+					"default": 0
+				},
+				"presences": {
+					"name": "presences",
+					"type": "jsonb",
+					"primaryKey": false,
+					"notNull": true,
+					"default": "'[]'"
+				},
+				"timestamp": {
+					"name": "timestamp",
+					"type": "timestamp with time zone",
+					"primaryKey": false,
+					"notNull": false,
+					"default": "now()"
+				}
+			},
+			"indexes": {
+				"idx_online_users_uuid": {
+					"name": "idx_online_users_uuid",
+					"columns": [
+						{
+							"expression": "uuid",
+							"isExpression": false,
+							"asc": true,
+							"nulls": "last"
+						}
+					],
+					"isUnique": false,
+					"concurrently": false,
+					"method": "btree",
+					"with": {}
+				},
+				"idx_online_users_timestamp": {
+					"name": "idx_online_users_timestamp",
+					"columns": [
+						{
+							"expression": "timestamp",
+							"isExpression": false,
+							"asc": true,
+							"nulls": "last"
+						}
+					],
+					"isUnique": false,
+					"concurrently": false,
+					"method": "btree",
+					"with": {}
+				}
+			},
+			"foreignKeys": {},
+			"compositePrimaryKeys": {},
+			"uniqueConstraints": {}
+		}
+	},
+	"enums": {},
+	"schemas": {},
+	"sequences": {},
+	"_meta": {
+		"columns": {},
+		"schemas": {},
+		"tables": {}
+	}
+}

apps/api-master/drizzle/meta/_journal.json

@@ -0,0 +1,41 @@
+{
+	"version": "7",
+	"dialect": "postgresql",
+	"entries": [
+		{
+			"idx": 0,
+			"version": "7",
+			"when": 1726516195146,
+			"tag": "0000_flippant_marrow",
+			"breakpoints": true
+		},
+		{
+			"idx": 1,
+			"version": "7",
+			"when": 1726516348344,
+			"tag": "0001_white_lifeguard",
+			"breakpoints": true
+		},
+		{
+			"idx": 2,
+			"version": "7",
+			"when": 1726516660134,
+			"tag": "0002_new_darkhawk",
+			"breakpoints": true
+		},
+		{
+			"idx": 3,
+			"version": "7",
+			"when": 1726517073510,
+			"tag": "0003_narrow_mastermind",
+			"breakpoints": true
+		},
+		{
+			"idx": 4,
+			"version": "7",
+			"when": 1726517405363,
+			"tag": "0004_tiresome_puff_adder",
+			"breakpoints": true
+		}
+	]
+}

apps/api-master/environment.d.ts

@@ -0,0 +1,16 @@
+declare module "ip-location-api" {
+	export function lookup(ip: string): Promise<{
+		latitude: number;
+		longitude: number;
+		country: string;
+	} | null>;
+
+	export function updateDb(options: { fields?: string[]; dataDir?: string; tmpDataDir?: string; smallMemory?: boolean }): Promise<void>;
+	export function reload(options: { fields?: string[]; dataDir?: string; tmpDataDir?: string; smallMemory?: boolean }): Promise<void>;
+}
+
+declare namespace NodeJS {
+	export interface ProcessEnv {
+		METRICS_DATABASE_URL?: string;
+	}
+}

apps/api-master/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@premid/api-master",
 	"type": "module",
-	"version": "0.0.24",
+	"version": "0.0.30",
 	"private": true,
 	"description": "PreMiD's api master",
 	"license": "MPL-2.0",
@@ -11,7 +11,11 @@
 	],
 	"scripts": {
 		"start": "node --enable-source-maps .",
-		"dev": "node --watch --env-file .env --enable-source-maps ."
+		"dev": "node --watch --env-file .env --enable-source-maps .",
+		"db:generate": "drizzle-kit generate",
+		"db:migrate": "drizzle-kit migrate",
+		"db:update": "pnpm db:generate && pnpm db:migrate",
+		"db:studio": "drizzle-kit studio"
 	},
 	"dependencies": {
 		"@envelop/sentry": "^9.0.0",
@@ -21,11 +25,15 @@
 		"@sentry/node": "^8.17.0",
 		"cron": "^3.1.7",
 		"debug": "^4.3.6",
+		"drizzle-orm": "^0.33.0",
 		"ioredis": "^5.3.2",
+		"ip-location-api": "^1.0.0",
 		"ky": "^1.7.2",
-		"p-limit": "^6.1.0"
+		"p-limit": "^6.1.0",
+		"postgres": "^3.4.4"
 	},
 	"devDependencies": {
-		"@types/debug": "^4.1.12"
+		"@types/debug": "^4.1.12",
+		"drizzle-kit": "^0.24.2"
 	}
 }

apps/api-master/src/db.ts

@@ -0,0 +1,27 @@
+import process from "node:process";
+import { decimal, index, integer, jsonb, pgTable, timestamp, uuid, varchar } from "drizzle-orm/pg-core";
+import { drizzle } from "drizzle-orm/postgres-js";
+import postgres from "postgres";
+
+// Define the schema
+export const onlineUsersIpData = pgTable("online_users_ip_data", {
+	uuid: uuid("uuid").primaryKey().defaultRandom(),
+	ip: varchar("ip", { length: 45 }).notNull(),
+	country: varchar("country", { length: 2 }).notNull(),
+	latitude: decimal("latitude", { precision: 10, scale: 8 }).notNull(),
+	longitude: decimal("longitude", { precision: 11, scale: 8 }).notNull(),
+	sessions: integer("sessions").notNull().default(0),
+	presences: jsonb("presences").notNull().default("[]").$type<string[]>(),
+	timestamp: timestamp("timestamp", { withTimezone: true }).defaultNow(),
+}, table => ({
+	idxOnlineUsersUuid: index("idx_online_users_uuid").on(table.uuid),
+	idxOnlineUsersTimestamp: index("idx_online_users_timestamp").on(table.timestamp),
+}));
+
+if (!process.env.METRICS_DATABASE_URL) {
+	throw new Error("METRICS_DATABASE_URL is not set");
+}
+
+export const sql = postgres(process.env.METRICS_DATABASE_URL);
+
+export const db = drizzle(sql);

apps/api-master/src/functions/cleanupOldUserData.ts

@@ -0,0 +1,10 @@
+import { lt, sql } from "drizzle-orm";
+import { db, onlineUsersIpData } from "../db.js";
+import { mainLog } from "../index.js";
+
+export async function cleanupOldUserData(retentionDays: number) {
+	mainLog("Cleaning up old user ip data");
+	const interval = `'${retentionDays} days'`;
+	await db.delete(onlineUsersIpData)
+		.where(lt(onlineUsersIpData.timestamp, sql`now() - interval ${sql.raw(interval)}`));
+}

apps/api-master/src/functions/clearOldSessions.ts

@@ -11,6 +11,7 @@ export async function clearOldSessions() {
 
 	inProgress = true;
 	const now = Date.now();
+	const pattern = "pmd-api.sessions.*";
 	let cursor = "0";
 	let totalSessions = 0;
 	let cleared = 0;
@@ -22,21 +23,15 @@ export async function clearOldSessions() {
 	const limit = pLimit(100); // Create a limit of 100 concurrent operations
 
 	do {
-		const [nextCursor, result] = await redis.hscan("pmd-api.sessions", cursor, "COUNT", batchSize);
-		cursor = nextCursor;
-		totalSessions += result.length / 2;
+		const [newCursor, keys] = await redis.scan(cursor, "MATCH", pattern, "COUNT", 1000); //* Use SCAN with COUNT for memory efficiency
 
-		const deletePromises = [];
+		cursor = newCursor;
+		totalSessions += keys.length;
 
-		for (let i = 0; i < result.length; i += 2) {
-			const key = result[i];
-			const value = result[i + 1];
+		const deletePromises: Promise<string>[] = [];
 
-			if (!key || !value) {
-				continue;
-			}
-
-			const session = JSON.parse(value) as {
+		for (const key of keys) {
+			const session = await redis.hgetall(key) as unknown as {
 				token: string;
 				session: string;
 				lastUpdated: number;
@@ -57,13 +52,13 @@ export async function clearOldSessions() {
 		});
 
 		if (keysToDelete.length >= batchSize) {
-			await redis.hdel("pmd-api.sessions", ...keysToDelete);
+			await redis.del(...keysToDelete);
 			keysToDelete = [];
 		}
 	} while (cursor !== "0");
 
 	if (keysToDelete.length > 0) {
-		await redis.hdel("pmd-api.sessions", ...keysToDelete);
+		await redis.del(...keysToDelete);
 	}
 
 	if (totalSessions === 0) {

apps/api-master/src/functions/clearableGaugeMetric.ts

@@ -0,0 +1,83 @@
+import type { ServerResponse } from "node:http";
+import type { Attributes } from "@opentelemetry/api";
+import { ValueType, diag } from "@opentelemetry/api";
+import type { PrometheusExporter, PrometheusSerializer } from "@opentelemetry/exporter-prometheus";
+import { AggregationTemporality, DataPointType, type GaugeMetricData, InstrumentType } from "@opentelemetry/sdk-metrics";
+
+const registeredMetrics = new Map<string, ClearableGaugeMetric>();
+
+//* Custom gauge metric class
+export class ClearableGaugeMetric {
+	private data = new Map<string, { value: number; attributes: Attributes }>();
+
+	constructor(private readonly name: string, private readonly description: string) {
+		registeredMetrics.set(name, this);
+	}
+
+	set(key: string, value: number, attributes: Attributes) {
+		this.data.set(key, { value, attributes });
+	}
+
+	clear({ except }: { except?: string[] }) {
+		for (const key of this.data.keys()) {
+			if (except && except.includes(key))
+				continue;
+
+			this.data.delete(key);
+		}
+	}
+
+	toMetricData(): GaugeMetricData {
+		return {
+			descriptor: {
+				name: this.name,
+				description: this.description,
+				unit: "",
+				type: InstrumentType.GAUGE,
+				valueType: ValueType.INT,
+			},
+			dataPointType: DataPointType.GAUGE,
+			dataPoints: Array.from(this.data.values()).map(({ value, attributes }) => ({
+				value,
+				attributes,
+				startTime: [0, 0],
+				endTime: [0, 0],
+			})),
+			aggregationTemporality: AggregationTemporality.CUMULATIVE,
+		};
+	}
+
+	get hasData() {
+		return this.data.size > 0;
+	}
+}
+
+export function updatePrometheusMetrics(prometheusExporter: PrometheusExporter) {
+	// @ts-expect-error We are modifying a private method
+	prometheusExporter._exportMetrics = function (this: PrometheusExporter, response: ServerResponse) {
+		response.statusCode = 200;
+		response.setHeader("content-type", "text/plain");
+		this.collect().then(
+			(collectionResult) => {
+				const { resourceMetrics, errors } = collectionResult;
+				if (errors.length) {
+					diag.error(
+						"PrometheusExporter: metrics collection errors",
+						...errors,
+					);
+				}
+
+				for (const metric of registeredMetrics.values()) {
+					if (metric.hasData) {
+						resourceMetrics.scopeMetrics[0]!.metrics.push(metric.toMetricData());
+					}
+				}
+
+				response.end((this as unknown as { _serializer: PrometheusSerializer })._serializer.serialize(resourceMetrics));
+			},
+			(err) => {
+				response.end(`# failed to export metrics: ${err}`);
+			},
+		);
+	}.bind(prometheusExporter);
+}

apps/api-master/src/functions/insertIpData.ts

@@ -0,0 +1,40 @@
+import type { InferInsertModel } from "drizzle-orm";
+import { db, onlineUsersIpData } from "../db.js";
+import { lookupIp } from "./lookupIp.js";
+
+const batchSize = 1000;
+
+export async function insertIpData(
+	data: Map<string, {
+		presences: string[];
+		sessions: number;
+	}>,
+) {
+	const timestamp = new Date();
+	const list = [...data.keys()];
+	//* Split into batches of batchSize
+	for (let i = 0; i < list.length; i += batchSize) {
+		const batch = list.slice(i, i + batchSize);
+		const mapped = await Promise.all(batch.map(async (ip) => {
+			const parsed = await lookupIp(ip);
+			if (parsed) {
+				const { presences, sessions } = data.get(ip)!;
+				return {
+					ip,
+					country: parsed.country,
+					latitude: parsed.latitude.toString(),
+					longitude: parsed.longitude.toString(),
+					presences,
+					sessions,
+					timestamp,
+				} satisfies InferInsertModel<typeof onlineUsersIpData>;
+			}
+		}));
+
+		const toInsert = mapped.filter(Boolean) as InferInsertModel<typeof onlineUsersIpData>[];
+
+		if (toInsert.length > 0) {
+			await db.insert(onlineUsersIpData).values(toInsert);
+		}
+	}
+}

apps/api-master/src/functions/lookupIp.ts

@@ -0,0 +1,47 @@
+import { join } from "node:path";
+import process from "node:process";
+import { lookup, reload, updateDb } from "ip-location-api";
+import { mainLog } from "../index.js";
+
+const fields = ["latitude", "longitude", "country"];
+
+const dataDir = join(process.cwd(), "data");
+const tmpDataDir = join(process.cwd(), "tmp");
+const smallMemory = true;
+
+let initialized = false;
+
+export async function lookupIp(ip: string): Promise<{ latitude: number; longitude: number; country: string } | undefined> {
+	if (!initialized) {
+		reloadIpLocationApi();
+		return undefined;
+	}
+	try {
+		return await lookup(ip) ?? undefined;
+	}
+	catch {
+		return undefined;
+	}
+}
+
+let reloading: Promise<void> | undefined = Promise.resolve();
+let log: debug.Debugger | undefined;
+
+export async function reloadIpLocationApi() {
+	log ??= mainLog.extend("IP-Location-API");
+
+	if (reloading)
+		return reloading;
+
+	reloading = new Promise((resolve, reject) => {
+		log?.("Reloading IP location API");
+		updateDb({ fields, dataDir, tmpDataDir, smallMemory }).then(async () => {
+			await reload({ fields, dataDir, tmpDataDir, smallMemory });
+			log?.("IP location API reloaded");
+			initialized = true;
+			reloading = undefined;
+			resolve();
+		}).catch(reject);
+	});
+	return reloading;
+}

apps/api-master/src/functions/setSessionCounter.ts

@@ -1,13 +1,13 @@
 import { redis } from "../index.js";
-import { counter } from "../tracing.js";
+import { activeSessionsCounter } from "../tracing.js";
 
 let activeActivities = 0;
-counter.add(0);
-export async function setCounter() {
+activeSessionsCounter.add(0);
+export async function setSessionCounter() {
 	const length = await redis.hlen("pmd-api.sessions");
 	if (length === activeActivities)
 		return;
 	const diff = length - activeActivities;
 	activeActivities = length;
-	counter.add(diff);
+	activeSessionsCounter.add(diff);
 }

apps/api-master/src/functions/updateActivePresenceGauge.ts

@@ -1,14 +1,19 @@
-import { redis } from "../index.js";
+import { mainLog, redis } from "../index.js";
 import { activePresenceGauge } from "../tracing.js";
+import { insertIpData } from "./insertIpData.js";
 
-//* Track previously recorded services
-const previousServices = new Set<string>();
-
+let log: debug.Debugger | undefined;
 //* Function to update the gauge with per-service counts
 export async function updateActivePresenceGauge() {
+	log ??= mainLog.extend("Heartbeat-Updates");
+
 	const pattern = "pmd-api.heartbeatUpdates.*";
 	let cursor: string = "0";
 	const serviceCounts = new Map<string, number>();
+	const ips = new Map<string, {
+		presences: string[];
+		sessions: number;
+	}>();
 
 	do {
 		const [newCursor, keys] = await redis.scan(cursor, "MATCH", pattern, "COUNT", 1000); //* Use SCAN with COUNT for memory efficiency
@@ -17,23 +22,36 @@ export async function updateActivePresenceGauge() {
 			const hash = await redis.hgetall(key);
 			const service = hash.service;
 			const version = hash.version; //* Get version from hash
-			serviceCounts.set(`${service}:${version}`, (serviceCounts.get(`${service}:${version}`) || 0) + 1);
+			const ip = hash.ip_address;
+			if (service && version) {
+				serviceCounts.set(`${service}:${version}`, (serviceCounts.get(`${service}:${version}`) || 0) + 1);
+			}
+			else {
+				serviceCounts.set("none", (serviceCounts.get("none") || 0) + 1);
+			}
+			if (ip) {
+				const presenceName = service && version ? `${service}:${version}` : undefined;
+				const ipData = ips.get(ip) || { presences: [], sessions: 0 };
+				ipData.presences = [...new Set<string>([...ipData.presences, presenceName].filter(Boolean) as string[])];
+				ipData.sessions++;
+				ips.set(ip, ipData);
+			}
 		}
 	} while (cursor !== "0");
 
-	// Set current counts and remove from previousServices
-	serviceCounts.forEach((count, serviceVersion) => {
-		const [presence_name, version] = serviceVersion.split(":");
-		activePresenceGauge.record(count, { presence_name, version });
-		previousServices.delete(serviceVersion);
-	});
+	log?.("Updating active presence gauge");
 
-	// Set gauge to 0 for services that are no longer active
-	previousServices.forEach((serviceVersion) => {
-		const [presence_name, version] = serviceVersion.split(":");
-		activePresenceGauge.record(0, { presence_name, version });
-	});
+	// Clear previous data
+	activePresenceGauge.clear({ except: [...serviceCounts.keys()] });
 
-	// Update the set of previous services
-	serviceCounts.forEach((_, serviceVersion) => previousServices.add(serviceVersion));
+	// Set new data
+	for (const [serviceVersion, count] of serviceCounts.entries()) {
+		const [presence_name, version] = serviceVersion.split(":");
+		activePresenceGauge.set(serviceVersion, count, {
+			presence_name,
+			version,
+		});
+	}
+
+	insertIpData(ips);
 }

apps/api-master/src/index.ts

@@ -3,9 +3,11 @@ import { CronJob } from "cron";
 import debug from "debug";
 import { clearOldSessions } from "./functions/clearOldSessions.js";
 import createRedis from "./functions/createRedis.js";
-import { setCounter } from "./functions/setCounter.js";
+import { setSessionCounter } from "./functions/setSessionCounter.js";
 import "./tracing.js";
-import { updateActivePresenceGauge } from "./functions/updateActivePresenceGauge.js"; //* Added import
+import { updateActivePresenceGauge } from "./functions/updateActivePresenceGauge.js";
+// import { reloadIpLocationApi } from "./functions/lookupIp.js";
+import { cleanupOldUserData } from "./functions/cleanupOldUserData.js";
 
 export const redis = createRedis();
 
@@ -27,7 +29,7 @@ void new CronJob(
 	// Every second
 	"* * * * * *",
 	() => {
-		setCounter();
+		setSessionCounter();
 	},
 	undefined,
 	true,
@@ -42,3 +44,29 @@ void new CronJob(
 	undefined,
 	true,
 );
+
+// void new CronJob(
+// 	// Every day at 9am
+// 	"0 9 * * *",
+// 	() => {
+// 		reloadIpLocationApi();
+// 	},
+// 	undefined,
+// 	true,
+// 	undefined,
+// 	undefined,
+// 	true,
+// );
+
+void new CronJob(
+	// Every day at 1am
+	"0 1 * * *",
+	() => {
+		cleanupOldUserData(14); // Keep 14 days of data
+	},
+	undefined,
+	true,
+	undefined,
+	undefined,
+	true,
+);

apps/api-master/src/tracing.ts

@@ -1,6 +1,7 @@
 import { ValueType } from "@opentelemetry/api";
 import { PrometheusExporter } from "@opentelemetry/exporter-prometheus";
 import { MeterProvider } from "@opentelemetry/sdk-metrics";
+import { ClearableGaugeMetric, updatePrometheusMetrics } from "./functions/clearableGaugeMetric.js";
 
 const prometheusExporter = new PrometheusExporter();
 
@@ -10,15 +11,16 @@ const provider = new MeterProvider({
 
 const meter = provider.getMeter("nice");
 
-export const counter = meter.createUpDownCounter("active_activites", {
-	description: "Number of active activities",
+export const activeSessionsCounter = meter.createUpDownCounter("active_sessions", {
+	description: "Number of active sessions",
 	valueType: ValueType.INT,
 });
 
-// * Replace Observable Gauge with regular Gauge
-export const activePresenceGauge = meter.createGauge("active_presence_names", {
-	description: "Number of active presence names per service",
-	valueType: ValueType.INT,
-});
+export const activePresenceGauge = new ClearableGaugeMetric(
+	"active_presences",
+	"Per presence name+version, active number of users",
+);
+
+updatePrometheusMetrics(prometheusExporter);
 
 prometheusExporter.startServer();

apps/api-master/tsconfig.app.json

@@ -3,6 +3,7 @@
 	"compilerOptions": {
 		"composite": true,
 		"rootDir": "src",
+		"types": ["./environment.d.ts"],
 		"outDir": "dist"
 	},
 	"include": ["src/**/*"]

apps/api-worker/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@premid/api-worker",
 	"type": "module",
-	"version": "0.0.12",
+	"version": "0.0.14",
 	"private": true,
 	"description": "PreMiD's api",
 	"license": "MPL-2.0",

apps/api-worker/src/functions/createServer.ts

@@ -1,6 +1,5 @@
 import { readFile } from "node:fs/promises";
 import { resolve } from "node:path";
-import { useSentry } from "@envelop/sentry";
 import { maxAliasesPlugin } from "@escape.tech/graphql-armor-max-aliases";
 import { maxDepthPlugin } from "@escape.tech/graphql-armor-max-depth";
 import { maxDirectivesPlugin } from "@escape.tech/graphql-armor-max-directives";

apps/api-worker/src/graphql/resolvers/v5/Mutation/addScience.ts

@@ -1,4 +1,5 @@
 import { type } from "arktype";
+import { GraphQLError } from "graphql";
 import { redis } from "../../../../functions/createServer.js";
 import type { MutationResolvers } from "../../../../generated/graphql-v5.js";
 
@@ -15,7 +16,7 @@ const mutation: MutationResolvers["addScience"] = async (_parent, input) => {
 	const out = addScienceSchema(input);
 
 	if (out instanceof type.errors)
-		throw new Error(out.summary);
+		throw new GraphQLError(out.summary);
 
 	await redis.hset(
 		"pmd-api.scienceUpdates",

apps/api-worker/src/graphql/resolvers/v5/Mutation/heartbeat.ts

@@ -1,16 +1,17 @@
 import { type } from "arktype";
+import { GraphQLError } from "graphql";
 import type { MutationResolvers } from "../../../../generated/graphql-v5.js";
 import { redis } from "../../../../functions/createServer.js";
 
 const heartbeatSchema = type({
-	identifier: "string.uuid & string.lower",
-	presence: {
+	"identifier": "string.uuid & string.lower",
+	"presence?": {
 		service: "string.trim",
 		version: "string.semver",
 		language: "string.trim",
 		since: "number.epoch",
 	},
-	extension: {
+	"extension": {
 		"version": "string.semver",
 		"language": "string.trim",
 		"connected?": {
@@ -20,23 +21,27 @@ const heartbeatSchema = type({
 	},
 });
 
-const mutation: MutationResolvers["heartbeat"] = async (_parent, input) => {
+const mutation: MutationResolvers["heartbeat"] = async (_parent, input, context) => {
 	const out = heartbeatSchema(input);
 
 	if (out instanceof type.errors)
-		throw new Error(out.summary);
+		throw new GraphQLError(out.summary);
+
+	//* Get the user's IP address from Cloudflare headers or fallback to the request IP
+	const userIp = context.request.headers.get("cf-connecting-ip") || context.request.ip;
 
 	// * Use Redis Hash with 'service' in the key to store heartbeat data
 	const redisKey = `pmd-api.heartbeatUpdates.${out.identifier}`;
 	await redis.hset(redisKey, {
-		service: out.presence.service,
-		version: out.presence.version,
-		language: out.presence.language,
-		since: out.presence.since.toString(),
+		service: out.presence?.service,
+		version: out.presence?.version,
+		language: out.presence?.language,
+		since: out.presence?.since.toString(),
 		extension_version: out.extension.version,
 		extension_language: out.extension.language,
 		extension_connected_app: out.extension.connected?.app?.toString(),
 		extension_connected_discord: out.extension.connected?.discord?.toString(),
+		ip_address: userIp,
 	});
 	await redis.expire(redisKey, 300);
 

apps/api-worker/src/routes/sessionKeepAlive.ts

@@ -31,17 +31,15 @@ export async function sessionKeepAlive(request: FastifyRequest, reply: FastifyRe
 	if (!await isTokenValid(out.token))
 		return reply.status(400).send({ code: "INVALID_TOKEN", message: "The token is invalid" });
 
-	await redis.hset(
-		"pmd-api.sessions",
-		out.scienceId,
-		JSON.stringify({
-			session: out.session,
-			token: out.token,
-			lastUpdated: Date.now(),
-		}),
-	);
+	const redisKey = `pmd-api.sessions.${out.scienceId}`;
+	await redis.hset(redisKey, {
+		session: out.session,
+		token: out.token,
+		lastUpdated: Date.now(),
+	});
+	await redis.expire(redisKey, 300); // 5 minutes
 
-	const interval = Number.parseInt(process.env.SESSION_KEEP_ALIVE_INTERVAL ?? "5000");
+	const interval = Number.parseInt(process.env.SESSION_KEEP_ALIVE_INTERVAL ?? "5000"); // 5 seconds
 
 	return reply.status(200).send({
 		code: "OK",

apps/website/stores/useExtension.ts

@@ -22,7 +22,6 @@ export const useExtensionStore = defineStore("extension", () => {
 		}
 	}
 
-	// eslint-disable-next-line unicorn/consistent-function-scoping
 	function fetchPresences() {
 		window.dispatchEvent(new CustomEvent("PreMiD_GetPresenceList"));
 	}

package.json

@@ -38,5 +38,10 @@
 		"prettier": "^3.2.5",
 		"typescript": "^5.5.4",
 		"vitest": "^2.0.2"
+	},
+	"pnpm": {
+		"patchedDependencies": {
+			"ip-location-api@1.0.0": "patches/ip-location-api@1.0.0.patch"
+		}
 	}
 }

patches/ip-location-api@1.0.0.patch

@@ -0,0 +1,62 @@
+diff --git a/browser/country/README.md b/browser/country/README.md
+deleted file mode 100644
+index ac8fc934b4998f2a2cb7a92bf68bbdadd9e3d36d..0000000000000000000000000000000000000000
+diff --git a/browser/country-extra/README.md b/browser/country-extra/README.md
+deleted file mode 100644
+index 71e7237722915b2697b56ccb14171524eb4b40fb..0000000000000000000000000000000000000000
+diff --git a/browser/geocode/README.md b/browser/geocode/README.md
+deleted file mode 100644
+index 9d9a2205061f332b363b82d7561a0e3829d5bf2c..0000000000000000000000000000000000000000
+diff --git a/browser/geocode-extra/README.md b/browser/geocode-extra/README.md
+deleted file mode 100644
+index 38e17eebdd8532d07b460fbe7f385f36625ece9d..0000000000000000000000000000000000000000
+diff --git a/src/db.mjs b/src/db.mjs
+index 378b8a22084f860cc89720d1783a235c034717b2..cbffe1eaa84a94df536059d4b4af3f8f5ceb0ca7 100644
+--- a/src/db.mjs
++++ b/src/db.mjs
+@@ -33,7 +33,12 @@ export const update = async () => {
+ 	if(refreshTmpDir || !fsSync.existsSync(setting.tmpDataDir)){
+ 		// refresh tmp folder
+ 		await rimraf(setting.tmpDataDir)
+-		await fs.mkdir(setting.tmpDataDir)
++		await fs.mkdir(setting.tmpDataDir, {recursive: true})
++	}
++
++	// When specifying a custom dataDir, it doesn't always exists
++	if (!fsSync.existsSync(setting.dataDir)){
++		await fs.mkdir(setting.dataDir, {recursive: true})
+ 	}
+ 
+ 	console.log('Downloading database')
+diff --git a/src/main.mjs b/src/main.mjs
+index d001aca60902bc7fe41271c6fa7a0b6648607b15..5b2c125d8e7590afee82c794bf771accd656b2b7 100644
+--- a/src/main.mjs
++++ b/src/main.mjs
+@@ -3,7 +3,7 @@ import fs from 'fs/promises'
+ import fsSync from 'fs'
+ import path from 'path'
+ import { exec } from 'child_process'
+-
++import { fileURLToPath } from "url"
+ import { countries, continents } from 'countries-list'
+ 
+ import { setting, setSetting, getSettingCmd } from './setting.mjs'
+@@ -14,6 +14,9 @@ const v6db = setting.v6
+ const locFieldHash = setting.locFieldHash
+ const mainFieldHash = setting.mainFieldHash
+ 
++const __filename = fileURLToPath(import.meta.url)
++const __dirname = path.dirname(__filename)
++
+ //---------------------------------------
+ // Database lookup
+ //---------------------------------------
+@@ -235,7 +238,7 @@ export const updateDb = (_setting) => {
+ 	// However, db.js import many external modules, it makes slow down the startup time and uses more memory.
+ 	// Therefore, we use exec() to run the script in the other process.
+ 	return new Promise((resolve, reject) => {
+-		var cmd = 'node ' + path.resolve(__dirname, '..', 'script', 'updatedb.js')
++		var cmd = 'node ' + path.resolve(__dirname, '..', 'script', 'updatedb.mjs')
+ 		var arg
+ 		if(_setting){
+ 			var oldSetting = Object.assign({}, setting)

tsconfig.base.json

@@ -29,5 +29,5 @@
 		"isolatedModules": true,
 		"skipLibCheck": true
 	},
-	"exclude": ["**/*/node_modules", "**/*/dist"]
+	"exclude": ["**/*/node_modules", "**/*/dist", "**/drizzle.config.ts"]
 }