chore: release v0.0.30

apps/api-master/environment.d.ts

@@ -5,8 +5,8 @@ declare module "ip-location-api" {
 		country: string;
 	} | null>;
 
-	export function updateDb(options: { fields?: string[]; dataDir?: string; tmpDataDir?: string }): Promise<void>;
-	export function reload(options: { fields?: string[]; dataDir?: string; tmpDataDir?: string }): Promise<void>;
+	export function updateDb(options: { fields?: string[]; dataDir?: string; tmpDataDir?: string; smallMemory?: boolean }): Promise<void>;
+	export function reload(options: { fields?: string[]; dataDir?: string; tmpDataDir?: string; smallMemory?: boolean }): Promise<void>;
 }
 
 declare namespace NodeJS {

apps/api-master/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@premid/api-master",
 	"type": "module",
-	"version": "0.0.26",
+	"version": "0.0.30",
 	"private": true,
 	"description": "PreMiD's api master",
 	"license": "MPL-2.0",

apps/api-master/src/functions/cleanupOldUserData.ts

@@ -1,7 +1,9 @@
 import { lt, sql } from "drizzle-orm";
 import { db, onlineUsersIpData } from "../db.js";
+import { mainLog } from "../index.js";
 
 export async function cleanupOldUserData(retentionDays: number) {
+	mainLog("Cleaning up old user ip data");
 	const interval = `'${retentionDays} days'`;
 	await db.delete(onlineUsersIpData)
 		.where(lt(onlineUsersIpData.timestamp, sql`now() - interval ${sql.raw(interval)}`));

apps/api-master/src/functions/clearableGaugeMetric.ts

@@ -8,14 +8,9 @@ const registeredMetrics = new Map<string, ClearableGaugeMetric>();
 
 //* Custom gauge metric class
 export class ClearableGaugeMetric {
-	private data: Map<string, { value: number; attributes: Attributes }>;
-	private name: string;
-	private description: string;
+	private data = new Map<string, { value: number; attributes: Attributes }>();
 
-	constructor(name: string, description: string) {
-		this.data = new Map();
-		this.name = name;
-		this.description = description;
+	constructor(private readonly name: string, private readonly description: string) {
 		registeredMetrics.set(name, this);
 	}
 
@@ -23,8 +18,13 @@ export class ClearableGaugeMetric {
 		this.data.set(key, { value, attributes });
 	}
 
-	clear() {
-		this.data.clear();
+	clear({ except }: { except?: string[] }) {
+		for (const key of this.data.keys()) {
+			if (except && except.includes(key))
+				continue;
+
+			this.data.delete(key);
+		}
 	}
 
 	toMetricData(): GaugeMetricData {

apps/api-master/src/functions/insertIpData.ts

@@ -2,7 +2,7 @@ import type { InferInsertModel } from "drizzle-orm";
 import { db, onlineUsersIpData } from "../db.js";
 import { lookupIp } from "./lookupIp.js";
 
-const batchSize = 10000;
+const batchSize = 1000;
 
 export async function insertIpData(
 	data: Map<string, {
@@ -11,13 +11,14 @@ export async function insertIpData(
 	}>,
 ) {
 	const timestamp = new Date();
-	const list = Array.from(data.entries());
+	const list = [...data.keys()];
 	//* Split into batches of batchSize
 	for (let i = 0; i < list.length; i += batchSize) {
 		const batch = list.slice(i, i + batchSize);
-		const mapped = await Promise.all(batch.map(async ([ip, { presences, sessions }]) => {
+		const mapped = await Promise.all(batch.map(async (ip) => {
 			const parsed = await lookupIp(ip);
 			if (parsed) {
+				const { presences, sessions } = data.get(ip)!;
 				return {
 					ip,
 					country: parsed.country,

apps/api-master/src/functions/lookupIp.ts

@@ -7,6 +7,7 @@ const fields = ["latitude", "longitude", "country"];
 
 const dataDir = join(process.cwd(), "data");
 const tmpDataDir = join(process.cwd(), "tmp");
+const smallMemory = true;
 
 let initialized = false;
 
@@ -23,7 +24,7 @@ export async function lookupIp(ip: string): Promise<{ latitude: number; longitud
 	}
 }
 
-let reloading: Promise<void> | undefined;
+let reloading: Promise<void> | undefined = Promise.resolve();
 let log: debug.Debugger | undefined;
 
 export async function reloadIpLocationApi() {
@@ -34,8 +35,8 @@ export async function reloadIpLocationApi() {
 
 	reloading = new Promise((resolve, reject) => {
 		log?.("Reloading IP location API");
-		updateDb({ fields, dataDir, tmpDataDir }).then(async () => {
-			await reload({ fields, dataDir, tmpDataDir });
+		updateDb({ fields, dataDir, tmpDataDir, smallMemory }).then(async () => {
+			await reload({ fields, dataDir, tmpDataDir, smallMemory });
 			log?.("IP location API reloaded");
 			initialized = true;
 			reloading = undefined;

apps/api-master/src/functions/updateActivePresenceGauge.ts

@@ -1,9 +1,12 @@
-import { redis } from "../index.js";
+import { mainLog, redis } from "../index.js";
 import { activePresenceGauge } from "../tracing.js";
 import { insertIpData } from "./insertIpData.js";
 
+let log: debug.Debugger | undefined;
 //* Function to update the gauge with per-service counts
 export async function updateActivePresenceGauge() {
+	log ??= mainLog.extend("Heartbeat-Updates");
+
 	const pattern = "pmd-api.heartbeatUpdates.*";
 	let cursor: string = "0";
 	const serviceCounts = new Map<string, number>();
@@ -36,8 +39,10 @@ export async function updateActivePresenceGauge() {
 		}
 	} while (cursor !== "0");
 
+	log?.("Updating active presence gauge");
+
 	// Clear previous data
-	activePresenceGauge.clear();
+	activePresenceGauge.clear({ except: [...serviceCounts.keys()] });
 
 	// Set new data
 	for (const [serviceVersion, count] of serviceCounts.entries()) {

apps/api-master/src/index.ts

@@ -6,7 +6,7 @@ import createRedis from "./functions/createRedis.js";
 import { setSessionCounter } from "./functions/setSessionCounter.js";
 import "./tracing.js";
 import { updateActivePresenceGauge } from "./functions/updateActivePresenceGauge.js";
-import { reloadIpLocationApi } from "./functions/lookupIp.js";
+// import { reloadIpLocationApi } from "./functions/lookupIp.js";
 import { cleanupOldUserData } from "./functions/cleanupOldUserData.js";
 
 export const redis = createRedis();
@@ -45,18 +45,18 @@ void new CronJob(
 	true,
 );
 
-void new CronJob(
-	// Every day at 9am
-	"0 9 * * *",
-	() => {
-		reloadIpLocationApi();
-	},
-	undefined,
-	true,
-	undefined,
-	undefined,
-	true,
-);
+// void new CronJob(
+// 	// Every day at 9am
+// 	"0 9 * * *",
+// 	() => {
+// 		reloadIpLocationApi();
+// 	},
+// 	undefined,
+// 	true,
+// 	undefined,
+// 	undefined,
+// 	true,
+// );
 
 void new CronJob(
 	// Every day at 1am