Compare commits

...

27 Commits

Author SHA1 Message Date
Bas950
5a90f95e58 chore: release v0.0.37 2024-09-24 17:07:06 +02:00
Bas950
64547dc0ef feat: ip data 2024-09-24 17:06:48 +02:00
Bas950
9cf3f93889 chore: release v0.0.36 2024-09-24 13:47:54 +02:00
Bas950
0e30a0d250 chore: scan keys instead 2024-09-24 13:47:39 +02:00
Bas950
4dc941bb91 chore: release v0.0.35 2024-09-24 11:08:50 +02:00
Bas950
3a78c6529e feat: use prom-client 2024-09-24 10:50:21 +02:00
Bas950
d4673720a0 chore: release v1.0.4 2024-09-20 19:03:37 +02:00
Bas950
dc859448bd feat: schema v1.11 2024-09-20 18:57:28 +02:00
Bas950
9cbb88beda chore: release v0.0.34 2024-09-17 10:54:54 +02:00
Bas950
09bcfe703f chore: scan count config 2024-09-17 10:54:48 +02:00
Bas950
d24eda8957 chore: release v0.0.33 2024-09-17 10:45:42 +02:00
Bas950
bfffcb94ee chore: some testing 2024-09-17 10:45:38 +02:00
Bas950
4db6a78816 chore: release v0.0.32 2024-09-17 10:35:04 +02:00
Bas950
666838874f chore: forgot to save 2024-09-17 10:34:58 +02:00
Bas950
697f3660c2 chore: some improvements 2024-09-17 10:34:46 +02:00
Florian Metz
a668add973 chore: release v0.0.31 2024-09-17 10:00:06 +02:00
Florian Metz
42b70b1259 chore: optimize active presence gauge update with concurrency limit 2024-09-17 09:59:41 +02:00
Bas950
253b680d3e chore: release v0.0.30 2024-09-17 09:36:49 +02:00
Bas950
e9a40dc553 chore: small updates 2024-09-17 09:36:43 +02:00
Bas950
b25880d4cd chore: release v0.0.29 2024-09-17 09:11:12 +02:00
Bas950
fb06227aeb chore: release v0.0.29 2024-09-17 09:07:58 +02:00
Bas950
ff3d00497b chore: release v0.0.29 2024-09-17 09:07:21 +02:00
Bas950
a06780f85a chore: reduce memory 2024-09-17 09:06:10 +02:00
Bas950
5b1969c7ab chore: release v0.0.28 2024-09-16 23:22:15 +02:00
Bas950
bedd34594c chore: disable ip stuff for now 2024-09-16 23:22:11 +02:00
Bas950
47feaa5c70 chore: release v0.0.27 2024-09-16 22:56:14 +02:00
Bas950
9fb32f53ae chore: reduce batch size 2024-09-16 22:56:10 +02:00
16 changed files with 823 additions and 502 deletions

View File

@@ -5,8 +5,8 @@ declare module "ip-location-api" {
country: string;
} | null>;
export function updateDb(options: { fields?: string[]; dataDir?: string; tmpDataDir?: string }): Promise<void>;
export function reload(options: { fields?: string[]; dataDir?: string; tmpDataDir?: string }): Promise<void>;
export function updateDb(options: { fields?: string[]; dataDir?: string; tmpDataDir?: string; smallMemory?: boolean; autoUpdate?: string }): Promise<void>;
export function reload(options: { fields?: string[]; dataDir?: string; tmpDataDir?: string; smallMemory?: boolean; autoUpdate?: string }): Promise<void>;
}
declare namespace NodeJS {

View File

@@ -1,7 +1,7 @@
{
"name": "@premid/api-master",
"type": "module",
"version": "0.0.26",
"version": "0.0.37",
"private": true,
"description": "PreMiD's api master",
"license": "MPL-2.0",
@@ -19,18 +19,16 @@
},
"dependencies": {
"@envelop/sentry": "^9.0.0",
"@opentelemetry/api": "^1.9.0",
"@opentelemetry/exporter-prometheus": "^0.52.1",
"@opentelemetry/node": "^0.24.0",
"@sentry/node": "^8.17.0",
"cron": "^3.1.7",
"debug": "^4.3.6",
"drizzle-orm": "^0.33.0",
"ioredis": "^5.3.2",
"ip-location-api": "^1.0.0",
"ip-location-api": "^2.0.1",
"ky": "^1.7.2",
"p-limit": "^6.1.0",
"postgres": "^3.4.4"
"postgres": "^3.4.4",
"prom-client": "^15.1.3"
},
"devDependencies": {
"@types/debug": "^4.1.12",

View File

@@ -1,7 +1,9 @@
import { lt, sql } from "drizzle-orm";
import { db, onlineUsersIpData } from "../db.js";
import { mainLog } from "../index.js";
export async function cleanupOldUserData(retentionDays: number) {
mainLog("Cleaning up old user ip data");
const interval = `'${retentionDays} days'`;
await db.delete(onlineUsersIpData)
.where(lt(onlineUsersIpData.timestamp, sql`now() - interval ${sql.raw(interval)}`));

View File

@@ -1,83 +0,0 @@
import type { ServerResponse } from "node:http";
import type { Attributes } from "@opentelemetry/api";
import { ValueType, diag } from "@opentelemetry/api";
import type { PrometheusExporter, PrometheusSerializer } from "@opentelemetry/exporter-prometheus";
import { AggregationTemporality, DataPointType, type GaugeMetricData, InstrumentType } from "@opentelemetry/sdk-metrics";
const registeredMetrics = new Map<string, ClearableGaugeMetric>();
//* Custom gauge metric class
export class ClearableGaugeMetric {
private data: Map<string, { value: number; attributes: Attributes }>;
private name: string;
private description: string;
constructor(name: string, description: string) {
this.data = new Map();
this.name = name;
this.description = description;
registeredMetrics.set(name, this);
}
set(key: string, value: number, attributes: Attributes) {
this.data.set(key, { value, attributes });
}
clear() {
this.data.clear();
}
toMetricData(): GaugeMetricData {
return {
descriptor: {
name: this.name,
description: this.description,
unit: "",
type: InstrumentType.GAUGE,
valueType: ValueType.INT,
},
dataPointType: DataPointType.GAUGE,
dataPoints: Array.from(this.data.values()).map(({ value, attributes }) => ({
value,
attributes,
startTime: [0, 0],
endTime: [0, 0],
})),
aggregationTemporality: AggregationTemporality.CUMULATIVE,
};
}
get hasData() {
return this.data.size > 0;
}
}
export function updatePrometheusMetrics(prometheusExporter: PrometheusExporter) {
// @ts-expect-error We are modifying a private method
prometheusExporter._exportMetrics = function (this: PrometheusExporter, response: ServerResponse) {
response.statusCode = 200;
response.setHeader("content-type", "text/plain");
this.collect().then(
(collectionResult) => {
const { resourceMetrics, errors } = collectionResult;
if (errors.length) {
diag.error(
"PrometheusExporter: metrics collection errors",
...errors,
);
}
for (const metric of registeredMetrics.values()) {
if (metric.hasData) {
resourceMetrics.scopeMetrics[0]!.metrics.push(metric.toMetricData());
}
}
response.end((this as unknown as { _serializer: PrometheusSerializer })._serializer.serialize(resourceMetrics));
},
(err) => {
response.end(`# failed to export metrics: ${err}`);
},
);
}.bind(prometheusExporter);
}

View File

@@ -2,7 +2,7 @@ import type { InferInsertModel } from "drizzle-orm";
import { db, onlineUsersIpData } from "../db.js";
import { lookupIp } from "./lookupIp.js";
const batchSize = 10000;
const batchSize = 1000;
export async function insertIpData(
data: Map<string, {
@@ -11,13 +11,14 @@ export async function insertIpData(
}>,
) {
const timestamp = new Date();
const list = Array.from(data.entries());
const list = [...data.keys()];
//* Split into batches of batchSize
for (let i = 0; i < list.length; i += batchSize) {
const batch = list.slice(i, i + batchSize);
const mapped = await Promise.all(batch.map(async ([ip, { presences, sessions }]) => {
const mapped = await Promise.all(batch.map(async (ip) => {
const parsed = await lookupIp(ip);
if (parsed) {
const { presences, sessions } = data.get(ip)!;
return {
ip,
country: parsed.country,

View File

@@ -1,12 +1,13 @@
import { join } from "node:path";
import process from "node:process";
import { lookup, reload, updateDb } from "ip-location-api";
import { lookup, reload } from "ip-location-api";
import { mainLog } from "../index.js";
const fields = ["latitude", "longitude", "country"];
const dataDir = join(process.cwd(), "data");
const tmpDataDir = join(process.cwd(), "tmp");
const smallMemory = true;
let initialized = false;
@@ -34,8 +35,7 @@ export async function reloadIpLocationApi() {
reloading = new Promise((resolve, reject) => {
log?.("Reloading IP location API");
updateDb({ fields, dataDir, tmpDataDir }).then(async () => {
await reload({ fields, dataDir, tmpDataDir });
reload({ fields, dataDir, tmpDataDir, smallMemory, autoUpdate: "0 9 * * *" }).then(() => {
log?.("IP location API reloaded");
initialized = true;
reloading = undefined;

View File

@@ -1,13 +0,0 @@
import { redis } from "../index.js";
import { activeSessionsCounter } from "../tracing.js";
let activeActivities = 0;
activeSessionsCounter.add(0);
export async function setSessionCounter() {
const length = await redis.hlen("pmd-api.sessions");
if (length === activeActivities)
return;
const diff = length - activeActivities;
activeActivities = length;
activeSessionsCounter.add(diff);
}

View File

@@ -0,0 +1,25 @@
import http from "node:http";
import { mainLog } from "../index.js";
import { register } from "../tracing.js";
export function setupServer() {
const server = http.createServer(async (req, res) => {
//* If it's a head request, just return 200
if (req.method === "HEAD")
return res.writeHead(200).end();
//* If it's a favicon request, just return 404
if (req.url === "/favicon.ico")
return res.writeHead(404).end();
//* Basic routing logic
res.writeHead(200, { "Content-Type": "text/plain" });
res.end(await register.metrics());
});
server.listen(9464, () => {
mainLog("Server running");
});
return server;
}

View File

@@ -1,52 +1,77 @@
import { redis } from "../index.js";
import { activePresenceGauge } from "../tracing.js";
import process from "node:process";
import pLimit from "p-limit";
import type { Gauge } from "prom-client";
import { mainLog, redis } from "../index.js";
import { insertIpData } from "./insertIpData.js";
//* Function to update the gauge with per-service counts
export async function updateActivePresenceGauge() {
const pattern = "pmd-api.heartbeatUpdates.*";
let cursor: string = "0";
const serviceCounts = new Map<string, number>();
const ips = new Map<string, {
presences: string[];
sessions: number;
}>();
export const updateActivePresenceGaugeLimit = pLimit(1);
let log: debug.Debugger | undefined;
do {
const [newCursor, keys] = await redis.scan(cursor, "MATCH", pattern, "COUNT", 1000); //* Use SCAN with COUNT for memory efficiency
cursor = newCursor;
for (const key of keys) {
const hash = await redis.hgetall(key);
const service = hash.service;
const version = hash.version; //* Get version from hash
const ip = hash.ip_address;
if (service && version) {
serviceCounts.set(`${service}:${version}`, (serviceCounts.get(`${service}:${version}`) || 0) + 1);
}
else {
serviceCounts.set("none", (serviceCounts.get("none") || 0) + 1);
}
if (ip) {
const presenceName = service && version ? `${service}:${version}` : undefined;
const ipData = ips.get(ip) || { presences: [], sessions: 0 };
ipData.presences = [...new Set<string>([...ipData.presences, presenceName].filter(Boolean) as string[])];
ipData.sessions++;
ips.set(ip, ipData);
const scanCount = Number.parseInt(process.env.SCAN_COUNT || "1000", 10);
export async function updateActivePresenceGauge(gauge: Gauge) {
await updateActivePresenceGaugeLimit(async () => {
log ??= mainLog.extend("Heartbeat-Updates");
log?.("Starting active presence gauge update");
const pattern = "pmd-api.heartbeatUpdates.*";
let cursor: string = "0";
const serviceCounts = new Map<string, number>();
const ips = new Map<string, {
presences: Set<string>;
sessions: number;
}>();
do {
const [newCursor, keys] = await redis.scan(cursor, "MATCH", pattern, "COUNT", scanCount);
cursor = newCursor;
//* Use pipelining for batch Redis operations
const pipeline = redis.pipeline();
keys.forEach(key => pipeline.hmget(key, "service", "version", "ip_address"));
const hashes = await pipeline.exec();
if (!hashes) {
log?.("No hashes found");
return;
}
hashes.forEach(([err, hash]) => {
if (err || !Array.isArray(hash))
return;
const [service, version, ip] = hash;
const serviceVersion = service && version ? `${service}:${version}` : "none";
serviceCounts.set(serviceVersion, (serviceCounts.get(serviceVersion) || 0) + 1);
if (ip) {
const ipData = ips.get(ip) || { presences: new Set(), sessions: 0 };
if (serviceVersion !== "none")
ipData.presences.add(serviceVersion);
ipData.sessions++;
ips.set(ip, ipData);
}
});
} while (cursor !== "0");
log?.("Updating active presence gauge");
//* Batch update the gauge
gauge.reset();
for (const [serviceVersion, count] of serviceCounts) {
const [presence_name, version] = serviceVersion.split(":");
gauge.set({ presence_name, version }, count);
}
} while (cursor !== "0");
// Clear previous data
activePresenceGauge.clear();
//* Convert IP data for insertion
const ipDataForInsertion = new Map(
Array.from(ips, ([ip, data]) => [ip, {
presences: Array.from(data.presences),
sessions: data.sessions,
}]),
);
// Set new data
for (const [serviceVersion, count] of serviceCounts.entries()) {
const [presence_name, version] = serviceVersion.split(":");
activePresenceGauge.set(serviceVersion, count, {
presence_name,
version,
});
}
insertIpData(ips);
await insertIpData(ipDataForInsertion);
log?.("Active presence gauge update completed");
});
}

View File

@@ -1,63 +1,35 @@
import process from "node:process";
import { CronJob } from "cron";
import debug from "debug";
import { clearOldSessions } from "./functions/clearOldSessions.js";
import createRedis from "./functions/createRedis.js";
import { setSessionCounter } from "./functions/setSessionCounter.js";
import "./tracing.js";
import { updateActivePresenceGauge } from "./functions/updateActivePresenceGauge.js";
import { reloadIpLocationApi } from "./functions/lookupIp.js";
import { cleanupOldUserData } from "./functions/cleanupOldUserData.js";
import { setupServer } from "./functions/setupServer.js";
export const redis = createRedis();
export const server = setupServer();
export const mainLog = debug("api-master");
debug("Starting cron jobs");
void new CronJob(
// Every 5 seconds
"*/5 * * * * *",
() => {
clearOldSessions();
},
undefined,
true,
);
void new CronJob(
// Every second
"* * * * * *",
() => {
setSessionCounter();
},
undefined,
true,
);
void reloadIpLocationApi();
void new CronJob(
// Every 5 seconds
"*/5 * * * * *",
() => {
updateActivePresenceGauge();
if (process.env.DISABLE_CLEAR_OLD_SESSIONS !== "true") {
clearOldSessions();
}
},
undefined,
true,
);
void new CronJob(
// Every day at 9am
"0 9 * * *",
() => {
reloadIpLocationApi();
},
undefined,
true,
undefined,
undefined,
true,
);
void new CronJob(
// Every day at 1am
"0 1 * * *",

View File

@@ -1,26 +1,41 @@
import { ValueType } from "@opentelemetry/api";
import { PrometheusExporter } from "@opentelemetry/exporter-prometheus";
import { MeterProvider } from "@opentelemetry/sdk-metrics";
import { ClearableGaugeMetric, updatePrometheusMetrics } from "./functions/clearableGaugeMetric.js";
import process from "node:process";
import { Counter, Gauge, Registry, collectDefaultMetrics } from "prom-client";
import { updateActivePresenceGauge, updateActivePresenceGaugeLimit } from "./functions/updateActivePresenceGauge.js";
import { redis } from "./index.js";
const prometheusExporter = new PrometheusExporter();
const scanCount = Number.parseInt(process.env.SCAN_COUNT || "1000", 10);
const provider = new MeterProvider({
readers: [prometheusExporter],
export const register = new Registry();
collectDefaultMetrics({ register });
export const activeSessionsCounter = new Counter({
name: "active_sessions",
help: "Number of active sessions",
async collect() {
this.reset();
let length = 0;
let cursor = "0";
do {
const reply = await redis.scan(cursor, "MATCH", "pmd-api.sessions.*", "COUNT", scanCount);
cursor = reply[0];
length += reply[1].length;
} while (cursor !== "0");
this.inc(length);
},
});
const meter = provider.getMeter("nice");
export const activeSessionsCounter = meter.createUpDownCounter("active_sessions", {
description: "Number of active sessions",
valueType: ValueType.INT,
export const activePresencesCounter = new Gauge({
name: "active_presences",
help: "Number of active presences",
labelNames: ["presence_name", "version"],
async collect() {
if (process.env.DISABLE_ACTIVE_PRESENCE_GAUGE !== "true") {
this.reset();
updateActivePresenceGaugeLimit.clearQueue();
await updateActivePresenceGauge(this);
}
},
});
export const activePresenceGauge = new ClearableGaugeMetric(
"active_presences",
"Per presence name+version, active number of users",
);
updatePrometheusMetrics(prometheusExporter);
prometheusExporter.startServer();
register.registerMetric(activeSessionsCounter);
register.registerMetric(activePresencesCounter);

View File

@@ -1,7 +1,7 @@
{
"name": "@premid/schema-server",
"type": "module",
"version": "1.0.3",
"version": "1.0.4",
"private": true,
"description": "A small service to serve the JSON schemas for PreMiD",
"license": "MPL-2.0",

View File

@@ -0,0 +1,260 @@
{
"$schema": "http://json-schema.org/draft-07/schema",
"$id": "https://schemas.premid.app/metadata/1.11",
"title": "Metadata",
"type": "object",
"description": "Metadata that describes a presence.",
"definitions": {
"user": {
"type": "object",
"description": "User information.",
"properties": {
"name": {
"type": "string",
"description": "The name of the user."
},
"id": {
"type": "string",
"description": "The Discord snowflake of the user.",
"pattern": "^\\d+$"
}
},
"additionalProperties": false,
"required": [
"name",
"id"
]
}
},
"properties": {
"$schema": {
"$comment": "This is required otherwise the schema will fail itself when it is applied to a document via $schema. This is optional so that validators that use this schema don't fail if the metadata doesn't have the $schema property.",
"type": "string",
"description": "The metadata schema URL."
},
"author": {
"$ref": "#/definitions/user",
"description": "The author of this presence."
},
"contributors": {
"type": "array",
"description": "Any extra contributors to this presence.",
"items": {
"$ref": "#/definitions/user"
}
},
"service": {
"type": "string",
"description": "The service this presence is for."
},
"altnames": {
"type": "array",
"description": "Alternative names for the service.",
"items": {
"type": "string",
"description": "An alternative name."
},
"minItems": 1
},
"description": {
"type": "object",
"description": "A description of the presence in multiple languages.",
"propertyNames": {
"type": "string",
"description": "The language key. The key must be languagecode(_REGIONCODE).",
"pattern": "^[a-z]{2}(?:_(?:[A-Z]{2}|[0-9]{1,3}))?$"
},
"patternProperties": {
"^[a-z]{2}(?:_(?:[A-Z]{2}|[0-9]{1,3}))?$": {
"type": "string",
"description": "The description of the presence in the key's language."
}
},
"additionalProperties": false,
"required": [
"en"
]
},
"url": {
"type": [
"string",
"array"
],
"description": "The service's website URL, or an array of URLs. Protocols should not be added.",
"pattern": "^(([a-z0-9-]+\\.)*[0-9a-z_-]+(\\.[a-z]+)+|(\\d{1,3}\\.){3}\\d{1,3}|localhost)$",
"items": {
"type": "string",
"description": "One of the service's website URLs.",
"pattern": "^(([a-z0-9-]+\\.)*[0-9a-z_-]+(\\.[a-z]+)+|(\\d{1,3}\\.){3}\\d{1,3}|localhost)$"
},
"minItems": 2
},
"version": {
"type": "string",
"description": "The SemVer version of the presence. Must just be major.minor.patch.",
"pattern": "^\\d+\\.\\d+\\.\\d+$"
},
"apiVersion": {
"type": "integer",
"description": "The Presence System version this Presence supports.",
"minimum": 1,
"maximum": 2
},
"logo": {
"type": "string",
"description": "The logo of the service this presence is for.",
"pattern": "^https?://.+\\.(png|jpe?g|gif|webp)$"
},
"thumbnail": {
"type": "string",
"description": "A thumbnail of the service this presence is for.",
"pattern": "^https?://.+\\.(png|jpe?g|gif|webp)$"
},
"color": {
"type": "string",
"description": "The theme color of the service this presence is for. Must be either a 6 digit or a 3 digit hex code.",
"pattern": "^#([A-Fa-f0-9]{3}){1,2}$"
},
"tags": {
"type": [
"array"
],
"description": "The tags for the presence.",
"items": {
"type": "string",
"description": "A tag.",
"pattern": "^[^A-Z\\s!\"#$%&'()*+,./:;<=>?@\\[\\\\\\]^_`{|}~]+$"
},
"minItems": 1
},
"category": {
"type": "string",
"description": "The category the presence falls under.",
"enum": [
"anime",
"games",
"music",
"socials",
"videos",
"other"
]
},
"iframe": {
"type": "boolean",
"description": "Whether or not the presence should run in IFrames."
},
"readLogs": {
"type": "boolean",
"description": "Whether or not the extension should be reading logs."
},
"regExp": {
"type": "string",
"description": "A regular expression used to match URLs for the presence to inject into."
},
"iFrameRegExp": {
"type": "string",
"description": "A regular expression used to match IFrames for the presence to inject into."
},
"button": {
"type": "boolean",
"description": "Controls whether the presence is automatically added when the extension is installed. For partner presences only."
},
"warning": {
"type": "boolean",
"description": "Shows a warning saying that it requires additional steps for the presence to function correctly."
},
"settings": {
"type": "array",
"description": "An array of settings the user can change in the presence.",
"items": {
"type": "object",
"description": "A setting.",
"properties": {
"id": {
"type": "string",
"description": "The ID of the setting."
},
"title": {
"type": "string",
"description": "The title of the setting. Required only if `multiLanguage` is disabled."
},
"icon": {
"type": "string",
"description": "The icon of the setting. Required only if `multiLanguage` is disabled.",
"pattern": "^fa([bsdrlt]|([-](brands|solid|duotone|regular|light|thin))) fa-[0-9a-z-]+$"
},
"if": {
"type": "object",
"description": "Restrict showing this setting if another setting is the defined value.",
"propertyNames": {
"type": "string",
"description": "The ID of the setting."
},
"patternProperties": {
"": {
"type": [
"string",
"number",
"boolean"
],
"description": "The value of the setting."
}
},
"additionalProperties": false
},
"placeholder": {
"type": "string",
"description": "The placeholder for settings that require input. Shown when the input is empty."
},
"value": {
"type": [
"string",
"number",
"boolean"
],
"description": "The default value of the setting. Not compatible with `values`."
},
"values": {
"type": "array",
"description": "The default values of the setting. Not compatible with `value`.",
"items": {
"type": [
"string",
"number",
"boolean"
],
"description": "The value of the setting."
}
},
"multiLanguage": {
"type": [
"string",
"boolean",
"array"
],
"description": "When false, multi-localization is disabled. When true, strings from the `general.json` file are available for use. When a string, it is the name of a file (excluding .json) of a used language from the localization GitHub repo. When an array of strings, it is all of the file names (excluding .json) of used languages from the localization GitHub repo.",
"items": {
"type": "string",
"description": "The name of a file from the localization GitHub repository."
}
}
},
"additionalProperties": false
}
}
},
"additionalProperties": false,
"required": [
"author",
"service",
"description",
"url",
"version",
"apiVersion",
"logo",
"thumbnail",
"color",
"tags",
"category"
]
}

View File

@@ -38,10 +38,5 @@
"prettier": "^3.2.5",
"typescript": "^5.5.4",
"vitest": "^2.0.2"
},
"pnpm": {
"patchedDependencies": {
"ip-location-api@1.0.0": "patches/ip-location-api@1.0.0.patch"
}
}
}

View File

@@ -1,62 +0,0 @@
diff --git a/browser/country/README.md b/browser/country/README.md
deleted file mode 100644
index ac8fc934b4998f2a2cb7a92bf68bbdadd9e3d36d..0000000000000000000000000000000000000000
diff --git a/browser/country-extra/README.md b/browser/country-extra/README.md
deleted file mode 100644
index 71e7237722915b2697b56ccb14171524eb4b40fb..0000000000000000000000000000000000000000
diff --git a/browser/geocode/README.md b/browser/geocode/README.md
deleted file mode 100644
index 9d9a2205061f332b363b82d7561a0e3829d5bf2c..0000000000000000000000000000000000000000
diff --git a/browser/geocode-extra/README.md b/browser/geocode-extra/README.md
deleted file mode 100644
index 38e17eebdd8532d07b460fbe7f385f36625ece9d..0000000000000000000000000000000000000000
diff --git a/src/db.mjs b/src/db.mjs
index 378b8a22084f860cc89720d1783a235c034717b2..cbffe1eaa84a94df536059d4b4af3f8f5ceb0ca7 100644
--- a/src/db.mjs
+++ b/src/db.mjs
@@ -33,7 +33,12 @@ export const update = async () => {
if(refreshTmpDir || !fsSync.existsSync(setting.tmpDataDir)){
// refresh tmp folder
await rimraf(setting.tmpDataDir)
- await fs.mkdir(setting.tmpDataDir)
+ await fs.mkdir(setting.tmpDataDir, {recursive: true})
+ }
+
+ // When specifying a custom dataDir, it doesn't always exists
+ if (!fsSync.existsSync(setting.dataDir)){
+ await fs.mkdir(setting.dataDir, {recursive: true})
}
console.log('Downloading database')
diff --git a/src/main.mjs b/src/main.mjs
index d001aca60902bc7fe41271c6fa7a0b6648607b15..5b2c125d8e7590afee82c794bf771accd656b2b7 100644
--- a/src/main.mjs
+++ b/src/main.mjs
@@ -3,7 +3,7 @@ import fs from 'fs/promises'
import fsSync from 'fs'
import path from 'path'
import { exec } from 'child_process'
-
+import { fileURLToPath } from "url"
import { countries, continents } from 'countries-list'
import { setting, setSetting, getSettingCmd } from './setting.mjs'
@@ -14,6 +14,9 @@ const v6db = setting.v6
const locFieldHash = setting.locFieldHash
const mainFieldHash = setting.mainFieldHash
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+
//---------------------------------------
// Database lookup
//---------------------------------------
@@ -235,7 +238,7 @@ export const updateDb = (_setting) => {
// However, db.js import many external modules, it makes slow down the startup time and uses more memory.
// Therefore, we use exec() to run the script in the other process.
return new Promise((resolve, reject) => {
- var cmd = 'node ' + path.resolve(__dirname, '..', 'script', 'updatedb.js')
+ var cmd = 'node ' + path.resolve(__dirname, '..', 'script', 'updatedb.mjs')
var arg
if(_setting){
var oldSetting = Object.assign({}, setting)

634
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff