monitor probe service

.github/workflows/build.yml

@@ -179,6 +179,21 @@ jobs:
       # build image probe api
       - name: build docker image
         run: sudo docker build -f ./Ingestor/Dockerfile .
+  
+  docker-build-telemetry-ingestor:
+    runs-on: ubuntu-latest
+    env:
+      CI_PIPELINE_ID: ${{github.run_number}}
+    steps:
+      - name: Checkout 
+        uses: actions/checkout@v2
+      
+      - name: Preinstall 
+        run: npm run prerun
+        
+      # build image probe api
+      - name: build docker image
+        run: sudo docker build -f ./TelemetryIngestor/Dockerfile .
 
   docker-build-status-page:
     runs-on: ubuntu-latest

.github/workflows/compile.yml

@@ -214,6 +214,20 @@ jobs:
       - run: cd Model && npm install
       - run: cd CommonServer && npm install
       - run: cd Ingestor && npm install && npm run compile && npm run dep-check
+
+  compile-telemetry-ingestor:
+    runs-on: ubuntu-latest
+    env:
+      CI_PIPELINE_ID: ${{github.run_number}}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 18.3.0
+      - run: cd Common && npm install
+      - run: cd Model && npm install
+      - run: cd CommonServer && npm install
+      - run: cd TelemetryIngestor && npm install && npm run compile && npm run dep-check
   
 
   compile-status-page:

Copilot/Dockerfile.tpl

@@ -87,5 +87,4 @@ COPY ./Copilot /usr/src/app
 RUN npm run compile
 #Run the app
 CMD [ "npm", "start" ]
-{{ end }}
-
+{{ end }}

Ingestor/Index.ts

@@ -1,8 +1,6 @@
 import AliveAPI from "./API/Alive";
-import FluentIngestAPI from "./API/FluentIngest";
 import IncomingRequestAPI from "./API/IncomingRequest";
 import MonitorAPI from "./API/Monitor";
-import OTelIngestAPI from "./API/OTelIngest";
 import Ingestor from "./API/Probe";
 import RegisterAPI from "./API/Register";
 import ServerMonitorAPI from "./API/ServerMonitor";
@@ -26,8 +24,6 @@ app.use([`/${APP_NAME}`, "/"], RegisterAPI);
 app.use([`/${APP_NAME}`, "/"], MonitorAPI);
 app.use([`/${APP_NAME}`, "/"], Ingestor);
 app.use([`/${APP_NAME}`, "/"], IncomingRequestAPI);
-app.use([`/${APP_NAME}`, "/"], OTelIngestAPI);
-app.use([`/${APP_NAME}`, "/"], FluentIngestAPI);
 app.use([`/${APP_NAME}`, "/"], ServerMonitorAPI);
 
 const init: PromiseVoidFunction = async (): Promise<void> => {

TelemetryIngestor/Index.ts

@@ -1,11 +1,5 @@
-import AliveAPI from "./API/Alive";
-import FluentIngestAPI from "./API/FluentIngest";
-import IncomingRequestAPI from "./API/IncomingRequest";
-import MonitorAPI from "./API/Monitor";
-import OTelIngestAPI from "./API/OTelIngest";
-import Ingestor from "./API/Probe";
-import RegisterAPI from "./API/Register";
-import ServerMonitorAPI from "./API/ServerMonitor";
+import FluentIngestAPI from "../TelemetryIngestor/API/FluentIngest";
+import OTelIngestAPI from "../TelemetryIngestor/API/OTelIngest";
 import { PromiseVoidFunction } from "Common/Types/FunctionTypes";
 import { ClickhouseAppInstance } from "CommonServer/Infrastructure/ClickhouseDatabase";
 import { PostgresAppInstance } from "CommonServer/Infrastructure/PostgresDatabase";
@@ -19,16 +13,10 @@ import "ejs";
 
 const app: ExpressApplication = Express.getExpressApp();
 
-const APP_NAME: string = "ingestor";
+const APP_NAME: string = "telemetry-ingestor";
 
-app.use([`/${APP_NAME}`, "/"], AliveAPI);
-app.use([`/${APP_NAME}`, "/"], RegisterAPI);
-app.use([`/${APP_NAME}`, "/"], MonitorAPI);
-app.use([`/${APP_NAME}`, "/"], Ingestor);
-app.use([`/${APP_NAME}`, "/"], IncomingRequestAPI);
 app.use([`/${APP_NAME}`, "/"], OTelIngestAPI);
 app.use([`/${APP_NAME}`, "/"], FluentIngestAPI);
-app.use([`/${APP_NAME}`, "/"], ServerMonitorAPI);
 
 const init: PromiseVoidFunction = async (): Promise<void> => {
   try {

TelemetryIngestor/Middleware/TelemetryIngest.ts

@@ -1,4 +1,3 @@
-import { ProbeExpressRequest } from "../Types/Request";
 import DiskSize from "Common/Types/DiskSize";
 import BadRequestException from "Common/Types/Exception/BadRequestException";
 import ProductType from "Common/Types/MeteredPlan/ProductType";
@@ -23,7 +22,7 @@ export interface TelemetryRequest extends ExpressRequest {
 
 export default class TelemetryIngest {
   public static async isAuthorizedServiceMiddleware(
-    req: ProbeExpressRequest,
+    req: ExpressRequest,
     _res: ExpressResponse,
     next: NextFunction,
   ): Promise<void> {

TelemetryIngestor/package-lock.json

@@ -1,11 +1,11 @@
 {
-    "name": "@oneuptime/ingestor",
+    "name": "@oneuptime/telemetry-ingestor",
     "version": "1.0.0",
     "lockfileVersion": 3,
     "requires": true,
     "packages": {
         "": {
-            "name": "@oneuptime/ingestor",
+            "name": "@oneuptime/telemetry-ingestor",
             "version": "1.0.0",
             "license": "Apache-2.0",
             "dependencies": {
@@ -25,21 +25,21 @@
             }
         },
         "../Common": {
-            "name": "common",
+            "name": "@oneuptime/common",
             "version": "1.0.0",
-            "license": "MIT",
+            "license": "Apache-2.0",
             "dependencies": {
-                "@types/crypto-js": "^4.1.1",
+                "@types/crypto-js": "^4.2.2",
                 "@types/uuid": "^8.3.4",
-                "axios": "^1.6.2",
+                "axios": "^1.7.2",
                 "crypto-js": "^4.1.1",
                 "json5": "^2.2.3",
-                "moment": "^2.29.2",
-                "moment-timezone": "^0.5.40",
-                "posthog-js": "^1.77.0",
-                "reflect-metadata": "^0.1.13",
+                "moment": "^2.30.1",
+                "moment-timezone": "^0.5.45",
+                "posthog-js": "^1.138.0",
+                "reflect-metadata": "^0.2.2",
                 "slugify": "^1.6.5",
-                "typeorm": "^0.3.6",
+                "typeorm": "^0.3.20",
                 "uuid": "^8.3.2"
             },
             "devDependencies": {
@@ -51,40 +51,49 @@
             }
         },
         "../CommonServer": {
-            "name": "common-server",
+            "name": "@oneuptime/common-server",
             "version": "1.0.0",
-            "license": "MIT",
+            "license": "Apache-2.0",
             "dependencies": {
-                "@clickhouse/client": "^0.2.1",
-                "@elastic/elasticsearch": "^8.1.0",
+                "@clickhouse/client": "^0.2.10",
+                "@elastic/elasticsearch": "^8.12.1",
                 "@opentelemetry/api": "^1.7.0",
-                "@opentelemetry/auto-instrumentations-node": "^0.40.1",
-                "@opentelemetry/sdk-node": "^0.45.1",
-                "@socket.io/redis-adapter": "^8.2.1",
+                "@opentelemetry/api-logs": "^0.49.1",
+                "@opentelemetry/auto-instrumentations-node": "^0.43.0",
+                "@opentelemetry/exporter-logs-otlp-http": "^0.49.1",
+                "@opentelemetry/exporter-metrics-otlp-proto": "^0.49.1",
+                "@opentelemetry/exporter-trace-otlp-proto": "^0.49.1",
+                "@opentelemetry/id-generator-aws-xray": "^1.2.1",
+                "@opentelemetry/sdk-logs": "^0.49.1",
+                "@opentelemetry/sdk-metrics": "^1.21.0",
+                "@opentelemetry/sdk-node": "^0.48.0",
+                "@opentelemetry/sdk-trace-node": "^1.21.0",
+                "@types/crypto-js": "^4.2.2",
+                "acme-client": "^5.3.0",
                 "airtable": "^0.12.2",
-                "axios": "^1.6.2",
-                "bullmq": "^3.6.6",
+                "bullmq": "^5.3.3",
                 "Common": "file:../Common",
                 "cookie-parser": "^1.4.6",
                 "cors": "^2.8.5",
                 "cron-parser": "^4.8.1",
-                "dotenv": "^16.0.0",
-                "ejs": "^3.1.8",
-                "express": "^4.17.3",
+                "crypto-js": "^4.2.0",
+                "dotenv": "^16.4.4",
+                "ejs": "^3.1.10",
+                "express": "^4.19.2",
                 "ioredis": "^5.3.2",
                 "json2csv": "^5.0.7",
                 "jsonwebtoken": "^9.0.0",
-                "markdown-it": "^13.0.1",
+                "marked": "^12.0.2",
                 "Model": "file:../Model",
-                "nodemailer": "^6.7.3",
+                "node-cron": "^3.0.3",
+                "nodemailer": "^6.9.10",
                 "pg": "^8.7.3",
-                "socket.io": "^4.7.2",
+                "redis-semaphore": "^5.5.1",
+                "socket.io": "^4.7.4",
                 "stripe": "^10.17.0",
-                "twilio": "^4.19.3",
-                "typeorm": "^0.3.10",
-                "typeorm-extension": "^2.2.13",
-                "vm2": "^3.9.14",
-                "winston": "^3.6.0"
+                "twilio": "^4.22.0",
+                "typeorm": "^0.3.20",
+                "typeorm-extension": "^2.2.13"
             },
             "devDependencies": {
                 "@faker-js/faker": "^6.3.1",
@@ -95,8 +104,8 @@
                 "@types/jest": "^27.4.1",
                 "@types/json2csv": "^5.0.3",
                 "@types/jsonwebtoken": "^8.5.9",
-                "@types/markdown-it": "^12.2.3",
                 "@types/node": "^17.0.22",
+                "@types/node-cron": "^3.0.7",
                 "@types/nodemailer": "^6.4.7",
                 "jest": "^27.5.1",
                 "jest-mock-extended": "^3.0.5",
@@ -104,12 +113,12 @@
             }
         },
         "../Model": {
-            "name": "model",
+            "name": "@oneuptime/model",
             "version": "1.0.0",
-            "license": "ISC",
+            "license": "Apache-2.0",
             "dependencies": {
                 "Common": "file:../Common",
-                "typeorm": "^0.3.17"
+                "typeorm": "^0.3.20"
             },
             "devDependencies": {
                 "@types/jest": "^27.4.1",

TelemetryIngestor/package.json

@@ -1,5 +1,5 @@
 {
-    "name": "@oneuptime/ingestor",
+    "name": "@oneuptime/telemetry-ingestor",
     "version": "1.0.0",
     "description": "",
     "main": "index.js",

Tests/Scripts/status-check.sh

@@ -49,6 +49,10 @@ bash $scriptDir/endpoint-status.sh "Ingestor (Ready Check)" $HOST_TO_CHECK/inges
 
 bash $scriptDir/endpoint-status.sh "Ingestor (Status Check)" $HOST_TO_CHECK/ingestor/status
 
+bash $scriptDir/endpoint-status.sh "Telemetry Ingestor (Ready Check)" $HOST_TO_CHECK/telemetry-ingestor/status/ready
+
+bash $scriptDir/endpoint-status.sh "Telemetry Ingestor (Status Check)" $HOST_TO_CHECK/telemetry-ingestor/status
+
 echo "🚀 OneUptime is up! 🚀"
 echo ""
 echo "🎉🎉🎉  Done! 🎉🎉🎉"

config.example.env

@@ -108,6 +108,7 @@ SERVER_OTEL_COLLECTOR_HOSTNAME=otel-collector
 
 APP_PORT=3002
 INGESTOR_PORT=3400
+TELEMETRY_INGESTOR_PORT=3674
 PROBE_PORT=3500
 TEST_SERVER_PORT=3800
 ACCOUNTS_PORT=3003

docker-compose.base.yml

@@ -364,6 +364,18 @@ services:
             driver: "local"
             options:
                 max-size: "1000m"
+
+    telemetry-ingestor:
+        networks:
+          - oneuptime
+        restart: always
+        environment:
+            <<: *common-server-variables
+            PORT: ${TELEMETRY_INGESTOR_PORT}
+        logging:
+            driver: "local"
+            options:
+                max-size: "1000m"
             
     
     e2e:

docker-compose.dev.yml

@@ -296,6 +296,30 @@ services:
             context: .
             dockerfile: ./Ingestor/Dockerfile
 
+    telemetry-ingestor:
+        volumes:
+            - ./TelemetryIngestor:/usr/src/app
+            # Use node modules of the container and not host system.
+            # https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder
+            - /usr/src/app/node_modules/
+            - ./Common:/usr/src/Common
+            - ./Model:/usr/src/Model
+            - ./CommonServer:/usr/src/CommonServer
+            - ./CommonUI:/usr/src/CommonUI
+            - /usr/src/Common/node_modules/
+            - /usr/src/CommonUI/node_modules/
+            - /usr/src/CommonServer/node_modules/
+            - /usr/src/Model/node_modules/
+        ports:
+          - '9932:9229' # Debugging port.
+        extends:
+            file: ./docker-compose.base.yml
+            service: telemetry-ingestor
+        build:
+            network: host
+            context: .
+            dockerfile: ./TelemetryIngestor/Dockerfile
+
 
     # Fluentd. Required only for development. In  production its the responsibility of the customer to run fluentd and pipe logs to OneUptime.
     # We run this container just for development, to see if logs are piped. 

docker-compose.yml

@@ -83,6 +83,12 @@ services:
             file: ./docker-compose.base.yml
             service: ingestor
 
+    telemetry-ingestor:
+        image:  oneuptime/telemetry-ingestor:${APP_TAG}
+        extends:
+            file: ./docker-compose.base.yml
+            service: telemetry-ingestor
+
     isolated-vm:
         image:  oneuptime/isolated-vm:${APP_TAG}
         extends: