mirror of
https://github.com/OneUptime/oneuptime.git
synced 2026-04-06 00:32:12 +02:00
feat(telemetry): add new Telemetry service (OTel, Syslog, Fluent, Metrics, Traces) and unified ingestion pipeline
- Add Telemetry service entrypoint - Telemetry/Index.ts: app bootstrap, routes mounting, infrastructure init and Telemetry SDK init. - Unified queue + worker - Telemetry/Jobs/TelemetryIngest/ProcessTelemetry.ts: single worker that dispatches queued jobs to specific processors (logs, traces, metrics, syslog, fluent logs). - Telemetry/Services/Queue/TelemetryQueueService.ts: central queue API and job payload types. - Per-type Queue wrappers (LogsQueueService, MetricsQueueService, TracesQueueService, FluentLogsQueueService, SyslogQueueService). - OpenTelemetry ingestion middleware and proto support - Telemetry/Middleware/OtelRequestMiddleware.ts: detect OTLP endpoint (logs/traces/metrics), decode protobuf bodies using protobufjs and set product type. - Telemetry/ProtoFiles/OTel/v1/*.proto: include common.proto, logs.proto, metrics.proto, resource.proto, traces.proto for OTLP v1 messages. - Ingest services - Telemetry/Services/OtelLogsIngestService.ts: parse incoming OTLP logs, map attributes, convert timestamps, batch insert logs. - Telemetry/Services/OtelTracesIngestService.ts: parse OTLP traces, build span rows, extract exceptions, batch insert spans and exceptions, save telemetry exception summary. - Telemetry/Services/OtelMetricsIngestService.ts: parse OTLP metrics, normalize datapoints, batch insert metrics and index metric name -> service map. - Telemetry/Services/SyslogIngestService.ts: syslog ingestion endpoints, parser integration, map syslog fields to attributes and logs. - Telemetry/Services/FluentLogsIngestService.ts: ingest Fluentd style logs, normalize entries and insert into log backend. - Telemetry/Services/OtelIngestBaseService.ts: helpers to resolve service name from attributes/headers. - Syslog parser and utilities - Telemetry/Utils/SyslogParser.ts: robust RFC5424 and RFC3164 parser, structured data extraction and sanitization. - Telemetry/Tests/Utils/SyslogParser.test.ts: unit tests for parser behavior. - Telemetry exception utilities - Telemetry/Utils/Exception.ts: generate exception fingerprint and upsert telemetry exception status (saveOrUpdateTelemetryException). - Queue & job integration - New integration with Common/Server/Infrastructure/Queue and QueueWorker, job id generation and telemetry job types. - Telemetry services add ingestion jobs instead of processing synchronously. - Config, build and dev tooling - Add Telemetry/package.json, package-lock.json, tsconfig.json, nodemon.json, jest config. - New script configs and dependencies (protobufjs, ts-node, jest, nodemon, etc). - Docker / environment updates - docker-compose.base.yml, docker-compose.dev.yml, docker-compose.yml: rename service from open-telemetry-ingest -> telemetry and wire TELEMETRY_* envs. - config.example.env: rename and consolidate environment variables (OPEN_TELEMETRY_* -> TELEMETRY_*, update hostnames and ports). - Tests/Scripts/status-check.sh: update ready-check target to telemetry/status/ready. - Other - Telemetry/Services/Queue/*: export helpers and legacy-compatible job interface shims. - Memory cleanup and batching safeguards across ingest services. - Logging and capture spans added to key code paths. BREAKING CHANGES / MIGRATION NOTES: - Environment variables and docker service names changed: - Replace OPEN_TELEMETRY_... vars with TELEMETRY_... (PORT, HOSTNAME, CONCURRENCY, DISABLE_TELEMETRY, etc). - docker-compose entries moved from "open-telemetry-ingest" to "telemetry" and image name changed to oneuptime/telemetry. - Update any deployment automation and monitoring checks referencing the old service name or endpoints. - Consumers: OTLP endpoints and behavior remain supported, but ingestion is now queued and processed asynchronously. Testing / Running: - Install deps in Telemetry/ (npm install) after syncing Common workspace. - Run dev: npx nodemon (nodemon.json) or build & start using provided scripts. - Run tests with jest (Telemetry test suite includes SyslogParser unit tests). Files added/modified (high level): - Added many files under Telemetry/: Index, Jobs, Middleware, ProtoFiles, Services, Utils, Tests, package and config artifacts. - Modified docker-compose.* and config.example.env and status check script to use new TELEMETRY service/vars.
This commit is contained in:
4
.github/workflows/build.yml
vendored
4
.github/workflows/build.yml
vendored
@@ -382,7 +382,7 @@ jobs:
|
||||
max_attempts: 3
|
||||
command: sudo docker build --no-cache -f ./ServerMonitorIngest/Dockerfile .
|
||||
|
||||
docker-build-open-telemetry-ingest:
|
||||
docker-build-telemetry:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
CI_PIPELINE_ID: ${{github.run_number}}
|
||||
@@ -403,7 +403,7 @@ jobs:
|
||||
with:
|
||||
timeout_minutes: 45
|
||||
max_attempts: 3
|
||||
command: sudo docker build --no-cache -f ./OpenTelemetryIngest/Dockerfile .
|
||||
command: sudo docker build --no-cache -f ./Telemetry/Dockerfile .
|
||||
|
||||
docker-build-incoming-request-ingest:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
6
.github/workflows/compile.yml
vendored
6
.github/workflows/compile.yml
vendored
@@ -319,7 +319,7 @@ jobs:
|
||||
max_attempts: 3
|
||||
command: cd ServerMonitorIngest && npm install && npm run compile && npm run dep-check
|
||||
|
||||
compile-open-telemetry-ingest:
|
||||
compile-telemetry:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
CI_PIPELINE_ID: ${{github.run_number}}
|
||||
@@ -329,12 +329,12 @@ jobs:
|
||||
with:
|
||||
node-version: latest
|
||||
- run: cd Common && npm install
|
||||
- name: Compile Open Telemetry Ingest
|
||||
- name: Compile Telemetry
|
||||
uses: nick-fields/retry@v3
|
||||
with:
|
||||
timeout_minutes: 30
|
||||
max_attempts: 3
|
||||
command: cd OpenTelemetryIngest && npm install && npm run compile && npm run dep-check
|
||||
command: cd Telemetry && npm install && npm run compile && npm run dep-check
|
||||
|
||||
|
||||
compile-incoming-request-ingest:
|
||||
|
||||
28
.github/workflows/release.yml
vendored
28
.github/workflows/release.yml
vendored
@@ -1108,7 +1108,7 @@ jobs:
|
||||
|
||||
|
||||
|
||||
open-telemetry-ingest-docker-image-deploy:
|
||||
telemetry-docker-image-deploy:
|
||||
needs: [generate-build-number, read-version]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
@@ -1117,8 +1117,8 @@ jobs:
|
||||
uses: docker/metadata-action@v4
|
||||
with:
|
||||
images: |
|
||||
oneuptime/open-telemetry-ingest
|
||||
ghcr.io/oneuptime/open-telemetry-ingest
|
||||
oneuptime/telemetry
|
||||
ghcr.io/oneuptime/telemetry
|
||||
tags: |
|
||||
type=raw,value=release,enable=true
|
||||
type=semver,value=${{needs.read-version.outputs.major_minor}}.${{needs.generate-build-number.outputs.build_number}},pattern={{version}},enable=true
|
||||
@@ -1144,7 +1144,7 @@ jobs:
|
||||
max_attempts: 3
|
||||
command: npm run prerun
|
||||
|
||||
# Build and deploy open-telemetry-ingest.
|
||||
# Build and deploy telemetry.
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: nick-fields/retry@v3
|
||||
@@ -1171,22 +1171,22 @@ jobs:
|
||||
VERSION="${{needs.read-version.outputs.major_minor}}.${{needs.generate-build-number.outputs.build_number}}"
|
||||
docker buildx build \
|
||||
--no-cache \
|
||||
--file ./OpenTelemetryIngest/Dockerfile \
|
||||
--file ./Telemetry/Dockerfile \
|
||||
--platform linux/amd64,linux/arm64 \
|
||||
--push \
|
||||
--tag oneuptime/open-telemetry-ingest:${VERSION} \
|
||||
--tag ghcr.io/oneuptime/open-telemetry-ingest:${VERSION} \
|
||||
--tag oneuptime/telemetry:${VERSION} \
|
||||
--tag ghcr.io/oneuptime/telemetry:${VERSION} \
|
||||
--build-arg GIT_SHA=${{ github.sha }} \
|
||||
--build-arg APP_VERSION=${VERSION} \
|
||||
--build-arg IS_ENTERPRISE_EDITION=false \
|
||||
.
|
||||
docker buildx build \
|
||||
--no-cache \
|
||||
--file ./OpenTelemetryIngest/Dockerfile \
|
||||
--file ./Telemetry/Dockerfile \
|
||||
--platform linux/amd64,linux/arm64 \
|
||||
--push \
|
||||
--tag oneuptime/open-telemetry-ingest:enterprise-${VERSION} \
|
||||
--tag ghcr.io/oneuptime/open-telemetry-ingest:enterprise-${VERSION} \
|
||||
--tag oneuptime/telemetry:enterprise-${VERSION} \
|
||||
--tag ghcr.io/oneuptime/telemetry:enterprise-${VERSION} \
|
||||
--build-arg GIT_SHA=${{ github.sha }} \
|
||||
--build-arg APP_VERSION=${VERSION} \
|
||||
--build-arg IS_ENTERPRISE_EDITION=true \
|
||||
@@ -2340,7 +2340,7 @@ jobs:
|
||||
- test-docker-image-deploy
|
||||
- probe-ingest-docker-image-deploy
|
||||
- server-monitor-ingest-docker-image-deploy
|
||||
- open-telemetry-ingest-docker-image-deploy
|
||||
- telemetry-docker-image-deploy
|
||||
- incoming-request-ingest-docker-image-deploy
|
||||
- probe-docker-image-deploy
|
||||
- admin-dashboard-docker-image-deploy
|
||||
@@ -2371,7 +2371,7 @@ jobs:
|
||||
"test",
|
||||
"probe-ingest",
|
||||
"server-monitor-ingest",
|
||||
"open-telemetry-ingest",
|
||||
"telemetry",
|
||||
"incoming-request-ingest",
|
||||
"probe",
|
||||
"admin-dashboard",
|
||||
@@ -2438,7 +2438,7 @@ jobs:
|
||||
|
||||
test-e2e-release-saas:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [open-telemetry-ingest-docker-image-deploy, publish-mcp-server, copilot-docker-image-deploy, docs-docker-image-deploy, api-reference-docker-image-deploy, workflow-docker-image-deploy, llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, probe-ingest-docker-image-deploy, server-monitor-ingest-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, read-version, nginx-docker-image-deploy, incoming-request-ingest-docker-image-deploy]
|
||||
needs: [telemetry-docker-image-deploy, publish-mcp-server, copilot-docker-image-deploy, docs-docker-image-deploy, api-reference-docker-image-deploy, workflow-docker-image-deploy, llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, probe-ingest-docker-image-deploy, server-monitor-ingest-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, read-version, nginx-docker-image-deploy, incoming-request-ingest-docker-image-deploy]
|
||||
env:
|
||||
CI_PIPELINE_ID: ${{github.run_number}}
|
||||
steps:
|
||||
@@ -2525,7 +2525,7 @@ jobs:
|
||||
test-e2e-release-self-hosted:
|
||||
runs-on: ubuntu-latest
|
||||
# After all the jobs runs
|
||||
needs: [open-telemetry-ingest-docker-image-deploy, publish-mcp-server, copilot-docker-image-deploy, incoming-request-ingest-docker-image-deploy, docs-docker-image-deploy, api-reference-docker-image-deploy, workflow-docker-image-deploy, llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, probe-ingest-docker-image-deploy, server-monitor-ingest-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, read-version, nginx-docker-image-deploy]
|
||||
needs: [telemetry-docker-image-deploy, publish-mcp-server, copilot-docker-image-deploy, incoming-request-ingest-docker-image-deploy, docs-docker-image-deploy, api-reference-docker-image-deploy, workflow-docker-image-deploy, llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, probe-ingest-docker-image-deploy, server-monitor-ingest-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, read-version, nginx-docker-image-deploy]
|
||||
env:
|
||||
CI_PIPELINE_ID: ${{github.run_number}}
|
||||
steps:
|
||||
|
||||
28
.github/workflows/test-release.yaml
vendored
28
.github/workflows/test-release.yaml
vendored
@@ -1245,7 +1245,7 @@ on:
|
||||
--build-arg IS_ENTERPRISE_EDITION=true \
|
||||
.
|
||||
|
||||
open-telemetry-ingest-docker-image-deploy:
|
||||
telemetry-docker-image-deploy:
|
||||
needs: [read-version, generate-build-number]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
@@ -1254,8 +1254,8 @@ on:
|
||||
uses: docker/metadata-action@v4
|
||||
with:
|
||||
images: |
|
||||
oneuptime/open-telemetry-ingest
|
||||
ghcr.io/oneuptime/open-telemetry-ingest
|
||||
oneuptime/telemetry
|
||||
ghcr.io/oneuptime/telemetry
|
||||
tags: |
|
||||
type=raw,value=test,enable=true
|
||||
type=raw,value=${{needs.read-version.outputs.major_minor}}.${{needs.generate-build-number.outputs.build_number}}-test,enable=true
|
||||
@@ -1309,26 +1309,26 @@ on:
|
||||
VERSION="${{needs.read-version.outputs.major_minor}}.${{needs.generate-build-number.outputs.build_number}}-test"
|
||||
docker buildx build \
|
||||
--no-cache \
|
||||
--file ./OpenTelemetryIngest/Dockerfile \
|
||||
--file ./Telemetry/Dockerfile \
|
||||
--platform linux/amd64,linux/arm64 \
|
||||
--push \
|
||||
--tag oneuptime/open-telemetry-ingest:test \
|
||||
--tag oneuptime/open-telemetry-ingest:${VERSION} \
|
||||
--tag ghcr.io/oneuptime/open-telemetry-ingest:test \
|
||||
--tag ghcr.io/oneuptime/open-telemetry-ingest:${VERSION} \
|
||||
--tag oneuptime/telemetry:test \
|
||||
--tag oneuptime/telemetry:${VERSION} \
|
||||
--tag ghcr.io/oneuptime/telemetry:test \
|
||||
--tag ghcr.io/oneuptime/telemetry:${VERSION} \
|
||||
--build-arg GIT_SHA=${{ github.sha }} \
|
||||
--build-arg APP_VERSION=${VERSION} \
|
||||
--build-arg IS_ENTERPRISE_EDITION=false \
|
||||
.
|
||||
docker buildx build \
|
||||
--no-cache \
|
||||
--file ./OpenTelemetryIngest/Dockerfile \
|
||||
--file ./Telemetry/Dockerfile \
|
||||
--platform linux/amd64,linux/arm64 \
|
||||
--push \
|
||||
--tag oneuptime/open-telemetry-ingest:enterprise-test \
|
||||
--tag oneuptime/open-telemetry-ingest:enterprise-${VERSION} \
|
||||
--tag ghcr.io/oneuptime/open-telemetry-ingest:enterprise-test \
|
||||
--tag ghcr.io/oneuptime/open-telemetry-ingest:enterprise-${VERSION} \
|
||||
--tag oneuptime/telemetry:enterprise-test \
|
||||
--tag oneuptime/telemetry:enterprise-${VERSION} \
|
||||
--tag ghcr.io/oneuptime/telemetry:enterprise-test \
|
||||
--tag ghcr.io/oneuptime/telemetry:enterprise-${VERSION} \
|
||||
--build-arg GIT_SHA=${{ github.sha }} \
|
||||
--build-arg APP_VERSION=${VERSION} \
|
||||
--build-arg IS_ENTERPRISE_EDITION=true \
|
||||
@@ -2236,7 +2236,7 @@ on:
|
||||
|
||||
test-helm-chart:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [infrastructure-agent-deploy, publish-mcp-server, llm-docker-image-deploy, publish-terraform-provider, open-telemetry-ingest-docker-image-deploy, copilot-docker-image-deploy, docs-docker-image-deploy, worker-docker-image-deploy, workflow-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, api-reference-docker-image-deploy, test-server-docker-image-deploy, test-docker-image-deploy, probe-ingest-docker-image-deploy, server-monitor-ingest-docker-image-deploy, probe-docker-image-deploy, dashboard-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, accounts-docker-image-deploy, otel-collector-docker-image-deploy, status-page-docker-image-deploy, nginx-docker-image-deploy, e2e-docker-image-deploy, incoming-request-ingest-docker-image-deploy]
|
||||
needs: [infrastructure-agent-deploy, publish-mcp-server, llm-docker-image-deploy, publish-terraform-provider, telemetry-docker-image-deploy, copilot-docker-image-deploy, docs-docker-image-deploy, worker-docker-image-deploy, workflow-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, api-reference-docker-image-deploy, test-server-docker-image-deploy, test-docker-image-deploy, probe-ingest-docker-image-deploy, server-monitor-ingest-docker-image-deploy, probe-docker-image-deploy, dashboard-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, accounts-docker-image-deploy, otel-collector-docker-image-deploy, status-page-docker-image-deploy, nginx-docker-image-deploy, e2e-docker-image-deploy, incoming-request-ingest-docker-image-deploy]
|
||||
env:
|
||||
CI_PIPELINE_ID: ${{github.run_number}}
|
||||
steps:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
name: OpenTelemetryIngest Test
|
||||
name: Telemetry Test
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
@@ -17,5 +17,5 @@ jobs:
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: latest
|
||||
- run: cd OpenTelemetryIngest && npm install && npm run test
|
||||
- run: cd Telemetry && npm install && npm run test
|
||||
|
||||
Reference in New Issue
Block a user