github/oneuptime
1
1
Fork 0
mirror of https://github.com/OneUptime/oneuptime.git synced 2026-04-06 00:32:12 +02:00
Code Issues 193 Packages Projects Releases Wiki Activity

Refactor OpenTelemetry Ingest Dockerfile and configuration; update environment variables and docker-compose for new service integration

Browse Source
This commit is contained in:
Simon Larsen
2024-11-21 17:08:35 +00:00
parent eec51342de
commit 3a1f5c7120
26 changed files with 497 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
.github/workflows/build.yml vendored
View File

@@ -255,6 +255,21 @@ jobs:
- name: build docker image
run: sudo docker build -f ./Ingestor/Dockerfile .
docker-build-open-telemetry-ingest:
runs-on: ubuntu-latest
env:
CI_PIPELINE_ID: ${{github.run_number}}
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Preinstall
run: npm run prerun
# build image probe api
- name: build docker image
run: sudo docker build -f ./OpenTelemetryIngest/Dockerfile .
docker-build-incoming-request-ingest:
runs-on: ubuntu-latest
env:

13
.github/workflows/compile.yml vendored
View File

@@ -216,6 +216,19 @@ jobs:
- run: cd Common && npm install
- run: cd Ingestor && npm install && npm run compile && npm run dep-check
compile-open-telemetry-ingest:
runs-on: ubuntu-latest
env:
CI_PIPELINE_ID: ${{github.run_number}}
steps:
- uses: actions/checkout@v2
- uses: actions/setup-node@v2
with:
node-version: 18.3.0
- run: cd Common && npm install
- run: cd OpenTelemetryIngest && npm install && npm run compile && npm run dep-check
compile-incoming-request-ingest:
runs-on: ubuntu-latest
env:

64
.github/workflows/release.yml vendored
View File

@@ -613,6 +613,66 @@ jobs:
GIT_SHA=${{ github.sha }}
APP_VERSION=7.0.${{needs.generate-build-number.outputs.build_number}}
open-telemetry-ingest-docker-image-deploy:
needs: [generate-build-number]
runs-on: ubuntu-latest
steps:
- name: Docker Meta
id: meta
uses: docker/metadata-action@v4
with:
images: |
oneuptime/open-telemetry-ingest
ghcr.io/oneuptime/open-telemetry-ingest
tags: |
type=raw,value=release,enable=true
type=semver,value=7.0.${{needs.generate-build-number.outputs.build_number}},pattern={{version}},enable=true
- uses: actions/checkout@v4
with:
ref: ${{ github.ref }}
- uses: actions/setup-node@v2
with:
node-version: 18.3.0
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Generate Dockerfile from Dockerfile.tpl
run: npm run prerun
# Build and deploy open-telemetry-ingest.
- name: Login to Docker Hub
uses: docker/login-action@v2.2.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v2.2.0
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push
uses: docker/build-push-action@v4
with:
file: ./OpenTelemetryIngest/Dockerfile
context: .
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
GIT_SHA=${{ github.sha }}
APP_VERSION=7.0.${{needs.generate-build-number.outputs.build_number}}
incoming-request-ingest-docker-image-deploy:
needs: [generate-build-number]
@@ -1530,7 +1590,7 @@ jobs:
test-e2e-release-saas:
runs-on: ubuntu-latest
needs: [copilot-docker-image-deploy, fluent-ingest-docker-image-deploy, docs-docker-image-deploy, api-reference-docker-image-deploy, workflow-docker-image-deploy, llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, haraka-docker-image-deploy, ingestor-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, nginx-docker-image-deploy, incoming-request-ingest-docker-image-deploy]
needs: [open-telemetry-ingest-docker-image-deploy, copilot-docker-image-deploy, fluent-ingest-docker-image-deploy, docs-docker-image-deploy, api-reference-docker-image-deploy, workflow-docker-image-deploy, llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, haraka-docker-image-deploy, ingestor-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, nginx-docker-image-deploy, incoming-request-ingest-docker-image-deploy]
env:
CI_PIPELINE_ID: ${{github.run_number}}
steps:
@@ -1583,7 +1643,7 @@ jobs:
test-e2e-release-self-hosted:
runs-on: ubuntu-latest
# After all the jobs runs
needs: [copilot-docker-image-deploy, incoming-request-ingest-docker-image-deploy, fluent-ingest-docker-image-deploy, docs-docker-image-deploy, api-reference-docker-image-deploy, workflow-docker-image-deploy, llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, haraka-docker-image-deploy, ingestor-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, nginx-docker-image-deploy]
needs: [open-telemetry-ingest-docker-image-deploy, copilot-docker-image-deploy, incoming-request-ingest-docker-image-deploy, fluent-ingest-docker-image-deploy, docs-docker-image-deploy, api-reference-docker-image-deploy, workflow-docker-image-deploy, llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, haraka-docker-image-deploy, ingestor-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, nginx-docker-image-deploy]
env:
CI_PIPELINE_ID: ${{github.run_number}}
steps:

122
.github/workflows/test-release.yaml vendored
View File

@@ -721,6 +721,128 @@ jobs:
GIT_SHA=${{ github.sha }}
APP_VERSION=7.0.${{needs.generate-build-number.outputs.build_number}}
ingestor-docker-image-deploy:
needs: generate-build-number
runs-on: ubuntu-latest
steps:
- name: Docker Meta
id: meta
uses: docker/metadata-action@v4
with:
images: |
oneuptime/ingestor
ghcr.io/oneuptime/ingestor
tags: |
type=raw,value=test,enable=true
type=semver,value=7.0.${{needs.generate-build-number.outputs.build_number}}-test,pattern={{version}},enable=true
- uses: actions/checkout@v4
with:
ref: ${{ github.ref }}
- uses: actions/setup-node@v2
with:
node-version: 18.3.0
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Generate Dockerfile from Dockerfile.tpl
run: npm run prerun
# Build and deploy ingestor.
- name: Login to Docker Hub
uses: docker/login-action@v2.2.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v2.2.0
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push
uses: docker/build-push-action@v4
with:
file: ./Ingestor/Dockerfile
context: .
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
GIT_SHA=${{ github.sha }}
APP_VERSION=7.0.${{needs.generate-build-number.outputs.build_number}}
open-telemetry-ingest-docker-image-deploy:
needs: generate-build-number
runs-on: ubuntu-latest
steps:
- name: Docker Meta
id: meta
uses: docker/metadata-action@v4
with:
images: |
oneuptime/open-telemetry-ingest
ghcr.io/oneuptime/open-telemetry-ingest
tags: |
type=raw,value=test,enable=true
type=semver,value=7.0.${{needs.generate-build-number.outputs.build_number}}-test,pattern={{version}},enable=true
- uses: actions/checkout@v4
with:
ref: ${{ github.ref }}
- uses: actions/setup-node@v2
with:
node-version: 18.3.0
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Generate Dockerfile from Dockerfile.tpl
run: npm run prerun
# Build and deploy incoming-request-ingest.
- name: Login to Docker Hub
uses: docker/login-action@v2.2.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v2.2.0
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push
uses: docker/build-push-action@v4
with:
file: ./OpenTelemetryIngest/Dockerfile
context: .
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
GIT_SHA=${{ github.sha }}
APP_VERSION=7.0.${{needs.generate-build-number.outputs.build_number}}
fluent-ingest-docker-image-deploy:
needs: generate-build-number
runs-on: ubuntu-latest

21
.github/workflows/test.open-telemetry-ingest.yaml vendored Normal file
View File

@@ -0,0 +1,21 @@
name: OpenTelemetryIngest Test
on:
pull_request:
push:
branches-ignore:
- 'hotfix-*' # excludes hotfix branches
- 'release'
jobs:
test:
runs-on: ubuntu-latest
env:
CI_PIPELINE_ID: ${{github.run_number}}
steps:
- uses: actions/checkout@v2
- uses: actions/setup-node@v2
with:
node-version: 18.3.0
- run: cd OpenTelemetryIngest && npm install && npm run test

16
.vscode/launch.json vendored
View File

@@ -189,11 +189,25 @@
"restart": true,
"autoAttachChildProcesses": true
},
{
"address": "127.0.0.1",
"localRoot": "${workspaceFolder}/OpenTelemetryIngest",
"name": "OpenTelemetryIngest: Debug with Docker",
"port": 9937,
"remoteRoot": "/usr/src/app",
"request": "attach",
"skipFiles": [
"<node_internals>/**"
],
"type": "node",
"restart": true,
"autoAttachChildProcesses": true
},
{
"address": "127.0.0.1",
"localRoot": "${workspaceFolder}/FluentIngest",
"name": "Fluent Ingest: Debug with Docker",
"port": 9937,
"port": 9938,
"remoteRoot": "/usr/src/app",
"request": "attach",
"skipFiles": [

6
Common/Server/EnvironmentConfig.ts
View File

@@ -94,6 +94,12 @@ export const IngestorHostname: Hostname = Hostname.fromString(
}`,
);
export const OpenTelemetryIngestHostname: Hostname = Hostname.fromString(
`${process.env["SERVER_OPEN_TELEMETRY_INGEST_HOSTNAME"] || "localhost"}:${
process.env["OPEN_TELEMETRY_INGEST_PORT"] || 80
}`,
);
export const IncomingRequestIngestHostname: Hostname = Hostname.fromString(
`${process.env["SERVER_INCOMING_REQUEST_INGEST_HOSTNAME"] || "localhost"}:${
process.env["INCOMING_REQUEST_INGEST_PORT"] || 80

12
OpenTelemetryIngest/Middleware/TelemetryIngest.ts → Common/Server/Middleware/TelemetryIngest.ts
View File

@@ -1,13 +1,13 @@
import BadRequestException from "Common/Types/Exception/BadRequestException";
import ProductType from "Common/Types/MeteredPlan/ProductType";
import ObjectID from "Common/Types/ObjectID";
import BadRequestException from "../../Types/Exception/BadRequestException";
import ProductType from "../../Types/MeteredPlan/ProductType";
import ObjectID from "../../Types/ObjectID";
import {
ExpressRequest,
ExpressResponse,
NextFunction,
} from "Common/Server/Utils/Express";
import TelemetryIngestionKeyService from "Common/Server/Services/TelemetryIngestionKeyService";
import TelemetryIngestionKey from "Common/Models/DatabaseModels/TelemetryIngestionKey";
} from "../../Server/Utils/Express";
import TelemetryIngestionKeyService from "../../Server/Services/TelemetryIngestionKeyService";
import TelemetryIngestionKey from "../../Models/DatabaseModels/TelemetryIngestionKey";
export interface TelemetryRequest extends ExpressRequest {
projectId: ObjectID; // Project ID

0
OpenTelemetryIngest/Service/OTelIngest.ts → Common/Server/Services/OpenTelemetryIngestService.ts
View File

4
Common/ServiceRoute.ts
View File

@@ -28,6 +28,10 @@ export const AdminDashboardRoute: Route = new Route("/admin");
export const IngestorRoute: Route = new Route("/ingestor");
export const OpenTelemetryIngestRoute: Route = new Route(
"/open-telemetry-ingest",
);
export const IncomingRequestIngestRoute: Route = new Route(
"/incoming-request-ingest",
);

10
Common/UI/Config.ts
View File

@@ -17,6 +17,7 @@ import {
WorkflowRoute,
FluentIngestRoute,
IncomingRequestIngestRoute,
OpenTelemetryIngestRoute,
} from "Common/ServiceRoute";
import Hostname from "Common/Types/API/Hostname";
import Protocol from "Common/Types/API/Protocol";
@@ -69,6 +70,9 @@ export const WORKFLOW_HOSTNAME: Hostname = Hostname.fromString(HOST);
export const INGESTOR_HOSTNAME: Hostname = Hostname.fromString(HOST);
export const OPEN_TELEMETRY_INGEST_HOSTNAME: Hostname =
Hostname.fromString(HOST);
export const INCOMING_REQUEST_INGEST_HOSTNAME: Hostname =
Hostname.fromString(HOST);
@@ -108,6 +112,12 @@ export const STATUS_PAGE_API_URL: URL = new URL(
StatusPageApiRoute,
);
export const OPEN_TELEMETRY_INGEST_URL: URL = new URL(
HTTP_PROTOCOL,
OPEN_TELEMETRY_INGEST_HOSTNAME,
OpenTelemetryIngestRoute,
);
export const FLUENT_INGEST_URL: URL = new URL(
HTTP_PROTOCOL,
FLUENT_INGEST_HOSTNAME,

47
E2E/Tests/OpenTelemetryIngest/StatusCheck.spec.ts Normal file
View File

@@ -0,0 +1,47 @@
import { BASE_URL } from "../../Config";
import { Page, expect, test } from "@playwright/test";
import URL from "Common/Types/API/URL";
test.describe("check live and health check of the open-telemetry-ingest", () => {
test("check if open-telemetry-ingest status is ok", async ({
page,
}: {
page: Page;
}) => {
await page.goto(
`${URL.fromString(BASE_URL.toString())
.addRoute("/open-telemetry-ingest/status")
.toString()}`,
);
const content: string = await page.content();
expect(content).toContain('{"status":"ok"}');
});
test("check if open-telemetry-ingest is ready", async ({
page,
}: {
page: Page;
}) => {
await page.goto(
`${URL.fromString(BASE_URL.toString())
.addRoute("/open-telemetry-ingest/status/ready")
.toString()}`,
);
const content: string = await page.content();
expect(content).toContain('{"status":"ok"}');
});
test("check if open-telemetry-ingest is live", async ({
page,
}: {
page: Page;
}) => {
await page.goto(
`${URL.fromString(BASE_URL.toString())
.addRoute("/open-telemetry-ingest/status/live")
.toString()}`,
);
const content: string = await page.content();
expect(content).toContain('{"status":"ok"}');
});
});

4
FluentIngest/API/FluentIngest.ts
View File

@@ -1,6 +1,6 @@
import TelemetryIngest, {
TelemetryRequest,
} from "../../Ingestor/Middleware/TelemetryIngest";
} from "Common/Server/Middleware/TelemetryIngest";
import OneUptimeDate from "Common/Types/Date";
import { JSONObject } from "Common/Types/JSON";
import ProductType from "Common/Types/MeteredPlan/ProductType";
@@ -15,7 +15,7 @@ import logger from "Common/Server/Utils/Logger";
import Response from "Common/Server/Utils/Response";
import Log from "Common/Models/AnalyticsModels/Log";
import LogSeverity from "Common/Types/Log/LogSeverity";
import OTelIngestService from "../../Ingestor/Service/OTelIngest";
import OTelIngestService from "Common/Server/Services/OpenTelemetryIngestService";
import ObjectID from "Common/Types/ObjectID";
import JSONFunctions from "Common/Types/JSONFunctions";

4
HelmChart/Public/oneuptime/templates/_helpers.tpl
View File

@@ -60,6 +60,8 @@ Usage:
value: {{ $.Release.Name }}-app.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }}
- name: SERVER_INGESTOR_HOSTNAME
value: {{ $.Release.Name }}-ingestor.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }}
- name: OPEN_TELEMETRY_INGEST_HOSTNAME
value: {{ $.Release.Name }}-open-telemetry-ingest.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }}
- name: SERVER_INCOMING_REQUEST_INGEST_HOSTNAME
value: {{ $.Release.Name }}-incoming-request-ingest.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }}
- name: SERVER_FLUENT_INGEST_HOSTNAME
@@ -81,6 +83,8 @@ Usage:
value: {{ $.Values.port.app | squote }}
- name: INGESTOR_PORT
value: {{ $.Values.port.ingestor | squote }}
- name: OPEN_TELEMETRY_INGEST_PORT
value: {{ $.Values.port.openTelemetryIngest | squote }}
- name: INCOMING_REQUEST_INGEST_PORT
value: {{ $.Values.port.incomingRequestIngest | squote }}
- name: FLUENT_INGEST_PORT

111
HelmChart/Public/oneuptime/templates/open-telemetry-ingest.yaml Normal file
View File

@@ -0,0 +1,111 @@
# OneUptime open-telemetry-ingest Deployment
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ printf "%s-%s" $.Release.Name "open-telemetry-ingest" }}
namespace: {{ $.Release.Namespace }}
labels:
app: {{ printf "%s-%s" $.Release.Name "open-telemetry-ingest" }}
app.kubernetes.io/part-of: oneuptime
app.kubernetes.io/managed-by: Helm
appname: oneuptime
date: "{{ now | unixEpoch }}"
spec:
selector:
matchLabels:
app: {{ printf "%s-%s" $.Release.Name "open-telemetry-ingest" }}
{{- if $.Values.deployment.open-telemetry-ingest.replicaCount }}
replicas: {{ $.Values.deployment.open-telemetry-ingest.replicaCount }}
{{- else }}
replicas: {{ $.Values.deployment.replicaCount }}
{{- end }}
template:
metadata:
labels:
app: {{ printf "%s-%s" $.Release.Name "open-telemetry-ingest" }}
date: "{{ now | unixEpoch }}"
appname: oneuptime
spec:
volumes:
- name: greenlockrc
emptyDir:
sizeLimit: "1Gi"
{{- if $.Values.podSecurityContext }}
securityContext: {{- $.Values.podSecurityContext | toYaml | nindent 8 }}
{{- end }}
{{- if $.Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml $.Values.imagePullSecrets | nindent 8 }}
{{- end }}
{{- if $.Values.affinity }}
affinity: {{- $.Values.affinity | toYaml | nindent 8 }}
{{- end }}
{{- if $.Values.tolerations }}
tolerations: {{- $.Values.tolerations | toYaml | nindent 8 }}
{{- end }}
{{- if $.Values.nodeSelector }}
nodeSelector: {{- $.Values.nodeSelector | toYaml | nindent 8 }}
{{- end }}
containers:
- image: {{ printf "%s/%s/%s:%s" $.Values.image.registry $.Values.image.repository "open-telemetry-ingest" $.Values.image.tag }}
name: {{ printf "%s-%s" $.Release.Name "open-telemetry-ingest" }}
{{- if $.Values.startupProbe.enabled }}
# Startup probe
startupProbe:
httpGet:
path: /status/live
port: {{ $.Values.port.openTelemetryIngest }}
periodSeconds: {{ $.Values.startupProbe.periodSeconds }}
failureThreshold: {{ $.Values.startupProbe.failureThreshold }}
{{- end }}
{{- if $.Values.livenessProbe.enabled }}
# Liveness probe
livenessProbe:
httpGet:
path: /status/live
port: {{ $.Values.port.openTelemetryIngest }}
periodSeconds: {{ $.Values.livenessProbe.periodSeconds }}
timeoutSeconds: {{ $.Values.livenessProbe.timeoutSeconds }}
initialDelaySeconds: {{ $.Values.livenessProbe.initialDelaySeconds }}
{{- end }}
{{- if $.Values.readinessProbe.enabled }}
# Readyness Probe
readinessProbe:
httpGet:
path: /status/ready
port: {{ $.Values.port.openTelemetryIngest }}
periodSeconds: {{ $.Values.readinessProbe.periodSeconds }}
initialDelaySeconds: {{ $.Values.readinessProbe.initialDelaySeconds }}
timeoutSeconds: {{ $.Values.readinessProbe.timeoutSeconds }}
{{- end }}
{{- if $.Values.containerSecurityContext }}
securityContext: {{- $.Values.containerSecurityContext | toYaml | nindent 12 }}
{{- end }}
imagePullPolicy: {{ $.Values.image.pullPolicy }}
env:
{{- include "oneuptime.env.common" . | nindent 12 }}
{{- include "oneuptime.env.commonServer" . | nindent 12 }}
{{- include "oneuptime.env.oneuptimeSecret" . | nindent 12 }}
- name: OPENTELEMETRY_EXPORTER_OTLP_HEADERS
value: {{ $.Values.openTelemetryExporter.headers }}
- name: PORT
value: {{ $.Values.port.openTelemetryIngest | quote }}
ports:
- containerPort: {{ $.Values.port.openTelemetryIngest }}
protocol: TCP
name: http
restartPolicy: {{ $.Values.image.restartPolicy }}
---
# OneUptime open-telemetry-ingest Service
{{- $open-telemetry-ingestPorts := dict "port" $.Values.port.openTelemetryIngest -}}
{{- $open-telemetry-ingestServiceArgs := dict "ServiceName" "open-telemetry-ingest" "Ports" $open-telemetry-ingestPorts "Release" $.Release "Values" $.Values -}}
{{- include "oneuptime.service" $open-telemetry-ingestServiceArgs }}
---
# OneUptime open-telemetry-ingest autoscaler
{{- $open-telemetry-ingestAutoScalerArgs := dict "ServiceName" "open-telemetry-ingest" "Release" $.Release "Values" $.Values -}}
{{- include "oneuptime.autoscaler" $open-telemetry-ingestAutoScalerArgs }}
---

1
HelmChart/Public/oneuptime/values.yaml
View File

@@ -204,6 +204,7 @@ probes:
port:
app: 3002
ingestor: 3400
openTelemetryIngest: 3403
fluentIngest: 3401
incomingRequestIngest: 3402
testServer: 3800

4
Nginx/default.conf.template
View File

@@ -10,6 +10,10 @@ upstream ingestor {
server ${SERVER_INGESTOR_HOSTNAME}:${INGESTOR_PORT} weight=10 max_fails=3 fail_timeout=30s;
}
upstream open-telemetry-ingest {
server ${SERVER_OPEN_TELEMETRY_INGEST_HOSTNAME}:${OPEN_TELEMETRY_INGEST_PORT} weight=10 max_fails=3 fail_timeout=30s;
}
upstream incoming-reuqets-ingest {
server ${SERVER_INCOMING_REQUEST_INGEST_HOSTNAME}:${INCOMING_REQUEST_INGEST_PORT} weight=10 max_fails=3 fail_timeout=30s;
}

2
OTelCollector/config.yaml
View File

@@ -24,7 +24,7 @@ receivers:
exporters:
otlphttp:
endpoint: "http://${env:SERVER_INGESTOR_HOSTNAME}:${env:INGESTOR_PORT}/otlp"
endpoint: "http://${env:SERVER_OPEN_TELEMETRY_INGEST_HOSTNAME}:${env:OPEN_TELEMETRY_INGEST_PORT}/otlp"
headers: {"Content-Type": "application/json"}
auth:
authenticator: headers_setter

4
OpenTelemetryIngest/API/OTelIngest.ts
View File

@@ -1,11 +1,11 @@
import ArrayUtil from "Common/Utils/Array";
import TelemetryIngest, {
TelemetryRequest,
} from "../Middleware/TelemetryIngest";
} from "Common/Server/Middleware/TelemetryIngest";
import OTelIngestService, {
OtelAggregationTemporality,
TelemetryServiceDataIngested,
} from "../Service/OTelIngest";
} from "Common/Server/Services/OpenTelemetryIngestService";
import OneUptimeDate from "Common/Types/Date";
import BadRequestException from "Common/Types/Exception/BadRequestException";
import { JSONArray, JSONObject } from "Common/Types/JSON";

10
OpenTelemetryIngest/Dockerfile.tpl
View File

@@ -1,5 +1,5 @@
#
# OneUptime-Ingestor Dockerfile
# OneUptime-OpenTelemetryIngest Dockerfile
#
# Pull base image nodejs image.
@@ -62,19 +62,19 @@ WORKDIR /usr/src/app
RUN npx playwright install --with-deps
# Install app dependencies
COPY ./Ingestor/package*.json /usr/src/app/
COPY ./OpenTelemetryIngest/package*.json /usr/src/app/
RUN npm install
# Expose ports.
# - 3400: OneUptime-ingestor
EXPOSE 3400
# - 3403: OneUptime-OpenTelemetryIngest
EXPOSE 3403
{{ if eq .Env.ENVIRONMENT "development" }}
#Run the app
CMD [ "npm", "run", "dev" ]
{{ else }}
# Copy app source
COPY ./Ingestor /usr/src/app
COPY ./OpenTelemetryIngest /usr/src/app
# Bundle app source
RUN npm run compile
#Run the app

12
OpenTelemetryIngest/Index.ts
View File

@@ -1,9 +1,4 @@
import IncomingRequestAPI from "../IncomingRequestIngest/API/IncomingRequest";
import MonitorAPI from "./API/Monitor";
import OTelIngestAPI from "./API/OTelIngest";
import Ingestor from "./API/Probe";
import RegisterAPI from "./API/Register";
import ServerMonitorAPI from "./API/ServerMonitor";
import { PromiseVoidFunction } from "Common/Types/FunctionTypes";
import { ClickhouseAppInstance } from "Common/Server/Infrastructure/ClickhouseDatabase";
import PostgresAppInstance from "Common/Server/Infrastructure/PostgresDatabase";
@@ -18,14 +13,9 @@ import "ejs";
const app: ExpressApplication = Express.getExpressApp();
const APP_NAME: string = "ingestor";
const APP_NAME: string = "open-telemetry-ingest";
app.use([`/${APP_NAME}`, "/"], RegisterAPI);
app.use([`/${APP_NAME}`, "/"], MonitorAPI);
app.use([`/${APP_NAME}`, "/"], Ingestor);
app.use([`/${APP_NAME}`, "/"], IncomingRequestAPI);
app.use([`/${APP_NAME}`, "/"], OTelIngestAPI);
app.use([`/${APP_NAME}`, "/"], ServerMonitorAPI);
const init: PromiseVoidFunction = async (): Promise<void> => {
try {

2
Tests/Scripts/status-check.sh
View File

@@ -51,6 +51,8 @@ bash $scriptDir/endpoint-status.sh "Admin Dashboard (Ready Check)" $HOST_TO_CHEC
bash $scriptDir/endpoint-status.sh "Ingestor (Ready Check)" $HOST_TO_CHECK/ingestor/status/ready
bash $scriptDir/endpoint-status.sh "OpenTelemetry Ingest (Ready Check)" $HOST_TO_CHECK/open-telemetry-ingest/status/ready
bash $scriptDir/endpoint-status.sh "Ingestor (Status Check)" $HOST_TO_CHECK/ingestor/status
echo "🚀 OneUptime is up! 🚀"

4
config.example.env
View File

@@ -95,10 +95,12 @@ REDIS_TLS_SENTINEL_MODE=false
INGESTOR_HOSTNAME=ingestor:3400
FLUENT_INGEST_HOSTNAME=fluent-ingest:3401
INCOMING_REQUEST_INGEST_HOSTNAME=incoming-request-ingest:3402
OPEN_TELEMETRY_INGEST_HOSTNAME=otel-telemetry-ingest:3403
SERVER_ACCOUNTS_HOSTNAME=accounts
SERVER_APP_HOSTNAME=app
SERVER_INGESTOR_HOSTNAME=ingestor
SERVER_OPEN_TELEMETRY_INGEST_HOSTNAME=otel-telemetry-ingest
SERVER_INCOMING_REQUEST_INGEST_HOSTNAME=incoming-request-ingest
SERVER_FLUENT_INGEST_HOSTNAME=fluent-ingest
SERVER_TEST_SERVER_HOSTNAME=test-server
@@ -114,6 +116,7 @@ SERVER_DOCS_HOSTNAME=docs
APP_PORT=3002
INGESTOR_PORT=3400
OPEN_TELEMETRY_INGEST_PORT=3403
FLUENT_INGEST_PORT=3401
INCOMING_REQUEST_INGEST_PORT=3402
PROBE_PORT=3500
@@ -269,6 +272,7 @@ LLM_SERVER_HUGGINGFACE_MODEL_NAME=
DISABLE_TELEMETRY_FOR_ACCOUNTS=true
DISABLE_TELEMETRY_FOR_APP=true
DISABLE_TELEMETRY_FOR_INGESTOR=true
DISABLE_TELEMETRY_FOR_OPEN_TELEMETRY_INGEST=true
DISABLE_TELEMETRY_FOR_FLUENT_INGEST=true
DISABLE_TELEMETRY_FOR_INCOMING_REQUEST_INGEST=true
DISABLE_TELEMETRY_FOR_TEST_SERVER=true

15
docker-compose.base.yml
View File

@@ -28,6 +28,7 @@ x-common-variables: &common-variables
SERVER_APP_HOSTNAME: app
SERVER_ALERT_HOSTNAME: alert
SERVER_INGESTOR_HOSTNAME: ingestor
SERVER_OPEN_TELEMETRY_INGEST_HOSTNAME: open-telemetry-ingest
SERVER_INCOMING_REQUEST_INGEST_HOSTNAME: incoming-request-ingest
SERVER_FLUENT_INGEST_HOSTNAME: fluent-ingest
SERVER_TEST_SERVER_HOSTNAME: test-server
@@ -46,6 +47,7 @@ x-common-variables: &common-variables
APP_PORT: ${APP_PORT}
HOME_PORT: ${HOME_PORT}
INGESTOR_PORT: ${INGESTOR_PORT}
OPEN_TELEMETRY_INGEST_PORT: ${OPEN_TELEMETRY_INGEST_PORT}
INCOMING_REQUEST_INGEST_PORT: ${INCOMING_REQUEST_INGEST_PORT}
FLUENT_INGEST_PORT: ${FLUENT_INGEST_PORT}
PROBE_PORT: ${PROBE_PORT}
@@ -450,6 +452,19 @@ services:
options:
max-size: "1000m"
open-telemetry-ingest:
networks:
- oneuptime
restart: always
environment:
<<: *common-server-variables
PORT: ${OPEN_TELEMETRY_INGEST_PORT}
DISABLE_TELEMETRY: ${DISABLE_TELEMETRY_FOR_OPEN_TELEMETRY_INGEST}
logging:
driver: "local"
options:
max-size: "1000m"
incoming-request-ingest:
networks:
- oneuptime

18
docker-compose.dev.yml
View File

@@ -351,6 +351,24 @@ services:
context: .
dockerfile: ./Ingestor/Dockerfile
open-telemetry-ingest:
volumes:
- ./OpenTelemetryIngest:/usr/src/app
# Use node modules of the container and not host system.
# https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder
- /usr/src/app/node_modules/
- ./Common:/usr/src/Common
- /usr/src/Common/node_modules/
ports:
- '9938:9229' # Debugging port.
extends:
file: ./docker-compose.base.yml
service: open-telemetry-ingest
build:
network: host
context: .
dockerfile: ./OpenTelemetryIngest/Dockerfile
incoming-request-ingest:
volumes:
- ./IncomingRequestIngest:/usr/src/app

6
docker-compose.yml
View File

@@ -112,6 +112,12 @@ services:
extends:
file: ./docker-compose.base.yml
service: ingestor
open-telemetry-ingest:
image: oneuptime/open-telemetry-ingest:${APP_TAG}
extends:
file: ./docker-compose.base.yml
service: open-telemetry-ingest
incoming-request-ingest:
image: oneuptime/incoming-request-ingest:${APP_TAG}