mirror of
https://github.com/OneUptime/oneuptime.git
synced 2026-04-06 00:32:12 +02:00
feat: Add telemetry API for probe and server monitor ingestion
- Implemented Probe registration endpoint to handle global probes. - Created Server Monitor ingestion API with endpoints for secret key verification and response ingestion. - Developed job processing functions for incoming request and probe ingestion. - Added middleware for probe authorization and request type definitions. - Enhanced monitor utility functions to populate secrets in monitor steps. - Introduced queue stats and failed jobs endpoints for server monitor.
This commit is contained in:
Nawaz Dhandala
69
.github/workflows/build.yml
vendored
69
.github/workflows/build.yml
vendored
@@ -290,52 +290,6 @@ jobs:
|
||||
max_attempts: 3
|
||||
command: sudo docker build --no-cache -f ./Probe/Dockerfile .
|
||||
|
||||
docker-build-probe-ingest:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
CI_PIPELINE_ID: ${{github.run_number}}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Preinstall
|
||||
uses: nick-fields/retry@v3
|
||||
with:
|
||||
timeout_minutes: 10
|
||||
max_attempts: 3
|
||||
command: npm run prerun
|
||||
|
||||
# build image probe api
|
||||
- name: build docker image
|
||||
uses: nick-fields/retry@v3
|
||||
with:
|
||||
timeout_minutes: 45
|
||||
max_attempts: 3
|
||||
command: sudo docker build --no-cache -f ./ProbeIngest/Dockerfile .
|
||||
|
||||
docker-build-server-monitor-ingest:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
CI_PIPELINE_ID: ${{github.run_number}}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Preinstall
|
||||
uses: nick-fields/retry@v3
|
||||
with:
|
||||
timeout_minutes: 10
|
||||
max_attempts: 3
|
||||
command: npm run prerun
|
||||
|
||||
# build image probe api
|
||||
- name: build docker image
|
||||
uses: nick-fields/retry@v3
|
||||
with:
|
||||
timeout_minutes: 45
|
||||
max_attempts: 3
|
||||
command: sudo docker build --no-cache -f ./ServerMonitorIngest/Dockerfile .
|
||||
|
||||
docker-build-telemetry:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
@@ -359,29 +313,6 @@ jobs:
|
||||
max_attempts: 3
|
||||
command: sudo docker build --no-cache -f ./Telemetry/Dockerfile .
|
||||
|
||||
docker-build-incoming-request-ingest:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
CI_PIPELINE_ID: ${{github.run_number}}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Preinstall
|
||||
uses: nick-fields/retry@v3
|
||||
with:
|
||||
timeout_minutes: 10
|
||||
max_attempts: 3
|
||||
command: npm run prerun
|
||||
|
||||
# build image probe api
|
||||
- name: build docker image
|
||||
uses: nick-fields/retry@v3
|
||||
with:
|
||||
timeout_minutes: 45
|
||||
max_attempts: 3
|
||||
command: sudo docker build --no-cache -f ./IncomingRequestIngest/Dockerfile .
|
||||
|
||||
docker-build-status-page:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
|
||||
51
.github/workflows/compile.yml
vendored
51
.github/workflows/compile.yml
vendored
@@ -251,40 +251,6 @@ jobs:
|
||||
max_attempts: 3
|
||||
command: cd Probe && npm install && npm run compile && npm run dep-check
|
||||
|
||||
compile-probe-ingest:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
CI_PIPELINE_ID: ${{github.run_number}}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: latest
|
||||
- run: cd Common && npm install
|
||||
- name: Compile Probe Ingest
|
||||
uses: nick-fields/retry@v3
|
||||
with:
|
||||
timeout_minutes: 30
|
||||
max_attempts: 3
|
||||
command: cd ProbeIngest && npm install && npm run compile && npm run dep-check
|
||||
|
||||
compile-server-monitor-ingest:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
CI_PIPELINE_ID: ${{github.run_number}}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: latest
|
||||
- run: cd Common && npm install
|
||||
- name: Compile Server Monitor Ingest
|
||||
uses: nick-fields/retry@v3
|
||||
with:
|
||||
timeout_minutes: 30
|
||||
max_attempts: 3
|
||||
command: cd ServerMonitorIngest && npm install && npm run compile && npm run dep-check
|
||||
|
||||
compile-telemetry:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
@@ -303,23 +269,6 @@ jobs:
|
||||
command: cd Telemetry && npm install && npm run compile && npm run dep-check
|
||||
|
||||
|
||||
compile-incoming-request-ingest:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
CI_PIPELINE_ID: ${{github.run_number}}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: latest
|
||||
- run: cd Common && npm install
|
||||
- name: Compile Incoming Request Ingest
|
||||
uses: nick-fields/retry@v3
|
||||
with:
|
||||
timeout_minutes: 30
|
||||
max_attempts: 3
|
||||
command: cd IncomingRequestIngest && npm install && npm run compile && npm run dep-check
|
||||
|
||||
compile-status-page:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
|
||||
217
.github/workflows/release.yml
vendored
217
.github/workflows/release.yml
vendored
@@ -764,145 +764,6 @@ jobs:
|
||||
--platforms linux/amd64,linux/arm64 \
|
||||
--git-sha "${{ github.sha }}"
|
||||
|
||||
probe-ingest-docker-image-deploy:
|
||||
needs: [generate-build-number, read-version]
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
QEMU_CPU: max
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: true
|
||||
swap-storage: true
|
||||
- name: Docker Meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v4
|
||||
with:
|
||||
images: |
|
||||
oneuptime/probe-ingest
|
||||
ghcr.io/oneuptime/probe-ingest
|
||||
tags: |
|
||||
type=raw,value=release,enable=true
|
||||
type=semver,value=${{needs.read-version.outputs.major_minor}},pattern={{version}},enable=true
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.ref }}
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: latest
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
with:
|
||||
image: tonistiigi/binfmt:qemu-v10.0.4
|
||||
|
||||
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Generate Dockerfile from Dockerfile.tpl
|
||||
run: npm run prerun
|
||||
|
||||
# Build and deploy probe-ingest.
|
||||
|
||||
- name: Login to Docker Hub
|
||||
run: |
|
||||
echo "${{ secrets.DOCKERHUB_PASSWORD }}" | docker login --username "${{ secrets.DOCKERHUB_USERNAME }}" --password-stdin
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
run: |
|
||||
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username "${{ github.repository_owner }}" --password-stdin
|
||||
|
||||
- name: Build and push
|
||||
run: |
|
||||
bash ./Scripts/GHA/build_docker_images.sh \
|
||||
--image probe-ingest \
|
||||
--version "${{needs.read-version.outputs.major_minor}}" \
|
||||
--dockerfile ./ProbeIngest/Dockerfile \
|
||||
--context . \
|
||||
--platforms linux/amd64,linux/arm64 \
|
||||
--git-sha "${{ github.sha }}"
|
||||
|
||||
|
||||
server-monitor-ingest-docker-image-deploy:
|
||||
needs: [generate-build-number, read-version]
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
QEMU_CPU: max
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: true
|
||||
swap-storage: true
|
||||
- name: Docker Meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v4
|
||||
with:
|
||||
images: |
|
||||
oneuptime/server-monitor-ingest
|
||||
ghcr.io/oneuptime/server-monitor-ingest
|
||||
tags: |
|
||||
type=raw,value=release,enable=true
|
||||
type=semver,value=${{needs.read-version.outputs.major_minor}},pattern={{version}},enable=true
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.ref }}
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: latest
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
with:
|
||||
image: tonistiigi/binfmt:qemu-v10.0.4
|
||||
|
||||
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Generate Dockerfile from Dockerfile.tpl
|
||||
run: npm run prerun
|
||||
|
||||
# Build and deploy probe-ingest.
|
||||
|
||||
- name: Login to Docker Hub
|
||||
run: |
|
||||
echo "${{ secrets.DOCKERHUB_PASSWORD }}" | docker login --username "${{ secrets.DOCKERHUB_USERNAME }}" --password-stdin
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
run: |
|
||||
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username "${{ github.repository_owner }}" --password-stdin
|
||||
|
||||
- name: Build and push
|
||||
run: |
|
||||
bash ./Scripts/GHA/build_docker_images.sh \
|
||||
--image server-monitor-ingest \
|
||||
--version "${{needs.read-version.outputs.major_minor}}" \
|
||||
--dockerfile ./ServerMonitorIngest/Dockerfile \
|
||||
--context . \
|
||||
--platforms linux/amd64,linux/arm64 \
|
||||
--git-sha "${{ github.sha }}"
|
||||
|
||||
|
||||
|
||||
telemetry-docker-image-deploy:
|
||||
needs: [generate-build-number, read-version]
|
||||
runs-on: ubuntu-latest
|
||||
@@ -972,74 +833,6 @@ jobs:
|
||||
--git-sha "${{ github.sha }}"
|
||||
|
||||
|
||||
incoming-request-ingest-docker-image-deploy:
|
||||
needs: [generate-build-number, read-version]
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
QEMU_CPU: max
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: true
|
||||
swap-storage: true
|
||||
- name: Docker Meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v4
|
||||
with:
|
||||
images: |
|
||||
oneuptime/incoming-request-ingest
|
||||
ghcr.io/oneuptime/incoming-request-ingest
|
||||
tags: |
|
||||
type=raw,value=release,enable=true
|
||||
type=semver,value=${{needs.read-version.outputs.major_minor}},pattern={{version}},enable=true
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.ref }}
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: latest
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
with:
|
||||
image: tonistiigi/binfmt:qemu-v10.0.4
|
||||
|
||||
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Generate Dockerfile from Dockerfile.tpl
|
||||
run: npm run prerun
|
||||
|
||||
# Build and deploy incoming-request-ingest.
|
||||
|
||||
- name: Login to Docker Hub
|
||||
run: |
|
||||
echo "${{ secrets.DOCKERHUB_PASSWORD }}" | docker login --username "${{ secrets.DOCKERHUB_USERNAME }}" --password-stdin
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
run: |
|
||||
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username "${{ github.repository_owner }}" --password-stdin
|
||||
|
||||
- name: Build and push
|
||||
run: |
|
||||
bash ./Scripts/GHA/build_docker_images.sh \
|
||||
--image incoming-request-ingest \
|
||||
--version "${{needs.read-version.outputs.major_minor}}" \
|
||||
--dockerfile ./IncomingRequestIngest/Dockerfile \
|
||||
--context . \
|
||||
--platforms linux/amd64,linux/arm64 \
|
||||
--git-sha "${{ github.sha }}"
|
||||
|
||||
probe-docker-image-deploy:
|
||||
needs: [generate-build-number, read-version]
|
||||
runs-on: ubuntu-latest
|
||||
@@ -1762,10 +1555,7 @@ jobs:
|
||||
- otel-collector-docker-image-deploy
|
||||
- status-page-docker-image-deploy
|
||||
- test-docker-image-deploy
|
||||
- probe-ingest-docker-image-deploy
|
||||
- server-monitor-ingest-docker-image-deploy
|
||||
- telemetry-docker-image-deploy
|
||||
- incoming-request-ingest-docker-image-deploy
|
||||
- probe-docker-image-deploy
|
||||
- admin-dashboard-docker-image-deploy
|
||||
- dashboard-docker-image-deploy
|
||||
@@ -1791,10 +1581,7 @@ jobs:
|
||||
"otel-collector",
|
||||
"status-page",
|
||||
"test",
|
||||
"probe-ingest",
|
||||
"server-monitor-ingest",
|
||||
"telemetry",
|
||||
"incoming-request-ingest",
|
||||
"probe",
|
||||
"admin-dashboard",
|
||||
"dashboard",
|
||||
@@ -1850,7 +1637,7 @@ jobs:
|
||||
|
||||
test-e2e-release-saas:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [telemetry-docker-image-deploy, mcp-docker-image-deploy, docs-docker-image-deploy, workflow-docker-image-deploy, accounts-docker-image-deploy, ai-agent-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, probe-ingest-docker-image-deploy, server-monitor-ingest-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, read-version, nginx-docker-image-deploy, incoming-request-ingest-docker-image-deploy]
|
||||
needs: [telemetry-docker-image-deploy, mcp-docker-image-deploy, docs-docker-image-deploy, workflow-docker-image-deploy, accounts-docker-image-deploy, ai-agent-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, read-version, nginx-docker-image-deploy]
|
||||
env:
|
||||
CI_PIPELINE_ID: ${{github.run_number}}
|
||||
steps:
|
||||
@@ -1981,7 +1768,7 @@ jobs:
|
||||
test-e2e-release-self-hosted:
|
||||
runs-on: ubuntu-latest
|
||||
# After all the jobs runs
|
||||
needs: [telemetry-docker-image-deploy, mcp-docker-image-deploy, incoming-request-ingest-docker-image-deploy, docs-docker-image-deploy, workflow-docker-image-deploy, accounts-docker-image-deploy, ai-agent-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, probe-ingest-docker-image-deploy, server-monitor-ingest-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, read-version, nginx-docker-image-deploy]
|
||||
needs: [telemetry-docker-image-deploy, mcp-docker-image-deploy, docs-docker-image-deploy, workflow-docker-image-deploy, accounts-docker-image-deploy, ai-agent-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, read-version, nginx-docker-image-deploy]
|
||||
env:
|
||||
CI_PIPELINE_ID: ${{github.run_number}}
|
||||
steps:
|
||||
|
||||
216
.github/workflows/test-release.yaml
vendored
216
.github/workflows/test-release.yaml
vendored
@@ -721,218 +721,6 @@ jobs:
|
||||
--extra-tags test \
|
||||
--extra-enterprise-tags enterprise-test
|
||||
|
||||
probe-ingest-docker-image-deploy:
|
||||
needs: [read-version, generate-build-number]
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
QEMU_CPU: max
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: true
|
||||
swap-storage: true
|
||||
- name: Docker Meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v4
|
||||
with:
|
||||
images: |
|
||||
oneuptime/probe-ingest
|
||||
ghcr.io/oneuptime/probe-ingest
|
||||
tags: |
|
||||
type=raw,value=test,enable=true
|
||||
type=raw,value=${{needs.read-version.outputs.major_minor}}-test,enable=true
|
||||
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.ref }}
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: latest
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
with:
|
||||
image: tonistiigi/binfmt:qemu-v10.0.4
|
||||
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Generate Dockerfile from Dockerfile.tpl
|
||||
run: npm run prerun
|
||||
|
||||
# Build and deploy probe-ingest.
|
||||
|
||||
- name: Login to Docker Hub
|
||||
run: |
|
||||
echo "${{ secrets.DOCKERHUB_PASSWORD }}" | docker login --username "${{ secrets.DOCKERHUB_USERNAME }}" --password-stdin
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
run: |
|
||||
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username "${{ github.repository_owner }}" --password-stdin
|
||||
|
||||
- name: Build and push
|
||||
run: |
|
||||
bash ./Scripts/GHA/build_docker_images.sh \
|
||||
--image probe-ingest \
|
||||
--version "${{needs.read-version.outputs.major_minor}}-test" \
|
||||
--dockerfile ./ProbeIngest/Dockerfile \
|
||||
--context . \
|
||||
--platforms linux/amd64,linux/arm64 \
|
||||
--git-sha "${{ github.sha }}" \
|
||||
--extra-tags test \
|
||||
--extra-enterprise-tags enterprise-test
|
||||
|
||||
|
||||
|
||||
server-monitor-ingest-docker-image-deploy:
|
||||
needs: [read-version, generate-build-number]
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
QEMU_CPU: max
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: true
|
||||
swap-storage: true
|
||||
- name: Docker Meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v4
|
||||
with:
|
||||
images: |
|
||||
oneuptime/server-monitor-ingest
|
||||
ghcr.io/oneuptime/server-monitor-ingest
|
||||
tags: |
|
||||
type=raw,value=test,enable=true
|
||||
type=raw,value=${{needs.read-version.outputs.major_minor}}-test,enable=true
|
||||
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.ref }}
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: latest
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
with:
|
||||
image: tonistiigi/binfmt:qemu-v10.0.4
|
||||
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Generate Dockerfile from Dockerfile.tpl
|
||||
run: npm run prerun
|
||||
|
||||
# Build and deploy ServerMonitorIngest.
|
||||
|
||||
- name: Login to Docker Hub
|
||||
run: |
|
||||
echo "${{ secrets.DOCKERHUB_PASSWORD }}" | docker login --username "${{ secrets.DOCKERHUB_USERNAME }}" --password-stdin
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
run: |
|
||||
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username "${{ github.repository_owner }}" --password-stdin
|
||||
|
||||
- name: Build and push
|
||||
run: |
|
||||
bash ./Scripts/GHA/build_docker_images.sh \
|
||||
--image server-monitor-ingest \
|
||||
--version "${{needs.read-version.outputs.major_minor}}-test" \
|
||||
--dockerfile ./ServerMonitorIngest/Dockerfile \
|
||||
--context . \
|
||||
--platforms linux/amd64,linux/arm64 \
|
||||
--git-sha "${{ github.sha }}" \
|
||||
--extra-tags test \
|
||||
--extra-enterprise-tags enterprise-test
|
||||
|
||||
|
||||
incoming-request-ingest-docker-image-deploy:
|
||||
needs: [read-version, generate-build-number]
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
QEMU_CPU: max
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: true
|
||||
swap-storage: true
|
||||
- name: Docker Meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v4
|
||||
with:
|
||||
images: |
|
||||
oneuptime/incoming-request-ingest
|
||||
ghcr.io/oneuptime/incoming-request-ingest
|
||||
tags: |
|
||||
type=raw,value=test,enable=true
|
||||
type=raw,value=${{needs.read-version.outputs.major_minor}}-test,enable=true
|
||||
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.ref }}
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: latest
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
with:
|
||||
image: tonistiigi/binfmt:qemu-v10.0.4
|
||||
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Generate Dockerfile from Dockerfile.tpl
|
||||
run: npm run prerun
|
||||
|
||||
# Build and deploy incoming-request-ingest.
|
||||
|
||||
- name: Login to Docker Hub
|
||||
run: |
|
||||
echo "${{ secrets.DOCKERHUB_PASSWORD }}" | docker login --username "${{ secrets.DOCKERHUB_USERNAME }}" --password-stdin
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
run: |
|
||||
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username "${{ github.repository_owner }}" --password-stdin
|
||||
|
||||
- name: Build and push
|
||||
run: |
|
||||
bash ./Scripts/GHA/build_docker_images.sh \
|
||||
--image incoming-request-ingest \
|
||||
--version "${{needs.read-version.outputs.major_minor}}-test" \
|
||||
--dockerfile ./IncomingRequestIngest/Dockerfile \
|
||||
--context . \
|
||||
--platforms linux/amd64,linux/arm64 \
|
||||
--git-sha "${{ github.sha }}" \
|
||||
--extra-tags test \
|
||||
--extra-enterprise-tags enterprise-test
|
||||
|
||||
telemetry-docker-image-deploy:
|
||||
needs: [read-version, generate-build-number]
|
||||
@@ -982,7 +770,7 @@ jobs:
|
||||
- name: Generate Dockerfile from Dockerfile.tpl
|
||||
run: npm run prerun
|
||||
|
||||
# Build and deploy incoming-request-ingest.
|
||||
# Build and deploy telemetry.
|
||||
|
||||
- name: Login to Docker Hub
|
||||
run: |
|
||||
@@ -1654,7 +1442,7 @@ jobs:
|
||||
|
||||
test-helm-chart:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [infrastructure-agent-deploy, mcp-docker-image-deploy, publish-terraform-provider, telemetry-docker-image-deploy, docs-docker-image-deploy, worker-docker-image-deploy, workflow-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, test-server-docker-image-deploy, test-docker-image-deploy, probe-ingest-docker-image-deploy, server-monitor-ingest-docker-image-deploy, probe-docker-image-deploy, dashboard-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, accounts-docker-image-deploy, ai-agent-docker-image-deploy, otel-collector-docker-image-deploy, status-page-docker-image-deploy, nginx-docker-image-deploy, e2e-docker-image-deploy, incoming-request-ingest-docker-image-deploy]
|
||||
needs: [infrastructure-agent-deploy, mcp-docker-image-deploy, publish-terraform-provider, telemetry-docker-image-deploy, docs-docker-image-deploy, worker-docker-image-deploy, workflow-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, test-server-docker-image-deploy, test-docker-image-deploy, probe-docker-image-deploy, dashboard-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, accounts-docker-image-deploy, ai-agent-docker-image-deploy, otel-collector-docker-image-deploy, status-page-docker-image-deploy, nginx-docker-image-deploy, e2e-docker-image-deploy]
|
||||
env:
|
||||
CI_PIPELINE_ID: ${{github.run_number}}
|
||||
steps:
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
name: Incoming Request Ingest Test
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches-ignore:
|
||||
- 'hotfix-*' # excludes hotfix branches
|
||||
- 'release'
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
CI_PIPELINE_ID: ${{github.run_number}}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: latest
|
||||
- run: cd IncomingRequestIngest && npm install && npm run test
|
||||
|
||||
@@ -175,24 +175,12 @@ export const AppApiHostname: Hostname = Hostname.fromString(
|
||||
}`,
|
||||
);
|
||||
|
||||
export const ProbeIngestHostname: Hostname = Hostname.fromString(
|
||||
`${process.env["SERVER_PROBE_INGEST_HOSTNAME"] || "localhost"}:${
|
||||
process.env["PROBE_INGEST_PORT"] || 80
|
||||
}`,
|
||||
);
|
||||
|
||||
export const OpenTelemetryIngestHostname: Hostname = Hostname.fromString(
|
||||
`${process.env["SERVER_TELEMETRY_HOSTNAME"] || "localhost"}:${
|
||||
process.env["TELEMETRY_PORT"] || 80
|
||||
}`,
|
||||
);
|
||||
|
||||
export const IncomingRequestIngestHostname: Hostname = Hostname.fromString(
|
||||
`${process.env["SERVER_INCOMING_REQUEST_INGEST_HOSTNAME"] || "localhost"}:${
|
||||
process.env["INCOMING_REQUEST_INGEST_PORT"] || 80
|
||||
}`,
|
||||
);
|
||||
|
||||
export const IsolatedVMHostname: Hostname = Hostname.fromString(
|
||||
`${process.env["SERVER_ISOLATED_VM_HOSTNAME"] || "localhost"}:${
|
||||
process.env["ISOLATED_VM_PORT"] || 80
|
||||
|
||||
@@ -14,9 +14,6 @@ export enum QueueName {
|
||||
Workflow = "Workflow",
|
||||
Worker = "Worker",
|
||||
Telemetry = "Telemetry",
|
||||
IncomingRequestIngest = "IncomingRequestIngest",
|
||||
ServerMonitorIngest = "ServerMonitorIngest",
|
||||
ProbeIngest = "ProbeIngest",
|
||||
}
|
||||
|
||||
export type QueueJob = Job;
|
||||
|
||||
@@ -30,10 +30,6 @@ export const ProbeIngestRoute: Route = new Route("/probe-ingest");
|
||||
|
||||
export const TelemetryRoute: Route = new Route("/telemetry");
|
||||
|
||||
export const IncomingRequestIngestRoute: Route = new Route(
|
||||
"/incoming-request-ingest",
|
||||
);
|
||||
|
||||
export const RealtimeRoute: Route = new Route("/realtime/socket");
|
||||
|
||||
export const DocsRoute: Route = new Route("/docs");
|
||||
|
||||
@@ -15,7 +15,6 @@ import {
|
||||
StatusPageApiRoute,
|
||||
StatusPageRoute,
|
||||
WorkflowRoute,
|
||||
IncomingRequestIngestRoute,
|
||||
TelemetryRoute,
|
||||
} from "../ServiceRoute";
|
||||
import Hostname from "../Types/API/Hostname";
|
||||
@@ -76,13 +75,8 @@ export const STATUS_PAGE_HOSTNAME: Hostname = Hostname.fromString(HOST);
|
||||
|
||||
export const WORKFLOW_HOSTNAME: Hostname = Hostname.fromString(HOST);
|
||||
|
||||
export const PROBE_INGEST_HOSTNAME: Hostname = Hostname.fromString(HOST);
|
||||
|
||||
export const TELEMETRY_HOSTNAME: Hostname = Hostname.fromString(HOST);
|
||||
|
||||
export const INCOMING_REQUEST_INGEST_HOSTNAME: Hostname =
|
||||
Hostname.fromString(HOST);
|
||||
|
||||
export const HELM_HOSTNAME: Hostname = Hostname.fromString(HOST);
|
||||
|
||||
export const API_DOCS_HOSTNAME: Hostname = Hostname.fromString(HOST);
|
||||
@@ -143,16 +137,10 @@ export const WORKFLOW_URL: URL = new URL(
|
||||
|
||||
export const PROBE_INGEST_URL: URL = new URL(
|
||||
HTTP_PROTOCOL,
|
||||
PROBE_INGEST_HOSTNAME,
|
||||
TELEMETRY_HOSTNAME,
|
||||
new Route(ProbeIngestRoute.toString()),
|
||||
);
|
||||
|
||||
export const INCOMING_REQUEST_INGEST_URL: URL = new URL(
|
||||
HTTP_PROTOCOL,
|
||||
INCOMING_REQUEST_INGEST_HOSTNAME,
|
||||
new Route(IncomingRequestIngestRoute.toString()),
|
||||
);
|
||||
|
||||
export const STATUS_PAGE_URL: URL = new URL(
|
||||
HTTP_PROTOCOL,
|
||||
STATUS_PAGE_HOSTNAME,
|
||||
|
||||
@@ -105,8 +105,6 @@ Usage:
|
||||
value: {{ default "" $.Values.inboundEmail.webhookSecret | quote }}
|
||||
- name: SERVER_ACCOUNTS_HOSTNAME
|
||||
value: {{ $.Release.Name }}-accounts.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }}
|
||||
- name: SERVER_SERVER_MONITOR_INGEST_HOSTNAME
|
||||
value: {{ $.Release.Name }}-server-monitor-ingest.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }}
|
||||
- name: SERVER_ISOLATED_VM_HOSTNAME
|
||||
value: {{ $.Release.Name }}-isolated-vm.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }}
|
||||
- name: SERVER_WORKFLOW_HOSTNAME
|
||||
@@ -117,12 +115,8 @@ Usage:
|
||||
value: {{ $.Release.Name }}-home.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }}
|
||||
- name: SERVER_APP_HOSTNAME
|
||||
value: {{ $.Release.Name }}-app.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }}
|
||||
- name: SERVER_PROBE_INGEST_HOSTNAME
|
||||
value: {{ $.Release.Name }}-probe-ingest.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }}
|
||||
- name: TELEMETRY_HOSTNAME
|
||||
value: {{ $.Release.Name }}-telemetry.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }}
|
||||
- name: SERVER_INCOMING_REQUEST_INGEST_HOSTNAME
|
||||
value: {{ $.Release.Name }}-incoming-request-ingest.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }}
|
||||
- name: SERVER_TELEMETRY_HOSTNAME
|
||||
value: {{ $.Release.Name }}-telemetry.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }}
|
||||
- name: SERVER_TEST_SERVER_HOSTNAME
|
||||
@@ -142,14 +136,8 @@ Usage:
|
||||
|
||||
- name: APP_PORT
|
||||
value: {{ $.Values.app.ports.http | squote }}
|
||||
- name: PROBE_INGEST_PORT
|
||||
value: {{ $.Values.probeIngest.ports.http | squote }}
|
||||
- name: SERVER_MONITOR_INGEST_PORT
|
||||
value: {{ $.Values.serverMonitorIngest.ports.http | squote }}
|
||||
- name: TELEMETRY_PORT
|
||||
value: {{ $.Values.telemetry.ports.http | squote }}
|
||||
- name: INCOMING_REQUEST_INGEST_PORT
|
||||
value: {{ $.Values.incomingRequestIngest.ports.http | squote }}
|
||||
- name: TEST_SERVER_PORT
|
||||
value: {{ $.Values.testServer.ports.http | squote }}
|
||||
- name: ACCOUNTS_PORT
|
||||
|
||||
@@ -1,143 +0,0 @@
|
||||
{{- if $.Values.incomingRequestIngest.enabled }}
|
||||
# OneUptime incoming-request-ingest Deployment
|
||||
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ printf "%s-%s" $.Release.Name "incoming-request-ingest" }}
|
||||
namespace: {{ $.Release.Namespace }}
|
||||
labels:
|
||||
app: {{ printf "%s-%s" $.Release.Name "incoming-request-ingest" }}
|
||||
app.kubernetes.io/part-of: oneuptime
|
||||
app.kubernetes.io/managed-by: Helm
|
||||
appname: oneuptime
|
||||
{{- if $.Values.deployment.includeTimestampLabel }}
|
||||
date: "{{ now | unixEpoch }}"
|
||||
{{- end }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ printf "%s-%s" $.Release.Name "incoming-request-ingest" }}
|
||||
{{- if $.Values.incomingRequestIngest.replicaCount }}
|
||||
replicas: {{ $.Values.incomingRequestIngest.replicaCount }}
|
||||
{{- else }}
|
||||
{{- if or (not $.Values.autoscaling.enabled) ($.Values.incomingRequestIngest.disableAutoscaler) }}
|
||||
replicas: {{ $.Values.deployment.replicaCount }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
strategy: {{- toYaml $.Values.deployment.updateStrategy | nindent 4 }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: {{ printf "%s-%s" $.Release.Name "incoming-request-ingest" }}
|
||||
{{- if $.Values.deployment.includeTimestampLabel }}
|
||||
date: "{{ now | unixEpoch }}"
|
||||
{{- end }}
|
||||
appname: oneuptime
|
||||
spec:
|
||||
volumes:
|
||||
- name: greenlockrc
|
||||
emptyDir:
|
||||
sizeLimit: "1Gi"
|
||||
{{- if $.Values.podSecurityContext }}
|
||||
securityContext: {{- $.Values.podSecurityContext | toYaml | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if $.Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml $.Values.imagePullSecrets | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if $.Values.affinity }}
|
||||
affinity: {{- $.Values.affinity | toYaml | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if $.Values.tolerations }}
|
||||
tolerations: {{- $.Values.tolerations | toYaml | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if $.Values.incomingRequestIngest.podSecurityContext }}
|
||||
securityContext:
|
||||
{{- toYaml $.Values.incomingRequestIngest.podSecurityContext | nindent 8 }}
|
||||
{{- else if $.Values.podSecurityContext }}
|
||||
securityContext:
|
||||
{{- toYaml $.Values.podSecurityContext | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if $.Values.incomingRequestIngest.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml $.Values.incomingRequestIngest.nodeSelector | nindent 8 }}
|
||||
{{- else if $.Values.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml $.Values.nodeSelector | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- image: {{ include "oneuptime.image" (dict "Values" $.Values "ServiceName" "incoming-request-ingest") }}
|
||||
name: {{ printf "%s-%s" $.Release.Name "incoming-request-ingest" }}
|
||||
{{- if $.Values.startupProbe.enabled }}
|
||||
# Startup probe
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /status/live
|
||||
port: {{ $.Values.incomingRequestIngest.ports.http }}
|
||||
periodSeconds: {{ $.Values.startupProbe.periodSeconds }}
|
||||
failureThreshold: {{ $.Values.startupProbe.failureThreshold }}
|
||||
{{- end }}
|
||||
{{- if $.Values.livenessProbe.enabled }}
|
||||
# Liveness probe
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /status/live
|
||||
port: {{ $.Values.incomingRequestIngest.ports.http }}
|
||||
periodSeconds: {{ $.Values.livenessProbe.periodSeconds }}
|
||||
timeoutSeconds: {{ $.Values.livenessProbe.timeoutSeconds }}
|
||||
initialDelaySeconds: {{ $.Values.livenessProbe.initialDelaySeconds }}
|
||||
{{- end }}
|
||||
{{- if $.Values.readinessProbe.enabled }}
|
||||
# Readyness Probe
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /status/ready
|
||||
port: {{ $.Values.incomingRequestIngest.ports.http }}
|
||||
periodSeconds: {{ $.Values.readinessProbe.periodSeconds }}
|
||||
initialDelaySeconds: {{ $.Values.readinessProbe.initialDelaySeconds }}
|
||||
timeoutSeconds: {{ $.Values.readinessProbe.timeoutSeconds }}
|
||||
{{- end }}
|
||||
{{- if $.Values.incomingRequestIngest.containerSecurityContext }}
|
||||
securityContext:
|
||||
{{- toYaml $.Values.incomingRequestIngest.containerSecurityContext | nindent 12 }}
|
||||
{{- else if $.Values.containerSecurityContext }}
|
||||
securityContext:
|
||||
{{- toYaml $.Values.containerSecurityContext | nindent 12 }}
|
||||
{{- end }}
|
||||
imagePullPolicy: {{ $.Values.image.pullPolicy }}
|
||||
env:
|
||||
{{- include "oneuptime.env.common" . | nindent 12 }}
|
||||
{{- include "oneuptime.env.runtime" (dict "Values" $.Values "Release" $.Release) | nindent 12 }}
|
||||
- name: PORT
|
||||
value: {{ $.Values.incomingRequestIngest.ports.http | quote }}
|
||||
- name: DISABLE_TELEMETRY
|
||||
value: {{ $.Values.incomingRequestIngest.disableTelemetryCollection | quote }}
|
||||
- name: INCOMING_REQUEST_INGEST_CONCURRENCY
|
||||
value: {{ $.Values.incomingRequestIngest.concurrency | squote }}
|
||||
ports:
|
||||
- containerPort: {{ $.Values.incomingRequestIngest.ports.http }}
|
||||
protocol: TCP
|
||||
name: http
|
||||
{{- if $.Values.incomingRequestIngest.resources }}
|
||||
resources:
|
||||
{{- toYaml $.Values.incomingRequestIngest.resources | nindent 12 }}
|
||||
{{- end }}
|
||||
restartPolicy: {{ $.Values.image.restartPolicy }}
|
||||
|
||||
---
|
||||
|
||||
# OneUptime incoming-request-ingest autoscaler
|
||||
{{- if and (not $.Values.incomingRequestIngest.disableAutoscaler) (not (and $.Values.keda.enabled $.Values.incomingRequestIngest.keda.enabled)) }}
|
||||
{{- $incomingRequestIngestAutoScalerArgs := dict "ServiceName" "incoming-request-ingest" "Release" $.Release "Values" $.Values -}}
|
||||
{{- include "oneuptime.autoscaler" $incomingRequestIngestAutoScalerArgs }}
|
||||
{{- end }}
|
||||
---
|
||||
|
||||
{{- end }}
|
||||
|
||||
# OneUptime incoming-request-ingest Service
|
||||
{{- $incomingRequestIngestPorts := dict "port" $.Values.incomingRequestIngest.ports.http -}}
|
||||
{{- $incomingRequestIngestServiceArgs := dict "ServiceName" "incoming-request-ingest" "Ports" $incomingRequestIngestPorts "Release" $.Release "Values" $.Values -}}
|
||||
{{- include "oneuptime.service" $incomingRequestIngestServiceArgs }}
|
||||
---
|
||||
@@ -9,27 +9,6 @@ KEDA ScaledObjects for various services
|
||||
{{- include "oneuptime.kedaScaledObject" $telemetryKedaArgs }}
|
||||
{{- end }}
|
||||
|
||||
{{/* Incoming Request Ingest KEDA ScaledObject */}}
|
||||
{{- if and .Values.keda.enabled .Values.incomingRequestIngest.enabled .Values.incomingRequestIngest.keda.enabled (not .Values.incomingRequestIngest.disableAutoscaler) }}
|
||||
{{- $metricsConfig := dict "enabled" .Values.incomingRequestIngest.keda.enabled "minReplicas" .Values.incomingRequestIngest.keda.minReplicas "maxReplicas" .Values.incomingRequestIngest.keda.maxReplicas "pollingInterval" .Values.incomingRequestIngest.keda.pollingInterval "cooldownPeriod" .Values.incomingRequestIngest.keda.cooldownPeriod "triggers" (list (dict "query" "oneuptime_incoming_request_ingest_queue_size" "threshold" .Values.incomingRequestIngest.keda.queueSizeThreshold "port" .Values.incomingRequestIngest.ports.http)) }}
|
||||
{{- $incomingRequestIngestKedaArgs := dict "ServiceName" "incoming-request-ingest" "Release" .Release "Values" .Values "MetricsConfig" $metricsConfig "DisableAutoscaler" .Values.incomingRequestIngest.disableAutoscaler }}
|
||||
{{- include "oneuptime.kedaScaledObject" $incomingRequestIngestKedaArgs }}
|
||||
{{- end }}
|
||||
|
||||
{{/* Server Monitor Ingest KEDA ScaledObject */}}
|
||||
{{- if and .Values.keda.enabled .Values.serverMonitorIngest.enabled .Values.serverMonitorIngest.keda.enabled (not .Values.serverMonitorIngest.disableAutoscaler) }}
|
||||
{{- $metricsConfig := dict "enabled" .Values.serverMonitorIngest.keda.enabled "minReplicas" .Values.serverMonitorIngest.keda.minReplicas "maxReplicas" .Values.serverMonitorIngest.keda.maxReplicas "pollingInterval" .Values.serverMonitorIngest.keda.pollingInterval "cooldownPeriod" .Values.serverMonitorIngest.keda.cooldownPeriod "triggers" (list (dict "query" "oneuptime_server_monitor_ingest_queue_size" "threshold" .Values.serverMonitorIngest.keda.queueSizeThreshold "port" .Values.serverMonitorIngest.ports.http)) }}
|
||||
{{- $serverMonitorIngestKedaArgs := dict "ServiceName" "server-monitor-ingest" "Release" .Release "Values" .Values "MetricsConfig" $metricsConfig "DisableAutoscaler" .Values.serverMonitorIngest.disableAutoscaler }}
|
||||
{{- include "oneuptime.kedaScaledObject" $serverMonitorIngestKedaArgs }}
|
||||
{{- end }}
|
||||
|
||||
{{/* Probe Ingest KEDA ScaledObject */}}
|
||||
{{- if and .Values.keda.enabled .Values.probeIngest.enabled .Values.probeIngest.keda.enabled (not .Values.probeIngest.disableAutoscaler) }}
|
||||
{{- $metricsConfig := dict "enabled" .Values.probeIngest.keda.enabled "minReplicas" .Values.probeIngest.keda.minReplicas "maxReplicas" .Values.probeIngest.keda.maxReplicas "pollingInterval" .Values.probeIngest.keda.pollingInterval "cooldownPeriod" .Values.probeIngest.keda.cooldownPeriod "triggers" (list (dict "query" "oneuptime_probe_ingest_queue_size" "threshold" .Values.probeIngest.keda.queueSizeThreshold "port" .Values.probeIngest.ports.http)) }}
|
||||
{{- $probeIngestKedaArgs := dict "ServiceName" "probe-ingest" "Release" .Release "Values" .Values "MetricsConfig" $metricsConfig "DisableAutoscaler" .Values.probeIngest.disableAutoscaler }}
|
||||
{{- include "oneuptime.kedaScaledObject" $probeIngestKedaArgs }}
|
||||
{{- end }}
|
||||
|
||||
{{/* Probe KEDA ScaledObjects - one for each probe configuration */}}
|
||||
{{- range $key, $val := $.Values.probes }}
|
||||
{{- $probeEnabled := or (not (hasKey $val "enabled")) $val.enabled }}
|
||||
|
||||
@@ -1,141 +0,0 @@
|
||||
{{- if $.Values.probeIngest.enabled }}
|
||||
# OneUptime probe-ingest Deployment
|
||||
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ printf "%s-%s" $.Release.Name "probe-ingest" }}
|
||||
namespace: {{ $.Release.Namespace }}
|
||||
labels:
|
||||
app: {{ printf "%s-%s" $.Release.Name "probe-ingest" }}
|
||||
app.kubernetes.io/part-of: oneuptime
|
||||
app.kubernetes.io/managed-by: Helm
|
||||
appname: oneuptime
|
||||
{{- if $.Values.deployment.includeTimestampLabel }}
|
||||
date: "{{ now | unixEpoch }}"
|
||||
{{- end }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ printf "%s-%s" $.Release.Name "probe-ingest" }}
|
||||
{{- if $.Values.probeIngest.replicaCount }}
|
||||
replicas: {{ $.Values.probeIngest.replicaCount }}
|
||||
{{- else }}
|
||||
{{- if or (not $.Values.autoscaling.enabled) ($.Values.probeIngest.disableAutoscaler) }}
|
||||
replicas: {{ $.Values.deployment.replicaCount }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
strategy: {{- toYaml $.Values.deployment.updateStrategy | nindent 4 }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: {{ printf "%s-%s" $.Release.Name "probe-ingest" }}
|
||||
{{- if $.Values.deployment.includeTimestampLabel }}
|
||||
date: "{{ now | unixEpoch }}"
|
||||
{{- end }}
|
||||
appname: oneuptime
|
||||
spec:
|
||||
volumes:
|
||||
- name: greenlockrc
|
||||
emptyDir:
|
||||
sizeLimit: "1Gi"
|
||||
{{- if $.Values.probeIngest.podSecurityContext }}
|
||||
securityContext:
|
||||
{{- toYaml $.Values.probeIngest.podSecurityContext | nindent 8 }}
|
||||
{{- else if $.Values.podSecurityContext }}
|
||||
securityContext:
|
||||
{{- toYaml $.Values.podSecurityContext | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if $.Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml $.Values.imagePullSecrets | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if $.Values.affinity }}
|
||||
affinity: {{- $.Values.affinity | toYaml | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if $.Values.tolerations }}
|
||||
tolerations: {{- $.Values.tolerations | toYaml | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if $.Values.probeIngest.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml $.Values.probeIngest.nodeSelector | nindent 8 }}
|
||||
{{- else if $.Values.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml $.Values.nodeSelector | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- image: {{ include "oneuptime.image" (dict "Values" $.Values "ServiceName" "probe-ingest") }}
|
||||
name: {{ printf "%s-%s" $.Release.Name "probe-ingest" }}
|
||||
{{- if $.Values.startupProbe.enabled }}
|
||||
# Startup probe
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /status/live
|
||||
port: {{ $.Values.probeIngest.ports.http }}
|
||||
periodSeconds: {{ $.Values.startupProbe.periodSeconds }}
|
||||
failureThreshold: {{ $.Values.startupProbe.failureThreshold }}
|
||||
{{- end }}
|
||||
{{- if $.Values.livenessProbe.enabled }}
|
||||
# Liveness probe
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /status/live
|
||||
port: {{ $.Values.probeIngest.ports.http }}
|
||||
periodSeconds: {{ $.Values.livenessProbe.periodSeconds }}
|
||||
timeoutSeconds: {{ $.Values.livenessProbe.timeoutSeconds }}
|
||||
initialDelaySeconds: {{ $.Values.livenessProbe.initialDelaySeconds }}
|
||||
{{- end }}
|
||||
{{- if $.Values.readinessProbe.enabled }}
|
||||
# Readyness Probe
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /status/ready
|
||||
port: {{ $.Values.probeIngest.ports.http }}
|
||||
periodSeconds: {{ $.Values.readinessProbe.periodSeconds }}
|
||||
initialDelaySeconds: {{ $.Values.readinessProbe.initialDelaySeconds }}
|
||||
timeoutSeconds: {{ $.Values.readinessProbe.timeoutSeconds }}
|
||||
{{- end }}
|
||||
{{- if $.Values.probeIngest.containerSecurityContext }}
|
||||
securityContext:
|
||||
{{- toYaml $.Values.probeIngest.containerSecurityContext | nindent 12 }}
|
||||
{{- else if $.Values.containerSecurityContext }}
|
||||
securityContext:
|
||||
{{- toYaml $.Values.containerSecurityContext | nindent 12 }}
|
||||
{{- end }}
|
||||
imagePullPolicy: {{ $.Values.image.pullPolicy }}
|
||||
env:
|
||||
{{- include "oneuptime.env.common" . | nindent 12 }}
|
||||
{{- include "oneuptime.env.runtime" (dict "Values" $.Values "Release" $.Release) | nindent 12 }}
|
||||
- name: PORT
|
||||
value: {{ $.Values.probeIngest.ports.http | quote }}
|
||||
- name: DISABLE_TELEMETRY
|
||||
value: {{ $.Values.probeIngest.disableTelemetryCollection | quote }}
|
||||
- name: PROBE_INGEST_CONCURRENCY
|
||||
value: {{ $.Values.probeIngest.concurrency | squote }}
|
||||
{{- include "oneuptime.env.registerProbeKey" (dict "Values" $.Values "Release" $.Release) | nindent 12 }}
|
||||
ports:
|
||||
- containerPort: {{ $.Values.probeIngest.ports.http }}
|
||||
protocol: TCP
|
||||
name: http
|
||||
{{- if $.Values.probeIngest.resources }}
|
||||
resources:
|
||||
{{- toYaml $.Values.probeIngest.resources | nindent 12 }}
|
||||
{{- end }}
|
||||
restartPolicy: {{ $.Values.image.restartPolicy }}
|
||||
|
||||
---
|
||||
|
||||
# OneUptime probe-ingest autoscaler
|
||||
{{- if and (not $.Values.probeIngest.disableAutoscaler) (not (and $.Values.keda.enabled $.Values.probeIngest.keda.enabled)) }}
|
||||
{{- $probeIngestAutoScalerArgs := dict "ServiceName" "probe-ingest" "Release" $.Release "Values" $.Values -}}
|
||||
{{- include "oneuptime.autoscaler" $probeIngestAutoScalerArgs }}
|
||||
{{- end }}
|
||||
---
|
||||
|
||||
{{- end }}
|
||||
|
||||
# OneUptime probe-ingest Service
|
||||
{{- $probeIngestPorts := dict "port" $.Values.probeIngest.ports.http -}}
|
||||
{{- $probeIngestServiceArgs := dict "ServiceName" "probe-ingest" "Ports" $probeIngestPorts "Release" $.Release "Values" $.Values -}}
|
||||
{{- include "oneuptime.service" $probeIngestServiceArgs }}
|
||||
---
|
||||
@@ -85,7 +85,7 @@ spec:
|
||||
- name: OPENTELEMETRY_EXPORTER_OTLP_ENDPOINT
|
||||
value: {{ $.Values.openTelemetryExporter.endpoint }}
|
||||
- name: ONEUPTIME_URL
|
||||
value: http://{{ $.Release.Name }}-probe-ingest.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }}:{{ $.Values.probeIngest.ports.http }}
|
||||
value: http://{{ $.Release.Name }}-telemetry.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }}:{{ $.Values.telemetry.ports.http }}
|
||||
- name: PROBE_NAME
|
||||
value: {{ $val.name }}
|
||||
- name: PROBE_DESCRIPTION
|
||||
|
||||
@@ -1,140 +0,0 @@
|
||||
{{- if $.Values.serverMonitorIngest.enabled }}
|
||||
# OneUptime server-monitor-ingest Deployment
|
||||
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ printf "%s-%s" $.Release.Name "server-monitor-ingest" }}
|
||||
namespace: {{ $.Release.Namespace }}
|
||||
labels:
|
||||
app: {{ printf "%s-%s" $.Release.Name "server-monitor-ingest" }}
|
||||
app.kubernetes.io/part-of: oneuptime
|
||||
app.kubernetes.io/managed-by: Helm
|
||||
appname: oneuptime
|
||||
{{- if $.Values.deployment.includeTimestampLabel }}
|
||||
date: "{{ now | unixEpoch }}"
|
||||
{{- end }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ printf "%s-%s" $.Release.Name "server-monitor-ingest" }}
|
||||
{{- if $.Values.serverMonitorIngest.replicaCount }}
|
||||
replicas: {{ $.Values.serverMonitorIngest.replicaCount }}
|
||||
{{- else }}
|
||||
{{- if or (not $.Values.autoscaling.enabled) ($.Values.serverMonitorIngest.disableAutoscaler) }}
|
||||
replicas: {{ $.Values.deployment.replicaCount }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
strategy: {{- toYaml $.Values.deployment.updateStrategy | nindent 4 }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: {{ printf "%s-%s" $.Release.Name "server-monitor-ingest" }}
|
||||
{{- if $.Values.deployment.includeTimestampLabel }}
|
||||
date: "{{ now | unixEpoch }}"
|
||||
{{- end }}
|
||||
appname: oneuptime
|
||||
spec:
|
||||
volumes:
|
||||
- name: greenlockrc
|
||||
emptyDir:
|
||||
sizeLimit: "1Gi"
|
||||
{{- if $.Values.serverMonitorIngest.podSecurityContext }}
|
||||
securityContext:
|
||||
{{- toYaml $.Values.serverMonitorIngest.podSecurityContext | nindent 8 }}
|
||||
{{- else if $.Values.podSecurityContext }}
|
||||
securityContext:
|
||||
{{- toYaml $.Values.podSecurityContext | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if $.Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml $.Values.imagePullSecrets | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if $.Values.affinity }}
|
||||
affinity: {{- $.Values.affinity | toYaml | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if $.Values.tolerations }}
|
||||
tolerations: {{- $.Values.tolerations | toYaml | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if $.Values.serverMonitorIngest.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml $.Values.serverMonitorIngest.nodeSelector | nindent 8 }}
|
||||
{{- else if $.Values.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml $.Values.nodeSelector | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- image: {{ include "oneuptime.image" (dict "Values" $.Values "ServiceName" "server-monitor-ingest") }}
|
||||
name: {{ printf "%s-%s" $.Release.Name "server-monitor-ingest" }}
|
||||
{{- if $.Values.startupProbe.enabled }}
|
||||
# Startup probe
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /status/live
|
||||
port: {{ $.Values.serverMonitorIngest.ports.http }}
|
||||
periodSeconds: {{ $.Values.startupProbe.periodSeconds }}
|
||||
failureThreshold: {{ $.Values.startupProbe.failureThreshold }}
|
||||
{{- end }}
|
||||
{{- if $.Values.livenessProbe.enabled }}
|
||||
# Liveness probe
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /status/live
|
||||
port: {{ $.Values.serverMonitorIngest.ports.http }}
|
||||
periodSeconds: {{ $.Values.livenessProbe.periodSeconds }}
|
||||
timeoutSeconds: {{ $.Values.livenessProbe.timeoutSeconds }}
|
||||
initialDelaySeconds: {{ $.Values.livenessProbe.initialDelaySeconds }}
|
||||
{{- end }}
|
||||
{{- if $.Values.readinessProbe.enabled }}
|
||||
# Readyness Probe
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /status/ready
|
||||
port: {{ $.Values.serverMonitorIngest.ports.http }}
|
||||
periodSeconds: {{ $.Values.readinessProbe.periodSeconds }}
|
||||
initialDelaySeconds: {{ $.Values.readinessProbe.initialDelaySeconds }}
|
||||
timeoutSeconds: {{ $.Values.readinessProbe.timeoutSeconds }}
|
||||
{{- end }}
|
||||
{{- if $.Values.serverMonitorIngest.containerSecurityContext }}
|
||||
securityContext:
|
||||
{{- toYaml $.Values.serverMonitorIngest.containerSecurityContext | nindent 12 }}
|
||||
{{- else if $.Values.containerSecurityContext }}
|
||||
securityContext:
|
||||
{{- toYaml $.Values.containerSecurityContext | nindent 12 }}
|
||||
{{- end }}
|
||||
imagePullPolicy: {{ $.Values.image.pullPolicy }}
|
||||
env:
|
||||
{{- include "oneuptime.env.common" . | nindent 12 }}
|
||||
{{- include "oneuptime.env.runtime" (dict "Values" $.Values "Release" $.Release) | nindent 12 }}
|
||||
- name: PORT
|
||||
value: {{ $.Values.serverMonitorIngest.ports.http | quote }}
|
||||
- name: DISABLE_TELEMETRY
|
||||
value: {{ $.Values.serverMonitorIngest.disableTelemetryCollection | quote }}
|
||||
- name: SERVER_MONITOR_INGEST_CONCURRENCY
|
||||
value: {{ $.Values.serverMonitorIngest.concurrency | squote }}
|
||||
ports:
|
||||
- containerPort: {{ $.Values.serverMonitorIngest.ports.http }}
|
||||
protocol: TCP
|
||||
name: http
|
||||
{{- if $.Values.serverMonitorIngest.resources }}
|
||||
resources:
|
||||
{{- toYaml $.Values.serverMonitorIngest.resources | nindent 12 }}
|
||||
{{- end }}
|
||||
restartPolicy: {{ $.Values.image.restartPolicy }}
|
||||
|
||||
---
|
||||
|
||||
# OneUptime server-monitor-ingest autoscaler
|
||||
{{- if and (not $.Values.serverMonitorIngest.disableAutoscaler) (not (and $.Values.keda.enabled $.Values.serverMonitorIngest.keda.enabled)) }}
|
||||
{{- $serverMonitorIngestAutoScalerArgs := dict "ServiceName" "server-monitor-ingest" "Release" $.Release "Values" $.Values -}}
|
||||
{{- include "oneuptime.autoscaler" $serverMonitorIngestAutoScalerArgs }}
|
||||
{{- end }}
|
||||
---
|
||||
|
||||
{{- end }}
|
||||
|
||||
# OneUptime server-monitor-ingest Service
|
||||
{{- $serverMonitorIngestPorts := dict "port" $.Values.serverMonitorIngest.ports.http -}}
|
||||
{{- $serverMonitorIngestServiceArgs := dict "ServiceName" "server-monitor-ingest" "Ports" $serverMonitorIngestPorts "Release" $.Release "Values" $.Values -}}
|
||||
{{- include "oneuptime.service" $serverMonitorIngestServiceArgs }}
|
||||
---
|
||||
@@ -112,6 +112,8 @@ spec:
|
||||
value: {{ $.Values.telemetry.disableTelemetryCollection | quote }}
|
||||
- name: TELEMETRY_CONCURRENCY
|
||||
value: {{ $.Values.telemetry.concurrency | squote }}
|
||||
- name: REGISTER_PROBE_KEY
|
||||
value: {{ $.Values.probes.one.key | quote }}
|
||||
ports:
|
||||
- containerPort: {{ $.Values.telemetry.ports.http }}
|
||||
protocol: TCP
|
||||
|
||||
@@ -1694,72 +1694,6 @@
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"probeIngest": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"replicaCount": {
|
||||
"type": "integer"
|
||||
},
|
||||
"disableTelemetryCollection": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"disableAutoscaler": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"concurrency": {
|
||||
"type": "integer"
|
||||
},
|
||||
"ports": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"http": {
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"resources": {
|
||||
"type": ["object", "null"]
|
||||
},
|
||||
"nodeSelector": {
|
||||
"type": "object"
|
||||
},
|
||||
"podSecurityContext": {
|
||||
"type": "object"
|
||||
},
|
||||
"containerSecurityContext": {
|
||||
"type": "object"
|
||||
},
|
||||
"keda": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"minReplicas": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxReplicas": {
|
||||
"type": "integer"
|
||||
},
|
||||
"queueSizeThreshold": {
|
||||
"type": "integer"
|
||||
},
|
||||
"pollingInterval": {
|
||||
"type": "integer"
|
||||
},
|
||||
"cooldownPeriod": {
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"telemetry": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -1826,72 +1760,6 @@
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"incomingRequestIngest": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"replicaCount": {
|
||||
"type": "integer"
|
||||
},
|
||||
"disableTelemetryCollection": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"disableAutoscaler": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"concurrency": {
|
||||
"type": "integer"
|
||||
},
|
||||
"ports": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"http": {
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"resources": {
|
||||
"type": ["object", "null"]
|
||||
},
|
||||
"nodeSelector": {
|
||||
"type": "object"
|
||||
},
|
||||
"podSecurityContext": {
|
||||
"type": "object"
|
||||
},
|
||||
"containerSecurityContext": {
|
||||
"type": "object"
|
||||
},
|
||||
"keda": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"minReplicas": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxReplicas": {
|
||||
"type": "integer"
|
||||
},
|
||||
"queueSizeThreshold": {
|
||||
"type": "integer"
|
||||
},
|
||||
"pollingInterval": {
|
||||
"type": "integer"
|
||||
},
|
||||
"cooldownPeriod": {
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"isolatedVM": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -1970,72 +1838,6 @@
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"serverMonitorIngest": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"replicaCount": {
|
||||
"type": "integer"
|
||||
},
|
||||
"disableTelemetryCollection": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"disableAutoscaler": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"concurrency": {
|
||||
"type": "integer"
|
||||
},
|
||||
"ports": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"http": {
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"resources": {
|
||||
"type": ["object", "null"]
|
||||
},
|
||||
"nodeSelector": {
|
||||
"type": "object"
|
||||
},
|
||||
"podSecurityContext": {
|
||||
"type": "object"
|
||||
},
|
||||
"containerSecurityContext": {
|
||||
"type": "object"
|
||||
},
|
||||
"keda": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"minReplicas": {
|
||||
"type": "integer"
|
||||
},
|
||||
"maxReplicas": {
|
||||
"type": "integer"
|
||||
},
|
||||
"queueSizeThreshold": {
|
||||
"type": "integer"
|
||||
},
|
||||
"pollingInterval": {
|
||||
"type": "integer"
|
||||
},
|
||||
"cooldownPeriod": {
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"slackApp": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
||||
@@ -695,30 +695,6 @@ app:
|
||||
podSecurityContext: {}
|
||||
containerSecurityContext: {}
|
||||
|
||||
probeIngest:
|
||||
enabled: true
|
||||
replicaCount: 1
|
||||
disableTelemetryCollection: false
|
||||
disableAutoscaler: false
|
||||
concurrency: 100
|
||||
ports:
|
||||
http: 3400
|
||||
resources:
|
||||
nodeSelector: {}
|
||||
podSecurityContext: {}
|
||||
containerSecurityContext: {}
|
||||
# KEDA autoscaling configuration based on queue metrics
|
||||
keda:
|
||||
enabled: false
|
||||
minReplicas: 1
|
||||
maxReplicas: 100
|
||||
# Scale up when queue size exceeds this threshold
|
||||
queueSizeThreshold: 100
|
||||
# Polling interval for metrics (in seconds)
|
||||
pollingInterval: 30
|
||||
# Cooldown period after scaling (in seconds)
|
||||
cooldownPeriod: 300
|
||||
|
||||
telemetry:
|
||||
enabled: true
|
||||
replicaCount: 1
|
||||
@@ -744,30 +720,6 @@ telemetry:
|
||||
# Cooldown period after scaling (in seconds)
|
||||
cooldownPeriod: 300
|
||||
|
||||
incomingRequestIngest:
|
||||
enabled: true
|
||||
replicaCount: 1
|
||||
disableTelemetryCollection: false
|
||||
disableAutoscaler: false
|
||||
concurrency: 100
|
||||
ports:
|
||||
http: 3402
|
||||
resources:
|
||||
nodeSelector: {}
|
||||
podSecurityContext: {}
|
||||
containerSecurityContext: {}
|
||||
# KEDA autoscaling configuration based on queue metrics
|
||||
keda:
|
||||
enabled: false
|
||||
minReplicas: 1
|
||||
maxReplicas: 100
|
||||
# Scale up when queue size exceeds this threshold
|
||||
queueSizeThreshold: 100
|
||||
# Polling interval for metrics (in seconds)
|
||||
pollingInterval: 30
|
||||
# Cooldown period after scaling (in seconds)
|
||||
cooldownPeriod: 300
|
||||
|
||||
isolatedVM:
|
||||
enabled: true
|
||||
replicaCount: 1
|
||||
@@ -822,30 +774,6 @@ aiAgent:
|
||||
# Cooldown period after scaling (in seconds)
|
||||
cooldownPeriod: 300
|
||||
|
||||
serverMonitorIngest:
|
||||
enabled: true
|
||||
replicaCount: 1
|
||||
disableTelemetryCollection: false
|
||||
disableAutoscaler: false
|
||||
concurrency: 100
|
||||
ports:
|
||||
http: 3404
|
||||
resources:
|
||||
nodeSelector: {}
|
||||
podSecurityContext: {}
|
||||
containerSecurityContext: {}
|
||||
# KEDA autoscaling configuration based on queue metrics
|
||||
keda:
|
||||
enabled: false
|
||||
minReplicas: 1
|
||||
maxReplicas: 100
|
||||
# Scale up when queue size exceeds this threshold
|
||||
queueSizeThreshold: 100
|
||||
# Polling interval for metrics (in seconds)
|
||||
pollingInterval: 30
|
||||
# Cooldown period after scaling (in seconds)
|
||||
cooldownPeriod: 300
|
||||
|
||||
slackApp:
|
||||
clientId:
|
||||
clientSecret:
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
.git
|
||||
|
||||
node_modules
|
||||
# See https://help.github.com/ignore-files/ for more about ignoring files.
|
||||
|
||||
# dependencies
|
||||
/node_modules
|
||||
node_modules
|
||||
|
||||
.idea
|
||||
# testing
|
||||
/coverage
|
||||
|
||||
# production
|
||||
/build
|
||||
|
||||
# misc
|
||||
.DS_Store
|
||||
|
||||
env.js
|
||||
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
|
||||
yarn.lock
|
||||
Untitled-1
|
||||
*.local.sh
|
||||
*.local.yaml
|
||||
run
|
||||
stop
|
||||
|
||||
nohup.out*
|
||||
|
||||
encrypted-credentials.tar
|
||||
encrypted-credentials/
|
||||
|
||||
_README.md
|
||||
|
||||
# Important Add production values to gitignore.
|
||||
values-saas-production.yaml
|
||||
kubernetes/values-saas-production.yaml
|
||||
|
||||
/private
|
||||
|
||||
/tls_cert.pem
|
||||
/tls_key.pem
|
||||
/keys
|
||||
|
||||
temp_readme.md
|
||||
|
||||
tests/coverage
|
||||
|
||||
settings.json
|
||||
|
||||
GoSDK/tester/
|
||||
1
IncomingRequestIngest/.gitattributes
vendored
1
IncomingRequestIngest/.gitattributes
vendored
@@ -1 +0,0 @@
|
||||
*.js text eol=lf
|
||||
16
IncomingRequestIngest/.gitignore
vendored
16
IncomingRequestIngest/.gitignore
vendored
@@ -1,16 +0,0 @@
|
||||
# See https://help.github.com/ignore-files/ for more about ignoring files.
|
||||
|
||||
# dependencies
|
||||
#/backend/node_modules
|
||||
/kubernetes
|
||||
/node_modules
|
||||
.idea
|
||||
|
||||
# misc
|
||||
.DS_Store
|
||||
|
||||
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
yarn.lock
|
||||
@@ -1,38 +0,0 @@
|
||||
import Express, {
|
||||
ExpressRequest,
|
||||
ExpressResponse,
|
||||
ExpressRouter,
|
||||
NextFunction,
|
||||
} from "Common/Server/Utils/Express";
|
||||
import IncomingRequestIngestQueueService from "../Services/Queue/IncomingRequestIngestQueueService";
|
||||
// import ClusterKeyAuthorization from "Common/Server/Middleware/ClusterKeyAuthorization";
|
||||
|
||||
const router: ExpressRouter = Express.getRouter();
|
||||
|
||||
/**
|
||||
* JSON metrics endpoint for KEDA autoscaling
|
||||
* Returns queue size as JSON for KEDA metrics-api scaler
|
||||
*/
|
||||
router.get(
|
||||
"/metrics/queue-size",
|
||||
// ClusterKeyAuthorization.isAuthorizedServiceMiddleware, // Temporarily disabled for KEDA debugging
|
||||
async (
|
||||
_req: ExpressRequest,
|
||||
res: ExpressResponse,
|
||||
next: NextFunction,
|
||||
): Promise<void> => {
|
||||
try {
|
||||
const queueSize: number =
|
||||
await IncomingRequestIngestQueueService.getQueueSize();
|
||||
|
||||
res.setHeader("Content-Type", "application/json");
|
||||
res.status(200).json({
|
||||
queueSize: queueSize,
|
||||
});
|
||||
} catch (err) {
|
||||
return next(err);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
export default router;
|
||||
@@ -1,10 +0,0 @@
|
||||
let concurrency: string | number =
|
||||
process.env["INCOMING_REQUEST_INGEST_CONCURRENCY"] || 100;
|
||||
|
||||
if (typeof concurrency === "string") {
|
||||
const parsed: number = parseInt(concurrency, 10);
|
||||
concurrency = !isNaN(parsed) && parsed > 0 ? parsed : 100;
|
||||
}
|
||||
|
||||
export const INCOMING_REQUEST_INGEST_CONCURRENCY: number =
|
||||
concurrency as number;
|
||||
@@ -1,83 +0,0 @@
|
||||
#
|
||||
# OneUptime-IncomingRequestIngest Dockerfile
|
||||
#
|
||||
|
||||
# Pull base image nodejs image.
|
||||
FROM public.ecr.aws/docker/library/node:23.8
|
||||
RUN mkdir /tmp/npm && chmod 2777 /tmp/npm && chown 1000:1000 /tmp/npm && npm config set cache /tmp/npm --global
|
||||
|
||||
RUN npm config set fetch-retries 5
|
||||
RUN npm config set fetch-retry-mintimeout 20000
|
||||
RUN npm config set fetch-retry-maxtimeout 60000
|
||||
|
||||
|
||||
ARG GIT_SHA
|
||||
ARG APP_VERSION
|
||||
ARG IS_ENTERPRISE_EDITION=false
|
||||
|
||||
ENV GIT_SHA=${GIT_SHA}
|
||||
ENV APP_VERSION=${APP_VERSION}
|
||||
ENV IS_ENTERPRISE_EDITION=${IS_ENTERPRISE_EDITION}
|
||||
ENV PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1
|
||||
ENV NODE_OPTIONS="--use-openssl-ca"
|
||||
|
||||
## Add Intermediate Certs
|
||||
COPY ./SslCertificates /usr/local/share/ca-certificates
|
||||
RUN update-ca-certificates
|
||||
|
||||
|
||||
# IF APP_VERSION is not set, set it to 1.0.0
|
||||
RUN if [ -z "$APP_VERSION" ]; then export APP_VERSION=1.0.0; fi
|
||||
|
||||
|
||||
RUN apt-get update
|
||||
|
||||
# Install bash.
|
||||
RUN apt-get install bash -y && apt-get install curl -y && apt-get install iputils-ping -y
|
||||
|
||||
# Install python
|
||||
RUN apt-get update && apt-get install -y .gyp python3 make g++
|
||||
|
||||
#Use bash shell by default
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
|
||||
# Install iputils
|
||||
RUN apt-get install net-tools -y
|
||||
|
||||
RUN mkdir -p /usr/src
|
||||
|
||||
WORKDIR /usr/src/Common
|
||||
COPY ./Common/package*.json /usr/src/Common/
|
||||
# Set version in ./Common/package.json to the APP_VERSION
|
||||
RUN sed -i "s/\"version\": \".*\"/\"version\": \"$APP_VERSION\"/g" /usr/src/Common/package.json
|
||||
RUN npm install
|
||||
COPY ./Common /usr/src/Common
|
||||
|
||||
|
||||
|
||||
ENV PRODUCTION=true
|
||||
|
||||
WORKDIR /usr/src/app
|
||||
|
||||
# Install app dependencies
|
||||
COPY ./IncomingRequestIngest/package*.json /usr/src/app/
|
||||
RUN npm install
|
||||
|
||||
# Expose ports.
|
||||
# - 3402: OneUptime-IncomingRequestIngest
|
||||
EXPOSE 3402
|
||||
|
||||
{{ if eq .Env.ENVIRONMENT "development" }}
|
||||
#Run the app
|
||||
CMD [ "npm", "run", "dev" ]
|
||||
{{ else }}
|
||||
# Copy app source
|
||||
COPY ./IncomingRequestIngest /usr/src/app
|
||||
# Bundle app source
|
||||
RUN npm run compile
|
||||
# Set permission to write logs and cache in case container run as non root
|
||||
RUN chown -R 1000:1000 "/tmp/npm" && chmod -R 2777 "/tmp/npm"
|
||||
#Run the app
|
||||
CMD [ "npm", "start" ]
|
||||
{{ end }}
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
import IncomingRequestAPI from "./API/IncomingRequest";
|
||||
import MetricsAPI from "./API/Metrics";
|
||||
import { PromiseVoidFunction } from "Common/Types/FunctionTypes";
|
||||
import { ClickhouseAppInstance } from "Common/Server/Infrastructure/ClickhouseDatabase";
|
||||
import PostgresAppInstance from "Common/Server/Infrastructure/PostgresDatabase";
|
||||
import Redis from "Common/Server/Infrastructure/Redis";
|
||||
import InfrastructureStatus from "Common/Server/Infrastructure/Status";
|
||||
import Express, { ExpressApplication } from "Common/Server/Utils/Express";
|
||||
import logger from "Common/Server/Utils/Logger";
|
||||
import Realtime from "Common/Server/Utils/Realtime";
|
||||
import App from "Common/Server/Utils/StartServer";
|
||||
import Telemetry from "Common/Server/Utils/Telemetry";
|
||||
import "./Jobs/IncomingRequestIngest/ProcessIncomingRequestIngest";
|
||||
import { INCOMING_REQUEST_INGEST_CONCURRENCY } from "./Config";
|
||||
import "ejs";
|
||||
|
||||
const app: ExpressApplication = Express.getExpressApp();
|
||||
|
||||
const APP_NAME: string = "incoming-request-ingest";
|
||||
|
||||
app.use([`/${APP_NAME}`, "/"], IncomingRequestAPI);
|
||||
app.use([`/${APP_NAME}`, "/"], MetricsAPI);
|
||||
|
||||
const init: PromiseVoidFunction = async (): Promise<void> => {
|
||||
try {
|
||||
const statusCheck: PromiseVoidFunction = async (): Promise<void> => {
|
||||
return await InfrastructureStatus.checkStatusWithRetry({
|
||||
checkClickhouseStatus: true,
|
||||
checkPostgresStatus: true,
|
||||
checkRedisStatus: true,
|
||||
retryCount: 3,
|
||||
});
|
||||
};
|
||||
|
||||
// Initialize telemetry
|
||||
Telemetry.init({
|
||||
serviceName: APP_NAME,
|
||||
});
|
||||
|
||||
logger.info(
|
||||
`IncomingRequestIngest Service - Queue concurrency: ${INCOMING_REQUEST_INGEST_CONCURRENCY}`,
|
||||
);
|
||||
|
||||
// init the app
|
||||
await App.init({
|
||||
appName: APP_NAME,
|
||||
statusOptions: {
|
||||
liveCheck: statusCheck,
|
||||
readyCheck: statusCheck,
|
||||
},
|
||||
});
|
||||
|
||||
// connect to the database.
|
||||
await PostgresAppInstance.connect();
|
||||
|
||||
// connect redis
|
||||
await Redis.connect();
|
||||
|
||||
await ClickhouseAppInstance.connect(
|
||||
ClickhouseAppInstance.getDatasourceOptions(),
|
||||
);
|
||||
|
||||
await Realtime.init();
|
||||
|
||||
// add default routes
|
||||
await App.addDefaultRoutes();
|
||||
} catch (err) {
|
||||
logger.error("App Init Failed:");
|
||||
logger.error(err);
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
init().catch((err: Error) => {
|
||||
logger.error(err);
|
||||
logger.error("Exiting node process");
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -1,80 +0,0 @@
|
||||
import Queue, { QueueName } from "Common/Server/Infrastructure/Queue";
|
||||
import { JSONObject } from "Common/Types/JSON";
|
||||
import OneUptimeDate from "Common/Types/Date";
|
||||
import logger from "Common/Server/Utils/Logger";
|
||||
import Dictionary from "Common/Types/Dictionary";
|
||||
|
||||
export interface IncomingRequestIngestJobData {
|
||||
secretKey: string;
|
||||
requestHeaders: Dictionary<string>;
|
||||
requestBody: string | JSONObject;
|
||||
requestMethod: string;
|
||||
ingestionTimestamp: Date;
|
||||
}
|
||||
|
||||
export default class IncomingRequestIngestQueueService {
|
||||
public static async addIncomingRequestIngestJob(data: {
|
||||
secretKey: string;
|
||||
requestHeaders: Dictionary<string>;
|
||||
requestBody: string | JSONObject;
|
||||
requestMethod: string;
|
||||
}): Promise<void> {
|
||||
try {
|
||||
const jobData: IncomingRequestIngestJobData = {
|
||||
secretKey: data.secretKey,
|
||||
requestHeaders: data.requestHeaders,
|
||||
requestBody: data.requestBody,
|
||||
requestMethod: data.requestMethod,
|
||||
ingestionTimestamp: OneUptimeDate.getCurrentDate(),
|
||||
};
|
||||
|
||||
const jobId: string = `incoming-request-${data.secretKey}-${OneUptimeDate.getCurrentDateAsUnixNano()}`;
|
||||
|
||||
await Queue.addJob(
|
||||
QueueName.IncomingRequestIngest,
|
||||
jobId,
|
||||
"ProcessIncomingRequestIngest",
|
||||
jobData as unknown as JSONObject,
|
||||
);
|
||||
|
||||
logger.debug(`Added incoming request ingestion job: ${jobId}`);
|
||||
} catch (error) {
|
||||
logger.error(`Error adding incoming request ingestion job:`);
|
||||
logger.error(error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
public static async getQueueSize(): Promise<number> {
|
||||
return Queue.getQueueSize(QueueName.IncomingRequestIngest);
|
||||
}
|
||||
|
||||
public static async getQueueStats(): Promise<{
|
||||
waiting: number;
|
||||
active: number;
|
||||
completed: number;
|
||||
failed: number;
|
||||
delayed: number;
|
||||
total: number;
|
||||
}> {
|
||||
return Queue.getQueueStats(QueueName.IncomingRequestIngest);
|
||||
}
|
||||
|
||||
public static getFailedJobs(options?: {
|
||||
start?: number;
|
||||
end?: number;
|
||||
}): Promise<
|
||||
Array<{
|
||||
id: string;
|
||||
name: string;
|
||||
data: JSONObject;
|
||||
failedReason: string;
|
||||
stackTrace?: string;
|
||||
processedOn: Date | null;
|
||||
finishedOn: Date | null;
|
||||
attemptsMade: number;
|
||||
}>
|
||||
> {
|
||||
return Queue.getFailedJobs(QueueName.IncomingRequestIngest, options);
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
{
|
||||
|
||||
"preset": "ts-jest",
|
||||
"testPathIgnorePatterns": [
|
||||
"node_modules",
|
||||
"dist"
|
||||
],
|
||||
"verbose": true,
|
||||
"globals": {
|
||||
"ts-jest": {
|
||||
"tsconfig": "tsconfig.json",
|
||||
"babelConfig": false
|
||||
}
|
||||
},
|
||||
"moduleFileExtensions": ["ts", "js", "json"],
|
||||
"transform": {
|
||||
".(ts|tsx)": "ts-jest"
|
||||
},
|
||||
"testEnvironment": "node",
|
||||
"collectCoverage": false,
|
||||
"coverageReporters": ["text", "lcov"],
|
||||
"testRegex": "./Tests/(.*).test.ts",
|
||||
"collectCoverageFrom": ["./**/*.(tsx||ts)"],
|
||||
"coverageThreshold": {
|
||||
"global": {
|
||||
"lines": 0,
|
||||
"functions": 0,
|
||||
"branches": 0,
|
||||
"statements": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
{
|
||||
"watch": ["./","../Common/Server", "../Common/Types", "../Common/Utils", "../Common/Models"],
|
||||
"ext": "ts,tsx",
|
||||
"ignore": ["./node_modules/**", "./public/**", "./bin/**", "./build/**"],
|
||||
"watchOptions": {"useFsEvents": false, "interval": 500},
|
||||
"env": {"TS_NODE_TRANSPILE_ONLY": "1", "TS_NODE_FILES": "false"},
|
||||
"exec": "node -r ts-node/register/transpile-only Index.ts"
|
||||
}
|
||||
4653
IncomingRequestIngest/package-lock.json
generated
4653
IncomingRequestIngest/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"name": "@oneuptime/incoming-request-ingest",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/OneUptime/oneuptime"
|
||||
},
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"start": "export NODE_OPTIONS='--max-old-space-size=8096' && node --require ts-node/register Index.ts",
|
||||
"compile": "tsc",
|
||||
"clear-modules": "rm -rf node_modules && rm package-lock.json && npm install",
|
||||
"dev": "npx nodemon",
|
||||
"audit": "npm audit --audit-level=low",
|
||||
"dep-check": "npm install -g depcheck && depcheck ./ --skip-missing=true",
|
||||
"test": "jest --passWithNoTests"
|
||||
},
|
||||
"author": "OneUptime <hello@oneuptime.com> (https://oneuptime.com/)",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"Common": "file:../Common",
|
||||
"ejs": "^3.1.10",
|
||||
"ts-node": "^10.9.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/jest": "^27.5.0",
|
||||
"@types/node": "^17.0.31",
|
||||
"jest": "^28.1.0",
|
||||
"nodemon": "^2.0.20",
|
||||
"ts-jest": "^28.0.2"
|
||||
}
|
||||
}
|
||||
@@ -1,115 +0,0 @@
|
||||
{
|
||||
"ts-node": {
|
||||
// these options are overrides used only by ts-node
|
||||
// same as the --compilerOptions flag and the TS_NODE_COMPILER_OPTIONS environment variable
|
||||
"compilerOptions": {
|
||||
"module": "commonjs",
|
||||
"resolveJsonModule": true,
|
||||
}
|
||||
},
|
||||
"compilerOptions": {
|
||||
/* Visit https://aka.ms/tsconfig.json to read more about this file */
|
||||
|
||||
/* Projects */
|
||||
// "incremental": true, /* Enable incremental compilation */
|
||||
// "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */
|
||||
// "tsBuildInfoFile": "./", /* Specify the folder for .tsbuildinfo incremental compilation files. */
|
||||
// "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects */
|
||||
// "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */
|
||||
// "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */
|
||||
|
||||
/* Language and Environment */
|
||||
"target": "es2017" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
|
||||
// "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */
|
||||
"jsx": "react" /* Specify what JSX code is generated. */,
|
||||
"experimentalDecorators": true /* Enable experimental support for TC39 stage 2 draft decorators. */,
|
||||
"emitDecoratorMetadata": true /* Emit design-type metadata for decorated declarations in source files. */,
|
||||
// "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h' */
|
||||
// "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
|
||||
// "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using `jsx: react-jsx*`.` */
|
||||
// "reactNamespace": "", /* Specify the object invoked for `createElement`. This only applies when targeting `react` JSX emit. */
|
||||
// "noLib": true, /* Disable including any library files, including the default lib.d.ts. */
|
||||
// "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */
|
||||
|
||||
/* Modules */
|
||||
// "module": "es2022" /* Specify what module code is generated. */,
|
||||
"rootDir": "" /* Specify the root folder within your source files. */,
|
||||
"moduleResolution": "node" /* Specify how TypeScript looks up a file from a given module specifier. */,
|
||||
// "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */
|
||||
// "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */
|
||||
// "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */
|
||||
"typeRoots": [
|
||||
"./node_modules/@types"
|
||||
] /* Specify multiple folders that act like `./node_modules/@types`. */,
|
||||
"types": [
|
||||
"node",
|
||||
"jest"
|
||||
] /* Specify type package names to be included without being referenced in a source file. */,
|
||||
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
|
||||
// "resolveJsonModule": true, /* Enable importing .json files */
|
||||
// "noResolve": true, /* Disallow `import`s, `require`s or `<reference>`s from expanding the number of files TypeScript should add to a project. */
|
||||
|
||||
/* JavaScript Support */
|
||||
// "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */
|
||||
// "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */
|
||||
// "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from `node_modules`. Only applicable with `allowJs`. */
|
||||
|
||||
/* Emit */
|
||||
// "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
|
||||
// "declarationMap": true, /* Create sourcemaps for d.ts files. */
|
||||
// "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */
|
||||
"sourceMap": true /* Create source map files for emitted JavaScript files. */,
|
||||
// "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If `declaration` is true, also designates a file that bundles all .d.ts output. */
|
||||
"outDir": "build/dist" /* Specify an output folder for all emitted files. */,
|
||||
// "removeComments": true, /* Disable emitting comments. */
|
||||
// "noEmit": true, /* Disable emitting files from a compilation. */
|
||||
// "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
|
||||
// "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types */
|
||||
// "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
|
||||
// "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */
|
||||
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
|
||||
// "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */
|
||||
// "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */
|
||||
// "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
|
||||
// "newLine": "crlf", /* Set the newline character for emitting files. */
|
||||
// "stripInternal": true, /* Disable emitting declarations that have `@internal` in their JSDoc comments. */
|
||||
// "noEmitHelpers": true, /* Disable generating custom helper functions like `__extends` in compiled output. */
|
||||
// "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */
|
||||
// "preserveConstEnums": true, /* Disable erasing `const enum` declarations in generated code. */
|
||||
// "declarationDir": "./", /* Specify the output directory for generated declaration files. */
|
||||
// "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */
|
||||
|
||||
/* Interop Constraints */
|
||||
// "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */
|
||||
// "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */
|
||||
"esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables `allowSyntheticDefaultImports` for type compatibility. */,
|
||||
// "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
|
||||
"forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
|
||||
|
||||
/* Type Checking */
|
||||
"strict": true /* Enable all strict type-checking options. */,
|
||||
"noImplicitAny": true /* Enable error reporting for expressions and declarations with an implied `any` type.. */,
|
||||
"strictNullChecks": true /* When type checking, take into account `null` and `undefined`. */,
|
||||
"strictFunctionTypes": true /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */,
|
||||
"strictBindCallApply": true /* Check that the arguments for `bind`, `call`, and `apply` methods match the original function. */,
|
||||
"strictPropertyInitialization": true /* Check for class properties that are declared but not set in the constructor. */,
|
||||
"noImplicitThis": true /* Enable error reporting when `this` is given the type `any`. */,
|
||||
"useUnknownInCatchVariables": true /* Type catch clause variables as 'unknown' instead of 'any'. */,
|
||||
"alwaysStrict": true /* Ensure 'use strict' is always emitted. */,
|
||||
"noUnusedLocals": true /* Enable error reporting when a local variables aren't read. */,
|
||||
"noUnusedParameters": true /* Raise an error when a function parameter isn't read */,
|
||||
"exactOptionalPropertyTypes": true /* Interpret optional property types as written, rather than adding 'undefined'. */,
|
||||
"noImplicitReturns": true /* Enable error reporting for codepaths that do not explicitly return in a function. */,
|
||||
"noFallthroughCasesInSwitch": true /* Enable error reporting for fallthrough cases in switch statements. */,
|
||||
"noUncheckedIndexedAccess": true /* Include 'undefined' in index signature results */,
|
||||
"noImplicitOverride": true /* Ensure overriding members in derived classes are marked with an override modifier. */,
|
||||
"noPropertyAccessFromIndexSignature": true /* Enforces using indexed accessors for keys declared using an indexed type */,
|
||||
// "allowUnusedLabels": true, /* Disable error reporting for unused labels. */
|
||||
// "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */
|
||||
|
||||
/* Completeness */
|
||||
// "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */
|
||||
"skipLibCheck": true, /* Skip type checking all .d.ts files. */
|
||||
"resolveJsonModule": true
|
||||
}
|
||||
}
|
||||
@@ -11,20 +11,8 @@ upstream app {
|
||||
server ${SERVER_APP_HOSTNAME}:${APP_PORT} weight=10 max_fails=3 fail_timeout=30s;
|
||||
}
|
||||
|
||||
upstream probe-ingest {
|
||||
server ${SERVER_PROBE_INGEST_HOSTNAME}:${PROBE_INGEST_PORT} weight=10 max_fails=3 fail_timeout=30s;
|
||||
}
|
||||
|
||||
upstream telemetry {
|
||||
server ${SERVER_TELEMETRY_HOSTNAME}:${TELEMETRY_PORT} weight=10 max_fails=3 fail_timeout=30s;
|
||||
}
|
||||
|
||||
upstream server-monitor-ingest {
|
||||
server ${SERVER_SERVER_MONITOR_INGEST_HOSTNAME}:${SERVER_MONITOR_INGEST_PORT} weight=10 max_fails=3 fail_timeout=30s;
|
||||
}
|
||||
|
||||
upstream incoming-request-ingest {
|
||||
server ${SERVER_INCOMING_REQUEST_INGEST_HOSTNAME}:${INCOMING_REQUEST_INGEST_PORT} weight=10 max_fails=3 fail_timeout=30s;
|
||||
server ${SERVER_TELEMETRY_HOSTNAME}:${TELEMETRY_PORT} weight=10 max_fails=3 fail_timeout=30s;
|
||||
}
|
||||
|
||||
upstream dashboard {
|
||||
@@ -505,14 +493,14 @@ ${PROVISION_SSL_CERTIFICATE_KEY_DIRECTIVE}
|
||||
|
||||
location /incoming-request-ingest {
|
||||
|
||||
# This is for nginx not to crash when service is not available.
|
||||
# This is for nginx not to crash when service is not available.
|
||||
resolver 127.0.0.1 valid=30s;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
proxy_pass http://incoming-request-ingest;
|
||||
proxy_pass http://telemetry;
|
||||
}
|
||||
|
||||
location /otlp/ {
|
||||
@@ -605,42 +593,42 @@ ${PROVISION_SSL_CERTIFICATE_KEY_DIRECTIVE}
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
proxy_pass http://probe-ingest;
|
||||
proxy_pass http://telemetry;
|
||||
|
||||
client_max_body_size 50M;
|
||||
}
|
||||
|
||||
# For backward compatibility with probes that are already deployed
|
||||
location /ingestor {
|
||||
# This is for nginx not to crash when service is not available.
|
||||
# This is for nginx not to crash when service is not available.
|
||||
resolver 127.0.0.1 valid=30s;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# enable WebSockets (for ws://sockjs not connected error in the accounts source: https://stackoverflow.com/questions/41381444/websocket-connection-failed-error-during-websocket-handshake-unexpected-respon)
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
proxy_pass http://probe-ingest;
|
||||
proxy_pass http://telemetry;
|
||||
|
||||
client_max_body_size 50M;
|
||||
}
|
||||
|
||||
location /server-monitor {
|
||||
# This is for nginx not to crash when service is not available.
|
||||
# This is for nginx not to crash when service is not available.
|
||||
resolver 127.0.0.1 valid=30s;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# enable WebSockets (for ws://sockjs not connected error in the accounts source: https://stackoverflow.com/questions/41381444/websocket-connection-failed-error-during-websocket-handshake-unexpected-respon)
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
proxy_pass http://server-monitor-ingest/server-monitor;
|
||||
proxy_pass http://telemetry/server-monitor;
|
||||
|
||||
client_max_body_size 50M;
|
||||
}
|
||||
@@ -881,7 +869,7 @@ ${PROVISION_SSL_CERTIFICATE_KEY_DIRECTIVE}
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
proxy_pass http://incoming-request-ingest/incoming-request;
|
||||
proxy_pass http://telemetry/incoming-request;
|
||||
|
||||
client_max_body_size 50M;
|
||||
}
|
||||
@@ -897,7 +885,7 @@ ${PROVISION_SSL_CERTIFICATE_KEY_DIRECTIVE}
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
proxy_pass http://probe-ingest/incoming-email;
|
||||
proxy_pass http://telemetry/incoming-email;
|
||||
|
||||
client_max_body_size 50M;
|
||||
}
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
.git
|
||||
|
||||
node_modules
|
||||
# See https://help.github.com/ignore-files/ for more about ignoring files.
|
||||
|
||||
# dependencies
|
||||
/node_modules
|
||||
node_modules
|
||||
|
||||
.idea
|
||||
# testing
|
||||
/coverage
|
||||
|
||||
# production
|
||||
/build
|
||||
|
||||
# misc
|
||||
.DS_Store
|
||||
|
||||
env.js
|
||||
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
|
||||
yarn.lock
|
||||
Untitled-1
|
||||
*.local.sh
|
||||
*.local.yaml
|
||||
run
|
||||
stop
|
||||
|
||||
nohup.out*
|
||||
|
||||
encrypted-credentials.tar
|
||||
encrypted-credentials/
|
||||
|
||||
_README.md
|
||||
|
||||
# Important Add production values to gitignore.
|
||||
values-saas-production.yaml
|
||||
kubernetes/values-saas-production.yaml
|
||||
|
||||
/private
|
||||
|
||||
/tls_cert.pem
|
||||
/tls_key.pem
|
||||
/keys
|
||||
|
||||
temp_readme.md
|
||||
|
||||
tests/coverage
|
||||
|
||||
settings.json
|
||||
|
||||
GoSDK/tester/
|
||||
1
ProbeIngest/.gitattributes
vendored
1
ProbeIngest/.gitattributes
vendored
@@ -1 +0,0 @@
|
||||
*.js text eol=lf
|
||||
16
ProbeIngest/.gitignore
vendored
16
ProbeIngest/.gitignore
vendored
@@ -1,16 +0,0 @@
|
||||
# See https://help.github.com/ignore-files/ for more about ignoring files.
|
||||
|
||||
# dependencies
|
||||
#/backend/node_modules
|
||||
/kubernetes
|
||||
/node_modules
|
||||
.idea
|
||||
|
||||
# misc
|
||||
.DS_Store
|
||||
|
||||
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
yarn.lock
|
||||
@@ -1,37 +0,0 @@
|
||||
import Express, {
|
||||
ExpressRequest,
|
||||
ExpressResponse,
|
||||
ExpressRouter,
|
||||
NextFunction,
|
||||
} from "Common/Server/Utils/Express";
|
||||
import ProbeIngestQueueService from "../Services/Queue/ProbeIngestQueueService";
|
||||
// import ClusterKeyAuthorization from "Common/Server/Middleware/ClusterKeyAuthorization";
|
||||
|
||||
const router: ExpressRouter = Express.getRouter();
|
||||
|
||||
/**
|
||||
* JSON metrics endpoint for KEDA autoscaling
|
||||
* Returns queue size as JSON for KEDA metrics-api scaler
|
||||
*/
|
||||
router.get(
|
||||
"/metrics/queue-size",
|
||||
// ClusterKeyAuthorization.isAuthorizedServiceMiddleware, // Temporarily disabled for KEDA debugging
|
||||
async (
|
||||
_req: ExpressRequest,
|
||||
res: ExpressResponse,
|
||||
next: NextFunction,
|
||||
): Promise<void> => {
|
||||
try {
|
||||
const queueSize: number = await ProbeIngestQueueService.getQueueSize();
|
||||
|
||||
res.setHeader("Content-Type", "application/json");
|
||||
res.status(200).json({
|
||||
queueSize: queueSize,
|
||||
});
|
||||
} catch (err) {
|
||||
return next(err);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
export default router;
|
||||
@@ -1,9 +0,0 @@
|
||||
let concurrency: string | number =
|
||||
process.env["PROBE_INGEST_CONCURRENCY"] || 100;
|
||||
|
||||
if (typeof concurrency === "string") {
|
||||
const parsed: number = parseInt(concurrency, 10);
|
||||
concurrency = !isNaN(parsed) && parsed > 0 ? parsed : 100;
|
||||
}
|
||||
|
||||
export const PROBE_INGEST_CONCURRENCY: number = concurrency as number;
|
||||
@@ -1,83 +0,0 @@
|
||||
#
|
||||
# OneUptime-ProbeIngest Dockerfile
|
||||
#
|
||||
|
||||
# Pull base image nodejs image.
|
||||
FROM public.ecr.aws/docker/library/node:23.8
|
||||
RUN mkdir /tmp/npm && chmod 2777 /tmp/npm && chown 1000:1000 /tmp/npm && npm config set cache /tmp/npm --global
|
||||
|
||||
RUN npm config set fetch-retries 5
|
||||
RUN npm config set fetch-retry-mintimeout 20000
|
||||
RUN npm config set fetch-retry-maxtimeout 60000
|
||||
|
||||
|
||||
ARG GIT_SHA
|
||||
ARG APP_VERSION
|
||||
ARG IS_ENTERPRISE_EDITION=false
|
||||
|
||||
ENV GIT_SHA=${GIT_SHA}
|
||||
ENV APP_VERSION=${APP_VERSION}
|
||||
ENV IS_ENTERPRISE_EDITION=${IS_ENTERPRISE_EDITION}
|
||||
ENV NODE_OPTIONS="--use-openssl-ca"
|
||||
|
||||
## Add Intermediate Certs
|
||||
COPY ./SslCertificates /usr/local/share/ca-certificates
|
||||
RUN update-ca-certificates
|
||||
|
||||
|
||||
# IF APP_VERSION is not set, set it to 1.0.0
|
||||
RUN if [ -z "$APP_VERSION" ]; then export APP_VERSION=1.0.0; fi
|
||||
|
||||
|
||||
RUN apt-get update
|
||||
|
||||
# Install bash.
|
||||
RUN apt-get install bash -y && apt-get install curl -y && apt-get install iputils-ping -y
|
||||
|
||||
# Install python
|
||||
RUN apt-get update && apt-get install -y .gyp python3 make g++
|
||||
|
||||
#Use bash shell by default
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
|
||||
# Install iputils
|
||||
RUN apt-get install net-tools -y
|
||||
|
||||
RUN mkdir -p /usr/src
|
||||
|
||||
WORKDIR /usr/src/Common
|
||||
COPY ./Common/package*.json /usr/src/Common/
|
||||
# Set version in ./Common/package.json to the APP_VERSION
|
||||
RUN sed -i "s/\"version\": \".*\"/\"version\": \"$APP_VERSION\"/g" /usr/src/Common/package.json
|
||||
RUN npm install
|
||||
COPY ./Common /usr/src/Common
|
||||
|
||||
|
||||
|
||||
ENV PRODUCTION=true
|
||||
ENV PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1
|
||||
|
||||
WORKDIR /usr/src/app
|
||||
|
||||
# Install app dependencies
|
||||
COPY ./ProbeIngest/package*.json /usr/src/app/
|
||||
RUN npm install
|
||||
|
||||
# Expose ports.
|
||||
# - 3400: OneUptime-probe-ingest
|
||||
EXPOSE 3400
|
||||
|
||||
{{ if eq .Env.ENVIRONMENT "development" }}
|
||||
#Run the app
|
||||
CMD [ "npm", "run", "dev" ]
|
||||
{{ else }}
|
||||
# Copy app source
|
||||
COPY ./ProbeIngest /usr/src/app
|
||||
# Bundle app source
|
||||
RUN npm run compile
|
||||
# Set permission to write logs and cache in case container run as non root
|
||||
RUN chown -R 1000:1000 "/tmp/npm" && chmod -R 2777 "/tmp/npm"
|
||||
#Run the app
|
||||
CMD [ "npm", "start" ]
|
||||
{{ end }}
|
||||
|
||||
@@ -1,85 +0,0 @@
|
||||
import MonitorAPI from "./API/Monitor";
|
||||
import ProbeIngest from "./API/Probe";
|
||||
import RegisterAPI from "./API/Register";
|
||||
import MetricsAPI from "./API/Metrics";
|
||||
import IncomingEmailAPI from "./API/IncomingEmail";
|
||||
import { PromiseVoidFunction } from "Common/Types/FunctionTypes";
|
||||
import { ClickhouseAppInstance } from "Common/Server/Infrastructure/ClickhouseDatabase";
|
||||
import PostgresAppInstance from "Common/Server/Infrastructure/PostgresDatabase";
|
||||
import Redis from "Common/Server/Infrastructure/Redis";
|
||||
import InfrastructureStatus from "Common/Server/Infrastructure/Status";
|
||||
import Express, { ExpressApplication } from "Common/Server/Utils/Express";
|
||||
import logger from "Common/Server/Utils/Logger";
|
||||
import Realtime from "Common/Server/Utils/Realtime";
|
||||
import App from "Common/Server/Utils/StartServer";
|
||||
import Telemetry from "Common/Server/Utils/Telemetry";
|
||||
import "./Jobs/ProbeIngest/ProcessProbeIngest";
|
||||
import { PROBE_INGEST_CONCURRENCY } from "./Config";
|
||||
import "ejs";
|
||||
|
||||
const app: ExpressApplication = Express.getExpressApp();
|
||||
|
||||
const APP_NAME: string = "probe-ingest";
|
||||
|
||||
// "/ingestor" is used here for backward compatibility because probes are already deployed with this path in client environments.
|
||||
app.use([`/${APP_NAME}`, "/ingestor", "/"], RegisterAPI);
|
||||
app.use([`/${APP_NAME}`, "/ingestor", "/"], MonitorAPI);
|
||||
app.use([`/${APP_NAME}`, "/ingestor", "/"], ProbeIngest);
|
||||
app.use([`/${APP_NAME}`, "/"], MetricsAPI);
|
||||
app.use([`/${APP_NAME}`, "/"], IncomingEmailAPI);
|
||||
|
||||
const init: PromiseVoidFunction = async (): Promise<void> => {
|
||||
try {
|
||||
const statusCheck: PromiseVoidFunction = async (): Promise<void> => {
|
||||
return await InfrastructureStatus.checkStatusWithRetry({
|
||||
checkClickhouseStatus: true,
|
||||
checkPostgresStatus: true,
|
||||
checkRedisStatus: true,
|
||||
retryCount: 3,
|
||||
});
|
||||
};
|
||||
|
||||
// Initialize telemetry
|
||||
Telemetry.init({
|
||||
serviceName: APP_NAME,
|
||||
});
|
||||
|
||||
logger.info(
|
||||
`ProbeIngest Service - Queue concurrency: ${PROBE_INGEST_CONCURRENCY}`,
|
||||
);
|
||||
|
||||
// init the app
|
||||
await App.init({
|
||||
appName: APP_NAME,
|
||||
statusOptions: {
|
||||
liveCheck: statusCheck,
|
||||
readyCheck: statusCheck,
|
||||
},
|
||||
});
|
||||
|
||||
// connect to the database.
|
||||
await PostgresAppInstance.connect();
|
||||
|
||||
// connect redis
|
||||
await Redis.connect();
|
||||
|
||||
await ClickhouseAppInstance.connect(
|
||||
ClickhouseAppInstance.getDatasourceOptions(),
|
||||
);
|
||||
|
||||
await Realtime.init();
|
||||
|
||||
// add default routes
|
||||
await App.addDefaultRoutes();
|
||||
} catch (err) {
|
||||
logger.error("App Init Failed:");
|
||||
logger.error(err);
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
init().catch((err: Error) => {
|
||||
logger.error(err);
|
||||
logger.error("Exiting node process");
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -1,151 +0,0 @@
|
||||
import Queue, { QueueName } from "Common/Server/Infrastructure/Queue";
|
||||
import { JSONObject } from "Common/Types/JSON";
|
||||
import OneUptimeDate from "Common/Types/Date";
|
||||
import logger from "Common/Server/Utils/Logger";
|
||||
import Dictionary from "Common/Types/Dictionary";
|
||||
|
||||
export type ProbeIngestJobType =
|
||||
| "probe-response"
|
||||
| "monitor-test"
|
||||
| "incoming-email";
|
||||
|
||||
export interface ProbeIngestJobData {
|
||||
jobType: ProbeIngestJobType;
|
||||
ingestionTimestamp: Date;
|
||||
// For probe-response and monitor-test
|
||||
probeMonitorResponse?: JSONObject | undefined;
|
||||
testId?: string | undefined;
|
||||
// For incoming-email
|
||||
incomingEmail?: IncomingEmailJobData | undefined;
|
||||
}
|
||||
|
||||
export interface IncomingEmailJobData {
|
||||
secretKey: string;
|
||||
emailFrom: string;
|
||||
emailTo: string;
|
||||
emailSubject: string;
|
||||
emailBody: string;
|
||||
emailBodyHtml: string | undefined;
|
||||
emailHeaders: Dictionary<string> | undefined;
|
||||
attachments:
|
||||
| Array<{
|
||||
filename: string;
|
||||
contentType: string;
|
||||
size: number;
|
||||
}>
|
||||
| undefined;
|
||||
}
|
||||
|
||||
export default class ProbeIngestQueueService {
|
||||
public static async addProbeIngestJob(data: {
|
||||
probeMonitorResponse: JSONObject;
|
||||
jobType: "probe-response" | "monitor-test";
|
||||
testId?: string;
|
||||
}): Promise<void> {
|
||||
try {
|
||||
const jobData: ProbeIngestJobData = {
|
||||
probeMonitorResponse: data.probeMonitorResponse,
|
||||
jobType: data.jobType,
|
||||
testId: data.testId,
|
||||
ingestionTimestamp: OneUptimeDate.getCurrentDate(),
|
||||
};
|
||||
|
||||
const jobId: string = `probe-${data.jobType}-${data.testId || "general"}-${OneUptimeDate.getCurrentDateAsUnixNano()}`;
|
||||
|
||||
await Queue.addJob(
|
||||
QueueName.ProbeIngest,
|
||||
jobId,
|
||||
"ProcessProbeIngest",
|
||||
jobData as unknown as JSONObject,
|
||||
);
|
||||
|
||||
logger.debug(`Added probe ingestion job: ${jobId}`);
|
||||
} catch (error) {
|
||||
logger.error(`Error adding probe ingestion job:`);
|
||||
logger.error(error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
public static async addIncomingEmailJob(data: {
|
||||
secretKey: string;
|
||||
emailFrom: string;
|
||||
emailTo: string;
|
||||
emailSubject: string;
|
||||
emailBody: string;
|
||||
emailBodyHtml?: string | undefined;
|
||||
emailHeaders?: Dictionary<string> | undefined;
|
||||
attachments?:
|
||||
| Array<{
|
||||
filename: string;
|
||||
contentType: string;
|
||||
size: number;
|
||||
}>
|
||||
| undefined;
|
||||
}): Promise<void> {
|
||||
try {
|
||||
const jobData: ProbeIngestJobData = {
|
||||
jobType: "incoming-email",
|
||||
ingestionTimestamp: OneUptimeDate.getCurrentDate(),
|
||||
incomingEmail: {
|
||||
secretKey: data.secretKey,
|
||||
emailFrom: data.emailFrom,
|
||||
emailTo: data.emailTo,
|
||||
emailSubject: data.emailSubject,
|
||||
emailBody: data.emailBody,
|
||||
emailBodyHtml: data.emailBodyHtml,
|
||||
emailHeaders: data.emailHeaders,
|
||||
attachments: data.attachments,
|
||||
},
|
||||
};
|
||||
|
||||
const jobId: string = `incoming-email-${data.secretKey}-${OneUptimeDate.getCurrentDateAsUnixNano()}`;
|
||||
|
||||
await Queue.addJob(
|
||||
QueueName.ProbeIngest,
|
||||
jobId,
|
||||
"ProcessProbeIngest",
|
||||
jobData as unknown as JSONObject,
|
||||
);
|
||||
|
||||
logger.debug(`Added incoming email ingestion job: ${jobId}`);
|
||||
} catch (error) {
|
||||
logger.error(`Error adding incoming email ingestion job:`);
|
||||
logger.error(error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
public static async getQueueSize(): Promise<number> {
|
||||
return Queue.getQueueSize(QueueName.ProbeIngest);
|
||||
}
|
||||
|
||||
public static async getQueueStats(): Promise<{
|
||||
waiting: number;
|
||||
active: number;
|
||||
completed: number;
|
||||
failed: number;
|
||||
delayed: number;
|
||||
total: number;
|
||||
}> {
|
||||
return Queue.getQueueStats(QueueName.ProbeIngest);
|
||||
}
|
||||
|
||||
public static getFailedJobs(options?: {
|
||||
start?: number;
|
||||
end?: number;
|
||||
}): Promise<
|
||||
Array<{
|
||||
id: string;
|
||||
name: string;
|
||||
data: JSONObject;
|
||||
failedReason: string;
|
||||
stackTrace?: string;
|
||||
processedOn: Date | null;
|
||||
finishedOn: Date | null;
|
||||
attemptsMade: number;
|
||||
}>
|
||||
> {
|
||||
return Queue.getFailedJobs(QueueName.ProbeIngest, options);
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
{
|
||||
|
||||
"preset": "ts-jest",
|
||||
"testPathIgnorePatterns": [
|
||||
"node_modules",
|
||||
"dist"
|
||||
],
|
||||
"verbose": true,
|
||||
"globals": {
|
||||
"ts-jest": {
|
||||
"tsconfig": "tsconfig.json",
|
||||
"babelConfig": false
|
||||
}
|
||||
},
|
||||
"moduleFileExtensions": ["ts", "js", "json"],
|
||||
"transform": {
|
||||
".(ts|tsx)": "ts-jest"
|
||||
},
|
||||
"testEnvironment": "node",
|
||||
"collectCoverage": false,
|
||||
"coverageReporters": ["text", "lcov"],
|
||||
"testRegex": "./Tests/(.*).test.ts",
|
||||
"collectCoverageFrom": ["./**/*.(tsx||ts)"],
|
||||
"coverageThreshold": {
|
||||
"global": {
|
||||
"lines": 0,
|
||||
"functions": 0,
|
||||
"branches": 0,
|
||||
"statements": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
{
|
||||
"watch": ["./","../Common/Server", "../Common/Types", "../Common/Utils", "../Common/Models"],
|
||||
"ext": "ts,tsx",
|
||||
"ignore": ["./node_modules/**", "./public/**", "./bin/**", "./build/**"],
|
||||
"watchOptions": {"useFsEvents": false, "interval": 500},
|
||||
"env": {"TS_NODE_TRANSPILE_ONLY": "1", "TS_NODE_FILES": "false"},
|
||||
"exec": "node -r ts-node/register/transpile-only Index.ts"
|
||||
}
|
||||
4653
ProbeIngest/package-lock.json
generated
4653
ProbeIngest/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"name": "@oneuptime/probe-ingest",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/OneUptime/oneuptime"
|
||||
},
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"start": "export NODE_OPTIONS='--max-old-space-size=8096' && node --require ts-node/register Index.ts",
|
||||
"compile": "tsc",
|
||||
"clear-modules": "rm -rf node_modules && rm package-lock.json && npm install",
|
||||
"dev": "npx nodemon",
|
||||
"audit": "npm audit --audit-level=low",
|
||||
"dep-check": "npm install -g depcheck && depcheck ./ --skip-missing=true",
|
||||
"test": "jest --passWithNoTests"
|
||||
},
|
||||
"author": "OneUptime <hello@oneuptime.com> (https://oneuptime.com/)",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"Common": "file:../Common",
|
||||
"ejs": "^3.1.10",
|
||||
"ts-node": "^10.9.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/jest": "^27.5.0",
|
||||
"@types/node": "^17.0.31",
|
||||
"jest": "^28.1.0",
|
||||
"nodemon": "^2.0.20",
|
||||
"ts-jest": "^28.0.2"
|
||||
}
|
||||
}
|
||||
@@ -1,115 +0,0 @@
|
||||
{
|
||||
"ts-node": {
|
||||
// these options are overrides used only by ts-node
|
||||
// same as the --compilerOptions flag and the TS_NODE_COMPILER_OPTIONS environment variable
|
||||
"compilerOptions": {
|
||||
"module": "commonjs",
|
||||
"resolveJsonModule": true,
|
||||
}
|
||||
},
|
||||
"compilerOptions": {
|
||||
/* Visit https://aka.ms/tsconfig.json to read more about this file */
|
||||
|
||||
/* Projects */
|
||||
// "incremental": true, /* Enable incremental compilation */
|
||||
// "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */
|
||||
// "tsBuildInfoFile": "./", /* Specify the folder for .tsbuildinfo incremental compilation files. */
|
||||
// "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects */
|
||||
// "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */
|
||||
// "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */
|
||||
|
||||
/* Language and Environment */
|
||||
"target": "es2017" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
|
||||
// "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */
|
||||
"jsx": "react" /* Specify what JSX code is generated. */,
|
||||
"experimentalDecorators": true /* Enable experimental support for TC39 stage 2 draft decorators. */,
|
||||
"emitDecoratorMetadata": true /* Emit design-type metadata for decorated declarations in source files. */,
|
||||
// "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h' */
|
||||
// "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
|
||||
// "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using `jsx: react-jsx*`.` */
|
||||
// "reactNamespace": "", /* Specify the object invoked for `createElement`. This only applies when targeting `react` JSX emit. */
|
||||
// "noLib": true, /* Disable including any library files, including the default lib.d.ts. */
|
||||
// "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */
|
||||
|
||||
/* Modules */
|
||||
// "module": "es2022" /* Specify what module code is generated. */,
|
||||
"rootDir": "" /* Specify the root folder within your source files. */,
|
||||
"moduleResolution": "node" /* Specify how TypeScript looks up a file from a given module specifier. */,
|
||||
// "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */
|
||||
// "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */
|
||||
// "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */
|
||||
"typeRoots": [
|
||||
"node_modules/@types"
|
||||
] /* Specify multiple folders that act like `./node_modules/@types`. */,
|
||||
"types": [
|
||||
"node",
|
||||
"jest"
|
||||
] /* Specify type package names to be included without being referenced in a source file. */,
|
||||
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
|
||||
// "resolveJsonModule": true, /* Enable importing .json files */
|
||||
// "noResolve": true, /* Disallow `import`s, `require`s or `<reference>`s from expanding the number of files TypeScript should add to a project. */
|
||||
|
||||
/* JavaScript Support */
|
||||
// "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */
|
||||
// "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */
|
||||
// "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from `node_modules`. Only applicable with `allowJs`. */
|
||||
|
||||
/* Emit */
|
||||
// "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
|
||||
// "declarationMap": true, /* Create sourcemaps for d.ts files. */
|
||||
// "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */
|
||||
"sourceMap": true /* Create source map files for emitted JavaScript files. */,
|
||||
// "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If `declaration` is true, also designates a file that bundles all .d.ts output. */
|
||||
"outDir": "build/dist" /* Specify an output folder for all emitted files. */,
|
||||
// "removeComments": true, /* Disable emitting comments. */
|
||||
// "noEmit": true, /* Disable emitting files from a compilation. */
|
||||
// "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
|
||||
// "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types */
|
||||
// "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
|
||||
// "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */
|
||||
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
|
||||
// "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */
|
||||
// "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */
|
||||
// "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
|
||||
// "newLine": "crlf", /* Set the newline character for emitting files. */
|
||||
// "stripInternal": true, /* Disable emitting declarations that have `@internal` in their JSDoc comments. */
|
||||
// "noEmitHelpers": true, /* Disable generating custom helper functions like `__extends` in compiled output. */
|
||||
// "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */
|
||||
// "preserveConstEnums": true, /* Disable erasing `const enum` declarations in generated code. */
|
||||
// "declarationDir": "./", /* Specify the output directory for generated declaration files. */
|
||||
// "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */
|
||||
|
||||
/* Interop Constraints */
|
||||
// "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */
|
||||
// "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */
|
||||
"esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables `allowSyntheticDefaultImports` for type compatibility. */,
|
||||
// "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
|
||||
"forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
|
||||
|
||||
/* Type Checking */
|
||||
"strict": true /* Enable all strict type-checking options. */,
|
||||
"noImplicitAny": true /* Enable error reporting for expressions and declarations with an implied `any` type.. */,
|
||||
"strictNullChecks": true /* When type checking, take into account `null` and `undefined`. */,
|
||||
"strictFunctionTypes": true /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */,
|
||||
"strictBindCallApply": true /* Check that the arguments for `bind`, `call`, and `apply` methods match the original function. */,
|
||||
"strictPropertyInitialization": true /* Check for class properties that are declared but not set in the constructor. */,
|
||||
"noImplicitThis": true /* Enable error reporting when `this` is given the type `any`. */,
|
||||
"useUnknownInCatchVariables": true /* Type catch clause variables as 'unknown' instead of 'any'. */,
|
||||
"alwaysStrict": true /* Ensure 'use strict' is always emitted. */,
|
||||
"noUnusedLocals": true /* Enable error reporting when a local variables aren't read. */,
|
||||
"noUnusedParameters": true /* Raise an error when a function parameter isn't read */,
|
||||
"exactOptionalPropertyTypes": true /* Interpret optional property types as written, rather than adding 'undefined'. */,
|
||||
"noImplicitReturns": true /* Enable error reporting for codepaths that do not explicitly return in a function. */,
|
||||
"noFallthroughCasesInSwitch": true /* Enable error reporting for fallthrough cases in switch statements. */,
|
||||
"noUncheckedIndexedAccess": true /* Include 'undefined' in index signature results */,
|
||||
"noImplicitOverride": true /* Ensure overriding members in derived classes are marked with an override modifier. */,
|
||||
"noPropertyAccessFromIndexSignature": true /* Enforces using indexed accessors for keys declared using an indexed type */,
|
||||
// "allowUnusedLabels": true, /* Disable error reporting for unused labels. */
|
||||
// "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */
|
||||
|
||||
/* Completeness */
|
||||
// "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */
|
||||
"skipLibCheck": true, /* Skip type checking all .d.ts files. */
|
||||
"resolveJsonModule": true
|
||||
}
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
.git
|
||||
|
||||
node_modules
|
||||
# See https://help.github.com/ignore-files/ for more about ignoring files.
|
||||
|
||||
# dependencies
|
||||
/node_modules
|
||||
node_modules
|
||||
|
||||
.idea
|
||||
# testing
|
||||
/coverage
|
||||
|
||||
# production
|
||||
/build
|
||||
|
||||
# misc
|
||||
.DS_Store
|
||||
|
||||
env.js
|
||||
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
|
||||
yarn.lock
|
||||
Untitled-1
|
||||
*.local.sh
|
||||
*.local.yaml
|
||||
run
|
||||
stop
|
||||
|
||||
nohup.out*
|
||||
|
||||
encrypted-credentials.tar
|
||||
encrypted-credentials/
|
||||
|
||||
_README.md
|
||||
|
||||
# Important Add production values to gitignore.
|
||||
values-saas-production.yaml
|
||||
kubernetes/values-saas-production.yaml
|
||||
|
||||
/private
|
||||
|
||||
/tls_cert.pem
|
||||
/tls_key.pem
|
||||
/keys
|
||||
|
||||
temp_readme.md
|
||||
|
||||
tests/coverage
|
||||
|
||||
settings.json
|
||||
|
||||
GoSDK/tester/
|
||||
1
ServerMonitorIngest/.gitattributes
vendored
1
ServerMonitorIngest/.gitattributes
vendored
@@ -1 +0,0 @@
|
||||
*.js text eol=lf
|
||||
16
ServerMonitorIngest/.gitignore
vendored
16
ServerMonitorIngest/.gitignore
vendored
@@ -1,16 +0,0 @@
|
||||
# See https://help.github.com/ignore-files/ for more about ignoring files.
|
||||
|
||||
# dependencies
|
||||
#/backend/node_modules
|
||||
/kubernetes
|
||||
/node_modules
|
||||
.idea
|
||||
|
||||
# misc
|
||||
.DS_Store
|
||||
|
||||
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
yarn.lock
|
||||
@@ -1,38 +0,0 @@
|
||||
import Express, {
|
||||
ExpressRequest,
|
||||
ExpressResponse,
|
||||
ExpressRouter,
|
||||
NextFunction,
|
||||
} from "Common/Server/Utils/Express";
|
||||
import ServerMonitorIngestQueueService from "../Services/Queue/ServerMonitorIngestQueueService";
|
||||
// import ClusterKeyAuthorization from "Common/Server/Middleware/ClusterKeyAuthorization";
|
||||
|
||||
const router: ExpressRouter = Express.getRouter();
|
||||
|
||||
/**
|
||||
* JSON metrics endpoint for KEDA autoscaling
|
||||
* Returns queue size as JSON for KEDA metrics-api scaler
|
||||
*/
|
||||
router.get(
|
||||
"/metrics/queue-size",
|
||||
// ClusterKeyAuthorization.isAuthorizedServiceMiddleware, // Temporarily disabled for KEDA debugging
|
||||
async (
|
||||
_req: ExpressRequest,
|
||||
res: ExpressResponse,
|
||||
next: NextFunction,
|
||||
): Promise<void> => {
|
||||
try {
|
||||
const queueSize: number =
|
||||
await ServerMonitorIngestQueueService.getQueueSize();
|
||||
|
||||
res.setHeader("Content-Type", "application/json");
|
||||
res.status(200).json({
|
||||
queueSize: queueSize,
|
||||
});
|
||||
} catch (err) {
|
||||
return next(err);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
export default router;
|
||||
@@ -1,9 +0,0 @@
|
||||
let concurrency: string | number =
|
||||
process.env["SERVER_MONITOR_INGEST_CONCURRENCY"] || 100;
|
||||
|
||||
if (typeof concurrency === "string") {
|
||||
const parsed: number = parseInt(concurrency, 10);
|
||||
concurrency = !isNaN(parsed) && parsed > 0 ? parsed : 100;
|
||||
}
|
||||
|
||||
export const SERVER_MONITOR_INGEST_CONCURRENCY: number = concurrency as number;
|
||||
@@ -1,83 +0,0 @@
|
||||
#
|
||||
# OneUptime-ServerMonitorIngest Dockerfile
|
||||
#
|
||||
|
||||
# Pull base image nodejs image.
|
||||
FROM public.ecr.aws/docker/library/node:23.8
|
||||
RUN mkdir /tmp/npm && chmod 2777 /tmp/npm && chown 1000:1000 /tmp/npm && npm config set cache /tmp/npm --global
|
||||
|
||||
RUN npm config set fetch-retries 5
|
||||
RUN npm config set fetch-retry-mintimeout 20000
|
||||
RUN npm config set fetch-retry-maxtimeout 60000
|
||||
|
||||
|
||||
ARG GIT_SHA
|
||||
ARG APP_VERSION
|
||||
ARG IS_ENTERPRISE_EDITION=false
|
||||
|
||||
ENV GIT_SHA=${GIT_SHA}
|
||||
ENV APP_VERSION=${APP_VERSION}
|
||||
ENV IS_ENTERPRISE_EDITION=${IS_ENTERPRISE_EDITION}
|
||||
ENV PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1
|
||||
ENV NODE_OPTIONS="--use-openssl-ca"
|
||||
|
||||
## Add Intermediate Certs
|
||||
COPY ./SslCertificates /usr/local/share/ca-certificates
|
||||
RUN update-ca-certificates
|
||||
|
||||
|
||||
# IF APP_VERSION is not set, set it to 1.0.0
|
||||
RUN if [ -z "$APP_VERSION" ]; then export APP_VERSION=1.0.0; fi
|
||||
|
||||
|
||||
RUN apt-get update
|
||||
|
||||
# Install bash.
|
||||
RUN apt-get install bash -y && apt-get install curl -y && apt-get install iputils-ping -y
|
||||
|
||||
# Install python
|
||||
RUN apt-get update && apt-get install -y .gyp python3 make g++
|
||||
|
||||
#Use bash shell by default
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
|
||||
# Install iputils
|
||||
RUN apt-get install net-tools -y
|
||||
|
||||
RUN mkdir -p /usr/src
|
||||
|
||||
WORKDIR /usr/src/Common
|
||||
COPY ./Common/package*.json /usr/src/Common/
|
||||
# Set version in ./Common/package.json to the APP_VERSION
|
||||
RUN sed -i "s/\"version\": \".*\"/\"version\": \"$APP_VERSION\"/g" /usr/src/Common/package.json
|
||||
RUN npm install
|
||||
COPY ./Common /usr/src/Common
|
||||
|
||||
|
||||
|
||||
ENV PRODUCTION=true
|
||||
|
||||
WORKDIR /usr/src/app
|
||||
|
||||
# Install app dependencies
|
||||
COPY ./ServerMonitorIngest/package*.json /usr/src/app/
|
||||
RUN npm install
|
||||
|
||||
# Expose ports.
|
||||
# - 3404: OneUptime-server-monitor-ingest
|
||||
EXPOSE 3404
|
||||
|
||||
{{ if eq .Env.ENVIRONMENT "development" }}
|
||||
#Run the app
|
||||
CMD [ "npm", "run", "dev" ]
|
||||
{{ else }}
|
||||
# Copy app source
|
||||
COPY ./ServerMonitorIngest /usr/src/app
|
||||
# Bundle app source
|
||||
RUN npm run compile
|
||||
# Set permission to write logs and cache in case container run as non root
|
||||
RUN chown -R 1000:1000 "/tmp/npm" && chmod -R 2777 "/tmp/npm"
|
||||
#Run the app
|
||||
CMD [ "npm", "start" ]
|
||||
{{ end }}
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
import ServerMonitorAPI from "./API/ServerMonitor";
|
||||
import MetricsAPI from "./API/Metrics";
|
||||
import { PromiseVoidFunction } from "Common/Types/FunctionTypes";
|
||||
import { ClickhouseAppInstance } from "Common/Server/Infrastructure/ClickhouseDatabase";
|
||||
import PostgresAppInstance from "Common/Server/Infrastructure/PostgresDatabase";
|
||||
import Redis from "Common/Server/Infrastructure/Redis";
|
||||
import InfrastructureStatus from "Common/Server/Infrastructure/Status";
|
||||
import Express, { ExpressApplication } from "Common/Server/Utils/Express";
|
||||
import logger from "Common/Server/Utils/Logger";
|
||||
import Realtime from "Common/Server/Utils/Realtime";
|
||||
import App from "Common/Server/Utils/StartServer";
|
||||
import Telemetry from "Common/Server/Utils/Telemetry";
|
||||
import "./Jobs/ServerMonitorIngest/ProcessServerMonitorIngest";
|
||||
import { SERVER_MONITOR_INGEST_CONCURRENCY } from "./Config";
|
||||
|
||||
const app: ExpressApplication = Express.getExpressApp();
|
||||
|
||||
const APP_NAME: string = "server-monitor-ingest";
|
||||
|
||||
app.use([`/${APP_NAME}`, "/"], ServerMonitorAPI);
|
||||
app.use([`/${APP_NAME}`, "/"], MetricsAPI);
|
||||
|
||||
const init: PromiseVoidFunction = async (): Promise<void> => {
|
||||
try {
|
||||
const statusCheck: PromiseVoidFunction = async (): Promise<void> => {
|
||||
return await InfrastructureStatus.checkStatusWithRetry({
|
||||
checkClickhouseStatus: true,
|
||||
checkPostgresStatus: true,
|
||||
checkRedisStatus: true,
|
||||
retryCount: 3,
|
||||
});
|
||||
};
|
||||
|
||||
// Initialize telemetry
|
||||
Telemetry.init({
|
||||
serviceName: APP_NAME,
|
||||
});
|
||||
|
||||
logger.info(
|
||||
`ServerMonitorIngest Service - Queue concurrency: ${SERVER_MONITOR_INGEST_CONCURRENCY}`,
|
||||
);
|
||||
|
||||
// init the app
|
||||
await App.init({
|
||||
appName: APP_NAME,
|
||||
statusOptions: {
|
||||
liveCheck: statusCheck,
|
||||
readyCheck: statusCheck,
|
||||
},
|
||||
});
|
||||
|
||||
// connect to the database.
|
||||
await PostgresAppInstance.connect();
|
||||
|
||||
// connect redis
|
||||
await Redis.connect();
|
||||
|
||||
await ClickhouseAppInstance.connect(
|
||||
ClickhouseAppInstance.getDatasourceOptions(),
|
||||
);
|
||||
|
||||
await Realtime.init();
|
||||
|
||||
// add default routes
|
||||
await App.addDefaultRoutes();
|
||||
} catch (err) {
|
||||
logger.error("App Init Failed:");
|
||||
logger.error(err);
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
init().catch((err: Error) => {
|
||||
logger.error(err);
|
||||
logger.error("Exiting node process");
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -1,73 +0,0 @@
|
||||
import Queue, { QueueName } from "Common/Server/Infrastructure/Queue";
|
||||
import { JSONObject } from "Common/Types/JSON";
|
||||
import OneUptimeDate from "Common/Types/Date";
|
||||
import logger from "Common/Server/Utils/Logger";
|
||||
|
||||
export interface ServerMonitorIngestJobData {
|
||||
secretKey: string;
|
||||
serverMonitorResponse: JSONObject;
|
||||
ingestionTimestamp: Date;
|
||||
}
|
||||
|
||||
export default class ServerMonitorIngestQueueService {
|
||||
public static async addServerMonitorIngestJob(data: {
|
||||
secretKey: string;
|
||||
serverMonitorResponse: JSONObject;
|
||||
}): Promise<void> {
|
||||
try {
|
||||
const jobData: ServerMonitorIngestJobData = {
|
||||
secretKey: data.secretKey,
|
||||
serverMonitorResponse: data.serverMonitorResponse,
|
||||
ingestionTimestamp: OneUptimeDate.getCurrentDate(),
|
||||
};
|
||||
|
||||
const jobId: string = `server-monitor-${data.secretKey}-${OneUptimeDate.getCurrentDateAsUnixNano()}`;
|
||||
|
||||
await Queue.addJob(
|
||||
QueueName.ServerMonitorIngest,
|
||||
jobId,
|
||||
"ProcessServerMonitorIngest",
|
||||
jobData as unknown as JSONObject,
|
||||
);
|
||||
|
||||
logger.debug(`Added server monitor ingestion job: ${jobId}`);
|
||||
} catch (error) {
|
||||
logger.error(`Error adding server monitor ingestion job:`);
|
||||
logger.error(error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
public static async getQueueSize(): Promise<number> {
|
||||
return Queue.getQueueSize(QueueName.ServerMonitorIngest);
|
||||
}
|
||||
|
||||
public static async getQueueStats(): Promise<{
|
||||
waiting: number;
|
||||
active: number;
|
||||
completed: number;
|
||||
failed: number;
|
||||
delayed: number;
|
||||
total: number;
|
||||
}> {
|
||||
return Queue.getQueueStats(QueueName.ServerMonitorIngest);
|
||||
}
|
||||
|
||||
public static getFailedJobs(options?: {
|
||||
start?: number;
|
||||
end?: number;
|
||||
}): Promise<
|
||||
Array<{
|
||||
id: string;
|
||||
name: string;
|
||||
data: JSONObject;
|
||||
failedReason: string;
|
||||
stackTrace?: string;
|
||||
processedOn: Date | null;
|
||||
finishedOn: Date | null;
|
||||
attemptsMade: number;
|
||||
}>
|
||||
> {
|
||||
return Queue.getFailedJobs(QueueName.ServerMonitorIngest, options);
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
{
|
||||
|
||||
"preset": "ts-jest",
|
||||
"testPathIgnorePatterns": [
|
||||
"node_modules",
|
||||
"dist"
|
||||
],
|
||||
"verbose": true,
|
||||
"globals": {
|
||||
"ts-jest": {
|
||||
"tsconfig": "tsconfig.json",
|
||||
"babelConfig": false
|
||||
}
|
||||
},
|
||||
"moduleFileExtensions": ["ts", "js", "json"],
|
||||
"transform": {
|
||||
".(ts|tsx)": "ts-jest"
|
||||
},
|
||||
"testEnvironment": "node",
|
||||
"collectCoverage": false,
|
||||
"coverageReporters": ["text", "lcov"],
|
||||
"testRegex": "./Tests/(.*).test.ts",
|
||||
"collectCoverageFrom": ["./**/*.(tsx||ts)"],
|
||||
"coverageThreshold": {
|
||||
"global": {
|
||||
"lines": 0,
|
||||
"functions": 0,
|
||||
"branches": 0,
|
||||
"statements": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
{
|
||||
"watch": ["./","../Common/Server", "../Common/Types", "../Common/Utils", "../Common/Models"],
|
||||
"ext": "ts,tsx",
|
||||
"ignore": ["./node_modules/**", "./public/**", "./bin/**", "./build/**"],
|
||||
"watchOptions": {"useFsEvents": false, "interval": 500},
|
||||
"env": {"TS_NODE_TRANSPILE_ONLY": "1", "TS_NODE_FILES": "false"},
|
||||
"exec": "node -r ts-node/register/transpile-only Index.ts"
|
||||
}
|
||||
4568
ServerMonitorIngest/package-lock.json
generated
4568
ServerMonitorIngest/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -1,32 +0,0 @@
|
||||
{
|
||||
"name": "@oneuptime/server-monitor-ingest",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/OneUptime/oneuptime"
|
||||
},
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"start": "export NODE_OPTIONS='--max-old-space-size=8096' && node --require ts-node/register Index.ts",
|
||||
"compile": "tsc",
|
||||
"clear-modules": "rm -rf node_modules && rm package-lock.json && npm install",
|
||||
"dev": "npx nodemon",
|
||||
"audit": "npm audit --audit-level=low",
|
||||
"dep-check": "npm install -g depcheck && depcheck ./ --skip-missing=true",
|
||||
"test": "jest --passWithNoTests"
|
||||
},
|
||||
"author": "OneUptime <hello@oneuptime.com> (https://oneuptime.com/)",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"Common": "file:../Common",
|
||||
"ts-node": "^10.9.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/jest": "^27.5.0",
|
||||
"@types/node": "^17.0.31",
|
||||
"jest": "^28.1.0",
|
||||
"nodemon": "^2.0.20",
|
||||
"ts-jest": "^28.0.2"
|
||||
}
|
||||
}
|
||||
@@ -1,115 +0,0 @@
|
||||
{
|
||||
"ts-node": {
|
||||
// these options are overrides used only by ts-node
|
||||
// same as the --compilerOptions flag and the TS_NODE_COMPILER_OPTIONS environment variable
|
||||
"compilerOptions": {
|
||||
"module": "commonjs",
|
||||
"resolveJsonModule": true,
|
||||
}
|
||||
},
|
||||
"compilerOptions": {
|
||||
/* Visit https://aka.ms/tsconfig.json to read more about this file */
|
||||
|
||||
/* Projects */
|
||||
// "incremental": true, /* Enable incremental compilation */
|
||||
// "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */
|
||||
// "tsBuildInfoFile": "./", /* Specify the folder for .tsbuildinfo incremental compilation files. */
|
||||
// "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects */
|
||||
// "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */
|
||||
// "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */
|
||||
|
||||
/* Language and Environment */
|
||||
"target": "es2017" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
|
||||
// "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */
|
||||
"jsx": "react" /* Specify what JSX code is generated. */,
|
||||
"experimentalDecorators": true /* Enable experimental support for TC39 stage 2 draft decorators. */,
|
||||
"emitDecoratorMetadata": true /* Emit design-type metadata for decorated declarations in source files. */,
|
||||
// "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h' */
|
||||
// "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
|
||||
// "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using `jsx: react-jsx*`.` */
|
||||
// "reactNamespace": "", /* Specify the object invoked for `createElement`. This only applies when targeting `react` JSX emit. */
|
||||
// "noLib": true, /* Disable including any library files, including the default lib.d.ts. */
|
||||
// "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */
|
||||
|
||||
/* Modules */
|
||||
// "module": "es2022" /* Specify what module code is generated. */,
|
||||
"rootDir": "" /* Specify the root folder within your source files. */,
|
||||
"moduleResolution": "node" /* Specify how TypeScript looks up a file from a given module specifier. */,
|
||||
// "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */
|
||||
// "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */
|
||||
// "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */
|
||||
"typeRoots": [
|
||||
"node_modules/@types"
|
||||
] /* Specify multiple folders that act like `./node_modules/@types`. */,
|
||||
"types": [
|
||||
"node",
|
||||
"jest"
|
||||
] /* Specify type package names to be included without being referenced in a source file. */,
|
||||
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
|
||||
// "resolveJsonModule": true, /* Enable importing .json files */
|
||||
// "noResolve": true, /* Disallow `import`s, `require`s or `<reference>`s from expanding the number of files TypeScript should add to a project. */
|
||||
|
||||
/* JavaScript Support */
|
||||
// "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */
|
||||
// "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */
|
||||
// "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from `node_modules`. Only applicable with `allowJs`. */
|
||||
|
||||
/* Emit */
|
||||
// "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
|
||||
// "declarationMap": true, /* Create sourcemaps for d.ts files. */
|
||||
// "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */
|
||||
"sourceMap": true /* Create source map files for emitted JavaScript files. */,
|
||||
// "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If `declaration` is true, also designates a file that bundles all .d.ts output. */
|
||||
"outDir": "build/dist" /* Specify an output folder for all emitted files. */,
|
||||
// "removeComments": true, /* Disable emitting comments. */
|
||||
// "noEmit": true, /* Disable emitting files from a compilation. */
|
||||
// "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
|
||||
// "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types */
|
||||
// "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
|
||||
// "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */
|
||||
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
|
||||
// "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */
|
||||
// "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */
|
||||
// "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
|
||||
// "newLine": "crlf", /* Set the newline character for emitting files. */
|
||||
// "stripInternal": true, /* Disable emitting declarations that have `@internal` in their JSDoc comments. */
|
||||
// "noEmitHelpers": true, /* Disable generating custom helper functions like `__extends` in compiled output. */
|
||||
// "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */
|
||||
// "preserveConstEnums": true, /* Disable erasing `const enum` declarations in generated code. */
|
||||
// "declarationDir": "./", /* Specify the output directory for generated declaration files. */
|
||||
// "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */
|
||||
|
||||
/* Interop Constraints */
|
||||
// "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */
|
||||
// "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */
|
||||
"esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables `allowSyntheticDefaultImports` for type compatibility. */,
|
||||
// "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
|
||||
"forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
|
||||
|
||||
/* Type Checking */
|
||||
"strict": true /* Enable all strict type-checking options. */,
|
||||
"noImplicitAny": true /* Enable error reporting for expressions and declarations with an implied `any` type.. */,
|
||||
"strictNullChecks": true /* When type checking, take into account `null` and `undefined`. */,
|
||||
"strictFunctionTypes": true /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */,
|
||||
"strictBindCallApply": true /* Check that the arguments for `bind`, `call`, and `apply` methods match the original function. */,
|
||||
"strictPropertyInitialization": true /* Check for class properties that are declared but not set in the constructor. */,
|
||||
"noImplicitThis": true /* Enable error reporting when `this` is given the type `any`. */,
|
||||
"useUnknownInCatchVariables": true /* Type catch clause variables as 'unknown' instead of 'any'. */,
|
||||
"alwaysStrict": true /* Ensure 'use strict' is always emitted. */,
|
||||
"noUnusedLocals": true /* Enable error reporting when a local variables aren't read. */,
|
||||
"noUnusedParameters": true /* Raise an error when a function parameter isn't read */,
|
||||
"exactOptionalPropertyTypes": true /* Interpret optional property types as written, rather than adding 'undefined'. */,
|
||||
"noImplicitReturns": true /* Enable error reporting for codepaths that do not explicitly return in a function. */,
|
||||
"noFallthroughCasesInSwitch": true /* Enable error reporting for fallthrough cases in switch statements. */,
|
||||
"noUncheckedIndexedAccess": true /* Include 'undefined' in index signature results */,
|
||||
"noImplicitOverride": true /* Ensure overriding members in derived classes are marked with an override modifier. */,
|
||||
"noPropertyAccessFromIndexSignature": true /* Enforces using indexed accessors for keys declared using an indexed type */,
|
||||
// "allowUnusedLabels": true, /* Disable error reporting for unused labels. */
|
||||
// "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */
|
||||
|
||||
/* Completeness */
|
||||
// "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */
|
||||
"skipLibCheck": true, /* Skip type checking all .d.ts files. */
|
||||
"resolveJsonModule": true
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,7 @@ import Express, {
|
||||
RequestHandler,
|
||||
} from "Common/Server/Utils/Express";
|
||||
import Response from "Common/Server/Utils/Response";
|
||||
import IncomingRequestIngestQueueService from "../Services/Queue/IncomingRequestIngestQueueService";
|
||||
import TelemetryQueueService from "../../Services/Queue/TelemetryQueueService";
|
||||
import ClusterKeyAuthorization from "Common/Server/Middleware/ClusterKeyAuthorization";
|
||||
|
||||
const router: ExpressRouter = Express.getRouter();
|
||||
@@ -35,7 +35,7 @@ const processIncomingRequest: RequestHandler = async (
|
||||
Response.sendEmptySuccessResponse(req, res);
|
||||
|
||||
// Add to queue for asynchronous processing
|
||||
await IncomingRequestIngestQueueService.addIncomingRequestIngestJob({
|
||||
await TelemetryQueueService.addIncomingRequestIngestJob({
|
||||
secretKey: monitorSecretKeyAsString,
|
||||
requestHeaders: requestHeaders,
|
||||
requestBody: requestBody,
|
||||
@@ -87,7 +87,7 @@ router.get(
|
||||
failed: number;
|
||||
delayed: number;
|
||||
total: number;
|
||||
} = await IncomingRequestIngestQueueService.getQueueStats();
|
||||
} = await TelemetryQueueService.getQueueStats();
|
||||
return Response.sendJsonObjectResponse(req, res, stats);
|
||||
} catch (err) {
|
||||
return next(err);
|
||||
@@ -105,8 +105,7 @@ router.get(
|
||||
next: NextFunction,
|
||||
): Promise<void> => {
|
||||
try {
|
||||
const size: number =
|
||||
await IncomingRequestIngestQueueService.getQueueSize();
|
||||
const size: number = await TelemetryQueueService.getQueueSize();
|
||||
return Response.sendJsonObjectResponse(req, res, { size });
|
||||
} catch (err) {
|
||||
return next(err);
|
||||
@@ -137,7 +136,7 @@ router.get(
|
||||
processedOn: Date | null;
|
||||
finishedOn: Date | null;
|
||||
attemptsMade: number;
|
||||
}> = await IncomingRequestIngestQueueService.getFailedJobs({
|
||||
}> = await TelemetryQueueService.getFailedJobs({
|
||||
start,
|
||||
end,
|
||||
});
|
||||
@@ -11,7 +11,7 @@ import InboundEmailProvider, {
|
||||
ParsedInboundEmail,
|
||||
} from "Common/Server/Services/InboundEmail/InboundEmailProvider";
|
||||
import { JSONObject } from "Common/Types/JSON";
|
||||
import ProbeIngestQueueService from "../Services/Queue/ProbeIngestQueueService";
|
||||
import TelemetryQueueService from "../../Services/Queue/TelemetryQueueService";
|
||||
import MultipartFormDataMiddleware from "Common/Server/Middleware/MultipartFormData";
|
||||
|
||||
const router: ExpressRouter = Express.getRouter();
|
||||
@@ -84,8 +84,8 @@ router.post(
|
||||
|
||||
logger.debug(`Extracted secret key: ${secretKey}`);
|
||||
|
||||
// Queue the email for async processing using the unified ProbeIngest queue
|
||||
await ProbeIngestQueueService.addIncomingEmailJob({
|
||||
// Queue the email for async processing using the unified Telemetry queue
|
||||
await TelemetryQueueService.addIncomingEmailJob({
|
||||
secretKey: secretKey,
|
||||
emailFrom: parsedEmail.from,
|
||||
emailTo: parsedEmail.to,
|
||||
@@ -1,6 +1,6 @@
|
||||
import ProbeAuthorization from "../Middleware/ProbeAuthorization";
|
||||
import { ProbeExpressRequest } from "../Types/Request";
|
||||
import MonitorUtil from "../Utils/Monitor";
|
||||
import ProbeAuthorization from "../../Middleware/ProbeAuthorization";
|
||||
import { ProbeExpressRequest } from "../../Types/Request";
|
||||
import MonitorUtil from "../../Utils/Monitor";
|
||||
import BaseModel from "Common/Models/DatabaseModels/DatabaseBaseModel/DatabaseBaseModel";
|
||||
import SortOrder from "Common/Types/BaseDatabase/SortOrder";
|
||||
import LIMIT_MAX from "Common/Types/Database/LimitMax";
|
||||
@@ -1,4 +1,4 @@
|
||||
import ProbeAuthorization from "../Middleware/ProbeAuthorization";
|
||||
import ProbeAuthorization from "../../Middleware/ProbeAuthorization";
|
||||
import Email from "Common/Types/Email";
|
||||
import EmailTemplateType from "Common/Types/Email/EmailTemplateType";
|
||||
import BadDataException from "Common/Types/Exception/BadDataException";
|
||||
@@ -22,7 +22,7 @@ import Response from "Common/Server/Utils/Response";
|
||||
import GlobalConfig from "Common/Models/DatabaseModels/GlobalConfig";
|
||||
import Probe from "Common/Models/DatabaseModels/Probe";
|
||||
import User from "Common/Models/DatabaseModels/User";
|
||||
import ProbeIngestQueueService from "../Services/Queue/ProbeIngestQueueService";
|
||||
import TelemetryQueueService from "../../Services/Queue/TelemetryQueueService";
|
||||
import ClusterKeyAuthorization from "Common/Server/Middleware/ClusterKeyAuthorization";
|
||||
import PositiveNumber from "Common/Types/PositiveNumber";
|
||||
import MonitorProbeService from "Common/Server/Services/MonitorProbeService";
|
||||
@@ -293,7 +293,7 @@ router.post(
|
||||
});
|
||||
|
||||
// Add to queue for asynchronous processing
|
||||
await ProbeIngestQueueService.addProbeIngestJob({
|
||||
await TelemetryQueueService.addProbeIngestJob({
|
||||
probeMonitorResponse: req.body,
|
||||
jobType: "probe-response",
|
||||
});
|
||||
@@ -340,7 +340,7 @@ router.post(
|
||||
Response.sendEmptySuccessResponse(req, res);
|
||||
|
||||
// Add to queue for asynchronous processing
|
||||
await ProbeIngestQueueService.addProbeIngestJob({
|
||||
await TelemetryQueueService.addProbeIngestJob({
|
||||
probeMonitorResponse: req.body,
|
||||
jobType: "monitor-test",
|
||||
testId: testId.toString(),
|
||||
@@ -370,7 +370,7 @@ router.get(
|
||||
failed: number;
|
||||
delayed: number;
|
||||
total: number;
|
||||
} = await ProbeIngestQueueService.getQueueStats();
|
||||
} = await TelemetryQueueService.getQueueStats();
|
||||
return Response.sendJsonObjectResponse(req, res, stats);
|
||||
} catch (err) {
|
||||
return next(err);
|
||||
@@ -388,7 +388,7 @@ router.get(
|
||||
next: NextFunction,
|
||||
): Promise<void> => {
|
||||
try {
|
||||
const size: number = await ProbeIngestQueueService.getQueueSize();
|
||||
const size: number = await TelemetryQueueService.getQueueSize();
|
||||
return Response.sendJsonObjectResponse(req, res, { size });
|
||||
} catch (err) {
|
||||
return next(err);
|
||||
@@ -475,7 +475,7 @@ router.get(
|
||||
processedOn: Date | null;
|
||||
finishedOn: Date | null;
|
||||
attemptsMade: number;
|
||||
}> = await ProbeIngestQueueService.getFailedJobs({
|
||||
}> = await TelemetryQueueService.getFailedJobs({
|
||||
start,
|
||||
end,
|
||||
});
|
||||
@@ -13,7 +13,7 @@ import Express, {
|
||||
import Response from "Common/Server/Utils/Response";
|
||||
import Monitor from "Common/Models/DatabaseModels/Monitor";
|
||||
import ProjectService from "Common/Server/Services/ProjectService";
|
||||
import ServerMonitorIngestQueueService from "../Services/Queue/ServerMonitorIngestQueueService";
|
||||
import TelemetryQueueService from "../../Services/Queue/TelemetryQueueService";
|
||||
import ClusterKeyAuthorization from "Common/Server/Middleware/ClusterKeyAuthorization";
|
||||
|
||||
const router: ExpressRouter = Express.getRouter();
|
||||
@@ -80,7 +80,7 @@ router.post(
|
||||
Response.sendEmptySuccessResponse(req, res);
|
||||
|
||||
// Add to queue for asynchronous processing
|
||||
await ServerMonitorIngestQueueService.addServerMonitorIngestJob({
|
||||
await TelemetryQueueService.addServerMonitorIngestJob({
|
||||
secretKey: monitorSecretKeyAsString,
|
||||
serverMonitorResponse: req.body as JSONObject,
|
||||
});
|
||||
@@ -109,7 +109,7 @@ router.get(
|
||||
failed: number;
|
||||
delayed: number;
|
||||
total: number;
|
||||
} = await ServerMonitorIngestQueueService.getQueueStats();
|
||||
} = await TelemetryQueueService.getQueueStats();
|
||||
return Response.sendJsonObjectResponse(req, res, stats);
|
||||
} catch (err) {
|
||||
return next(err);
|
||||
@@ -127,7 +127,7 @@ router.get(
|
||||
next: NextFunction,
|
||||
): Promise<void> => {
|
||||
try {
|
||||
const size: number = await ServerMonitorIngestQueueService.getQueueSize();
|
||||
const size: number = await TelemetryQueueService.getQueueSize();
|
||||
return Response.sendJsonObjectResponse(req, res, { size });
|
||||
} catch (err) {
|
||||
return next(err);
|
||||
@@ -158,7 +158,7 @@ router.get(
|
||||
processedOn: Date | null;
|
||||
finishedOn: Date | null;
|
||||
attemptsMade: number;
|
||||
}> = await ServerMonitorIngestQueueService.getFailedJobs({
|
||||
}> = await TelemetryQueueService.getFailedJobs({
|
||||
start,
|
||||
end,
|
||||
});
|
||||
@@ -2,6 +2,15 @@ import OTelIngestAPI from "./API/OTelIngest";
|
||||
import MetricsAPI from "./API/Metrics";
|
||||
import SyslogAPI from "./API/Syslog";
|
||||
import FluentAPI from "./API/Fluent";
|
||||
// ProbeIngest routes
|
||||
import ProbeIngestRegisterAPI from "./API/ProbeIngest/Register";
|
||||
import ProbeIngestMonitorAPI from "./API/ProbeIngest/Monitor";
|
||||
import ProbeIngestAPI from "./API/ProbeIngest/Probe";
|
||||
import IncomingEmailAPI from "./API/ProbeIngest/IncomingEmail";
|
||||
// ServerMonitorIngest routes
|
||||
import ServerMonitorAPI from "./API/ServerMonitorIngest/ServerMonitor";
|
||||
// IncomingRequestIngest routes
|
||||
import IncomingRequestAPI from "./API/IncomingRequestIngest/IncomingRequest";
|
||||
import { PromiseVoidFunction } from "Common/Types/FunctionTypes";
|
||||
import { ClickhouseAppInstance } from "Common/Server/Infrastructure/ClickhouseDatabase";
|
||||
import PostgresAppInstance from "Common/Server/Infrastructure/PostgresDatabase";
|
||||
@@ -20,12 +29,29 @@ import "ejs";
|
||||
const app: ExpressApplication = Express.getExpressApp();
|
||||
|
||||
const APP_NAME: string = "telemetry";
|
||||
const ROUTE_PREFIXES: Array<string> = [`/${APP_NAME}`, "/"];
|
||||
const TELEMETRY_PREFIXES: Array<string> = [`/${APP_NAME}`, "/"];
|
||||
|
||||
app.use(ROUTE_PREFIXES, OTelIngestAPI);
|
||||
app.use(ROUTE_PREFIXES, MetricsAPI);
|
||||
app.use(ROUTE_PREFIXES, SyslogAPI);
|
||||
app.use(ROUTE_PREFIXES, FluentAPI);
|
||||
// Existing telemetry routes
|
||||
app.use(TELEMETRY_PREFIXES, OTelIngestAPI);
|
||||
app.use(TELEMETRY_PREFIXES, MetricsAPI);
|
||||
app.use(TELEMETRY_PREFIXES, SyslogAPI);
|
||||
app.use(TELEMETRY_PREFIXES, FluentAPI);
|
||||
|
||||
// ProbeIngest routes under ["/probe-ingest", "/ingestor", "/"]
|
||||
// "/ingestor" is used for backward compatibility because probes are already deployed with this path in client environments.
|
||||
const PROBE_INGEST_PREFIXES: Array<string> = ["/probe-ingest", "/ingestor", "/"];
|
||||
app.use(PROBE_INGEST_PREFIXES, ProbeIngestRegisterAPI);
|
||||
app.use(PROBE_INGEST_PREFIXES, ProbeIngestMonitorAPI);
|
||||
app.use(PROBE_INGEST_PREFIXES, ProbeIngestAPI);
|
||||
app.use(["/probe-ingest", "/"], IncomingEmailAPI);
|
||||
|
||||
// ServerMonitorIngest routes under ["/server-monitor-ingest", "/"]
|
||||
const SERVER_MONITOR_PREFIXES: Array<string> = ["/server-monitor-ingest", "/"];
|
||||
app.use(SERVER_MONITOR_PREFIXES, ServerMonitorAPI);
|
||||
|
||||
// IncomingRequestIngest routes under ["/incoming-request-ingest", "/"]
|
||||
const INCOMING_REQUEST_PREFIXES: Array<string> = ["/incoming-request-ingest", "/"];
|
||||
app.use(INCOMING_REQUEST_PREFIXES, IncomingRequestAPI);
|
||||
|
||||
const init: PromiseVoidFunction = async (): Promise<void> => {
|
||||
try {
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
import { IncomingRequestIngestJobData } from "../../Services/Queue/IncomingRequestIngestQueueService";
|
||||
import { IncomingRequestIngestJobData } from "../../Services/Queue/TelemetryQueueService";
|
||||
import logger from "Common/Server/Utils/Logger";
|
||||
import { QueueJob, QueueName } from "Common/Server/Infrastructure/Queue";
|
||||
import QueueWorker from "Common/Server/Infrastructure/QueueWorker";
|
||||
import HTTPMethod from "Common/Types/API/HTTPMethod";
|
||||
import OneUptimeDate from "Common/Types/Date";
|
||||
import Dictionary from "Common/Types/Dictionary";
|
||||
@@ -14,46 +12,8 @@ import ObjectID from "Common/Types/ObjectID";
|
||||
import MonitorService from "Common/Server/Services/MonitorService";
|
||||
import MonitorResourceUtil from "Common/Server/Utils/Monitor/MonitorResource";
|
||||
import Monitor from "Common/Models/DatabaseModels/Monitor";
|
||||
import { INCOMING_REQUEST_INGEST_CONCURRENCY } from "../../Config";
|
||||
|
||||
// Set up the worker for processing incoming request ingest queue
|
||||
QueueWorker.getWorker(
|
||||
QueueName.IncomingRequestIngest,
|
||||
async (job: QueueJob): Promise<void> => {
|
||||
logger.debug(`Processing incoming request ingestion job: ${job.name}`);
|
||||
|
||||
try {
|
||||
const jobData: IncomingRequestIngestJobData =
|
||||
job.data as IncomingRequestIngestJobData;
|
||||
|
||||
await processIncomingRequestFromQueue(jobData);
|
||||
|
||||
logger.debug(
|
||||
`Successfully processed incoming request ingestion job: ${job.name}`,
|
||||
);
|
||||
} catch (error) {
|
||||
/*
|
||||
* Certain BadDataException cases are expected / non-actionable and should not fail the job.
|
||||
* These include disabled monitors (manual, maintenance, explicitly disabled) and missing monitors
|
||||
* (e.g. secret key referencing a deleted monitor). Retrying provides no value and only creates noise.
|
||||
*/
|
||||
if (
|
||||
error instanceof BadDataException &&
|
||||
(error.message === ExceptionMessages.MonitorNotFound ||
|
||||
error.message === ExceptionMessages.MonitorDisabled)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
logger.error(`Error processing incoming request ingestion job:`);
|
||||
logger.error(error);
|
||||
throw error; // rethrow other errors so they are visible and retried if needed.
|
||||
}
|
||||
},
|
||||
{ concurrency: INCOMING_REQUEST_INGEST_CONCURRENCY }, // Configurable via env, defaults to 100
|
||||
);
|
||||
|
||||
async function processIncomingRequestFromQueue(
|
||||
export async function processIncomingRequestFromQueue(
|
||||
jobData: IncomingRequestIngestJobData,
|
||||
): Promise<void> {
|
||||
const requestHeaders: Dictionary<string> = jobData.requestHeaders;
|
||||
@@ -116,4 +76,4 @@ async function processIncomingRequestFromQueue(
|
||||
await MonitorResourceUtil.monitorResource(incomingRequest);
|
||||
}
|
||||
|
||||
logger.debug("Incoming request ingest worker initialized");
|
||||
logger.debug("Incoming request ingest processing functions loaded");
|
||||
@@ -1,10 +1,8 @@
|
||||
import {
|
||||
ProbeIngestJobData,
|
||||
IncomingEmailJobData,
|
||||
} from "../../Services/Queue/ProbeIngestQueueService";
|
||||
} from "../../Services/Queue/TelemetryQueueService";
|
||||
import logger from "Common/Server/Utils/Logger";
|
||||
import { QueueJob, QueueName } from "Common/Server/Infrastructure/Queue";
|
||||
import QueueWorker from "Common/Server/Infrastructure/QueueWorker";
|
||||
import BadDataException from "Common/Types/Exception/BadDataException";
|
||||
import JSONFunctions from "Common/Types/JSONFunctions";
|
||||
import ObjectID from "Common/Types/ObjectID";
|
||||
@@ -18,48 +16,9 @@ import MonitorType from "Common/Types/Monitor/MonitorType";
|
||||
import Monitor from "Common/Models/DatabaseModels/Monitor";
|
||||
import { MonitorStepProbeResponse } from "Common/Models/DatabaseModels/MonitorProbe";
|
||||
import { JSONObject } from "Common/Types/JSON";
|
||||
import { PROBE_INGEST_CONCURRENCY } from "../../Config";
|
||||
import ExceptionMessages from "Common/Types/Exception/ExceptionMessages";
|
||||
|
||||
// Set up the worker for processing probe ingest queue
|
||||
QueueWorker.getWorker(
|
||||
QueueName.ProbeIngest,
|
||||
async (job: QueueJob): Promise<void> => {
|
||||
logger.debug(`Processing probe ingestion job: ${job.name}`);
|
||||
|
||||
try {
|
||||
const jobData: ProbeIngestJobData = job.data as ProbeIngestJobData;
|
||||
|
||||
if (jobData.jobType === "incoming-email") {
|
||||
await processIncomingEmailFromQueue(jobData);
|
||||
} else {
|
||||
await processProbeFromQueue(jobData);
|
||||
}
|
||||
|
||||
logger.debug(`Successfully processed probe ingestion job: ${job.name}`);
|
||||
} catch (error) {
|
||||
/*
|
||||
* Certain BadDataException cases are expected / non-actionable and should not fail the job.
|
||||
* These include disabled monitors (manual, maintenance, explicitly disabled) and missing monitors
|
||||
* (e.g. secret key referencing a deleted monitor). Retrying provides no value and only creates noise.
|
||||
*/
|
||||
if (
|
||||
error instanceof BadDataException &&
|
||||
(error.message === ExceptionMessages.MonitorNotFound ||
|
||||
error.message === ExceptionMessages.MonitorDisabled)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
logger.error(`Error processing probe ingestion job:`);
|
||||
logger.error(error);
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
{ concurrency: PROBE_INGEST_CONCURRENCY }, // Configurable via env, defaults to 100
|
||||
);
|
||||
|
||||
async function processProbeFromQueue(
|
||||
export async function processProbeFromQueue(
|
||||
jobData: ProbeIngestJobData,
|
||||
): Promise<void> {
|
||||
const probeResponse: ProbeMonitorResponse = JSONFunctions.deserialize(
|
||||
@@ -103,7 +62,7 @@ async function processProbeFromQueue(
|
||||
}
|
||||
}
|
||||
|
||||
async function processIncomingEmailFromQueue(
|
||||
export async function processIncomingEmailFromQueue(
|
||||
jobData: ProbeIngestJobData,
|
||||
): Promise<void> {
|
||||
const emailData: IncomingEmailJobData | undefined = jobData.incomingEmail;
|
||||
@@ -177,4 +136,4 @@ async function processIncomingEmailFromQueue(
|
||||
await MonitorResourceUtil.monitorResource(incomingEmailRequest);
|
||||
}
|
||||
|
||||
logger.debug("Probe ingest worker initialized");
|
||||
logger.debug("Probe ingest processing functions loaded");
|
||||
@@ -1,7 +1,5 @@
|
||||
import { ServerMonitorIngestJobData } from "../../Services/Queue/ServerMonitorIngestQueueService";
|
||||
import { ServerMonitorIngestJobData } from "../../Services/Queue/TelemetryQueueService";
|
||||
import logger from "Common/Server/Utils/Logger";
|
||||
import { QueueJob, QueueName } from "Common/Server/Infrastructure/Queue";
|
||||
import QueueWorker from "Common/Server/Infrastructure/QueueWorker";
|
||||
import BadDataException from "Common/Types/Exception/BadDataException";
|
||||
import ExceptionMessages from "Common/Types/Exception/ExceptionMessages";
|
||||
import { JSONObject } from "Common/Types/JSON";
|
||||
@@ -14,46 +12,8 @@ import MonitorResourceUtil from "Common/Server/Utils/Monitor/MonitorResource";
|
||||
import Monitor from "Common/Models/DatabaseModels/Monitor";
|
||||
import OneUptimeDate from "Common/Types/Date";
|
||||
import ProjectService from "Common/Server/Services/ProjectService";
|
||||
import { SERVER_MONITOR_INGEST_CONCURRENCY } from "../../Config";
|
||||
|
||||
// Set up the worker for processing server monitor ingest queue
|
||||
QueueWorker.getWorker(
|
||||
QueueName.ServerMonitorIngest,
|
||||
async (job: QueueJob): Promise<void> => {
|
||||
logger.debug(`Processing server monitor ingestion job: ${job.name}`);
|
||||
|
||||
try {
|
||||
const jobData: ServerMonitorIngestJobData =
|
||||
job.data as ServerMonitorIngestJobData;
|
||||
|
||||
await processServerMonitorFromQueue(jobData);
|
||||
|
||||
logger.debug(
|
||||
`Successfully processed server monitor ingestion job: ${job.name}`,
|
||||
);
|
||||
} catch (error) {
|
||||
/*
|
||||
* Certain BadDataException cases are expected / non-actionable and should not fail the job.
|
||||
* These include disabled monitors (manual, maintenance, explicitly disabled) and missing monitors
|
||||
* (e.g. secret key referencing a deleted monitor). Retrying provides no value and only creates noise.
|
||||
*/
|
||||
if (
|
||||
error instanceof BadDataException &&
|
||||
(error.message === ExceptionMessages.MonitorNotFound ||
|
||||
error.message === ExceptionMessages.MonitorDisabled)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
logger.error(`Error processing server monitor ingestion job:`);
|
||||
logger.error(error);
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
{ concurrency: SERVER_MONITOR_INGEST_CONCURRENCY }, // Configurable via env, defaults to 100
|
||||
);
|
||||
|
||||
async function processServerMonitorFromQueue(
|
||||
export async function processServerMonitorFromQueue(
|
||||
jobData: ServerMonitorIngestJobData,
|
||||
): Promise<void> {
|
||||
const monitorSecretKeyAsString: string = jobData.secretKey;
|
||||
@@ -104,4 +64,4 @@ async function processServerMonitorFromQueue(
|
||||
await MonitorResourceUtil.monitorResource(serverMonitorResponse);
|
||||
}
|
||||
|
||||
logger.debug("Server monitor ingest worker initialized");
|
||||
logger.debug("Server monitor ingest processing functions loaded");
|
||||
@@ -7,11 +7,19 @@ import OtelTracesIngestService from "../../Services/OtelTracesIngestService";
|
||||
import OtelMetricsIngestService from "../../Services/OtelMetricsIngestService";
|
||||
import SyslogIngestService from "../../Services/SyslogIngestService";
|
||||
import FluentLogsIngestService from "../../Services/FluentLogsIngestService";
|
||||
import {
|
||||
processProbeFromQueue,
|
||||
processIncomingEmailFromQueue,
|
||||
} from "../ProbeIngest/ProcessProbeIngest";
|
||||
import { processServerMonitorFromQueue } from "../ServerMonitorIngest/ProcessServerMonitorIngest";
|
||||
import { processIncomingRequestFromQueue } from "../IncomingRequestIngest/ProcessIncomingRequestIngest";
|
||||
import { TelemetryRequest } from "Common/Server/Middleware/TelemetryIngest";
|
||||
import logger from "Common/Server/Utils/Logger";
|
||||
import { QueueJob, QueueName } from "Common/Server/Infrastructure/Queue";
|
||||
import QueueWorker from "Common/Server/Infrastructure/QueueWorker";
|
||||
import ObjectID from "Common/Types/ObjectID";
|
||||
import BadDataException from "Common/Types/Exception/BadDataException";
|
||||
import ExceptionMessages from "Common/Types/Exception/ExceptionMessages";
|
||||
import {
|
||||
TELEMETRY_CONCURRENCY,
|
||||
TELEMETRY_LOCK_DURATION_MS,
|
||||
@@ -27,54 +35,123 @@ QueueWorker.getWorker(
|
||||
const jobData: TelemetryIngestJobData =
|
||||
job.data as TelemetryIngestJobData;
|
||||
|
||||
// Create a mock request object with the queued data
|
||||
const mockRequest: TelemetryRequest = {
|
||||
projectId: new ObjectID(jobData.projectId.toString()),
|
||||
body: jobData.requestBody,
|
||||
headers: jobData.requestHeaders,
|
||||
} as TelemetryRequest;
|
||||
|
||||
// Process based on telemetry type
|
||||
switch (jobData.type) {
|
||||
case TelemetryType.Logs:
|
||||
case TelemetryType.Logs: {
|
||||
// Create a mock request object with the queued data
|
||||
const mockRequest: TelemetryRequest = {
|
||||
projectId: new ObjectID(jobData.projectId!.toString()),
|
||||
body: jobData.requestBody!,
|
||||
headers: jobData.requestHeaders!,
|
||||
} as TelemetryRequest;
|
||||
|
||||
await OtelLogsIngestService.processLogsFromQueue(mockRequest);
|
||||
logger.debug(
|
||||
`Successfully processed logs for project: ${jobData.projectId}`,
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
case TelemetryType.Traces: {
|
||||
const mockRequest: TelemetryRequest = {
|
||||
projectId: new ObjectID(jobData.projectId!.toString()),
|
||||
body: jobData.requestBody!,
|
||||
headers: jobData.requestHeaders!,
|
||||
} as TelemetryRequest;
|
||||
|
||||
case TelemetryType.Traces:
|
||||
await OtelTracesIngestService.processTracesFromQueue(mockRequest);
|
||||
logger.debug(
|
||||
`Successfully processed traces for project: ${jobData.projectId}`,
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
case TelemetryType.Metrics: {
|
||||
const mockRequest: TelemetryRequest = {
|
||||
projectId: new ObjectID(jobData.projectId!.toString()),
|
||||
body: jobData.requestBody!,
|
||||
headers: jobData.requestHeaders!,
|
||||
} as TelemetryRequest;
|
||||
|
||||
case TelemetryType.Metrics:
|
||||
await OtelMetricsIngestService.processMetricsFromQueue(mockRequest);
|
||||
logger.debug(
|
||||
`Successfully processed metrics for project: ${jobData.projectId}`,
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
case TelemetryType.Syslog: {
|
||||
const mockRequest: TelemetryRequest = {
|
||||
projectId: new ObjectID(jobData.projectId!.toString()),
|
||||
body: jobData.requestBody!,
|
||||
headers: jobData.requestHeaders!,
|
||||
} as TelemetryRequest;
|
||||
|
||||
case TelemetryType.Syslog:
|
||||
await SyslogIngestService.processSyslogFromQueue(mockRequest);
|
||||
logger.debug(
|
||||
`Successfully processed syslog payload for project: ${jobData.projectId}`,
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
case TelemetryType.FluentLogs: {
|
||||
const mockRequest: TelemetryRequest = {
|
||||
projectId: new ObjectID(jobData.projectId!.toString()),
|
||||
body: jobData.requestBody!,
|
||||
headers: jobData.requestHeaders!,
|
||||
} as TelemetryRequest;
|
||||
|
||||
case TelemetryType.FluentLogs:
|
||||
await FluentLogsIngestService.processFluentLogsFromQueue(mockRequest);
|
||||
logger.debug(
|
||||
`Successfully processed fluent logs for project: ${jobData.projectId}`,
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
case TelemetryType.ProbeIngest:
|
||||
if (jobData.probeIngest) {
|
||||
if (jobData.probeIngest.jobType === "incoming-email") {
|
||||
await processIncomingEmailFromQueue(jobData.probeIngest);
|
||||
} else {
|
||||
await processProbeFromQueue(jobData.probeIngest);
|
||||
}
|
||||
}
|
||||
logger.debug(`Successfully processed probe ingest job`);
|
||||
break;
|
||||
|
||||
case TelemetryType.ServerMonitorIngest:
|
||||
if (jobData.serverMonitorIngest) {
|
||||
await processServerMonitorFromQueue(jobData.serverMonitorIngest);
|
||||
}
|
||||
logger.debug(`Successfully processed server monitor ingest job`);
|
||||
break;
|
||||
|
||||
case TelemetryType.IncomingRequestIngest:
|
||||
if (jobData.incomingRequestIngest) {
|
||||
await processIncomingRequestFromQueue(
|
||||
jobData.incomingRequestIngest,
|
||||
);
|
||||
}
|
||||
logger.debug(`Successfully processed incoming request ingest job`);
|
||||
break;
|
||||
|
||||
default:
|
||||
throw new Error(`Unknown telemetry type: ${jobData.type}`);
|
||||
}
|
||||
} catch (error) {
|
||||
/*
|
||||
* Certain BadDataException cases are expected / non-actionable and should not fail the job.
|
||||
* These include disabled monitors (manual, maintenance, explicitly disabled) and missing monitors
|
||||
* (e.g. secret key referencing a deleted monitor). Retrying provides no value and only creates noise.
|
||||
*/
|
||||
if (
|
||||
error instanceof BadDataException &&
|
||||
(error.message === ExceptionMessages.MonitorNotFound ||
|
||||
error.message === ExceptionMessages.MonitorDisabled)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
logger.error(`Error processing telemetry job:`);
|
||||
logger.error(error);
|
||||
throw error;
|
||||
|
||||
@@ -3,6 +3,7 @@ import Queue, { QueueName } from "Common/Server/Infrastructure/Queue";
|
||||
import { JSONObject } from "Common/Types/JSON";
|
||||
import OneUptimeDate from "Common/Types/Date";
|
||||
import logger from "Common/Server/Utils/Logger";
|
||||
import Dictionary from "Common/Types/Dictionary";
|
||||
|
||||
export enum TelemetryType {
|
||||
Logs = "logs",
|
||||
@@ -10,14 +11,69 @@ export enum TelemetryType {
|
||||
Metrics = "metrics",
|
||||
Syslog = "syslog",
|
||||
FluentLogs = "fluentlogs",
|
||||
ProbeIngest = "probe-ingest",
|
||||
ServerMonitorIngest = "server-monitor-ingest",
|
||||
IncomingRequestIngest = "incoming-request-ingest",
|
||||
}
|
||||
|
||||
export type ProbeIngestJobType =
|
||||
| "probe-response"
|
||||
| "monitor-test"
|
||||
| "incoming-email";
|
||||
|
||||
export interface IncomingEmailJobData {
|
||||
secretKey: string;
|
||||
emailFrom: string;
|
||||
emailTo: string;
|
||||
emailSubject: string;
|
||||
emailBody: string;
|
||||
emailBodyHtml: string | undefined;
|
||||
emailHeaders: Dictionary<string> | undefined;
|
||||
attachments:
|
||||
| Array<{
|
||||
filename: string;
|
||||
contentType: string;
|
||||
size: number;
|
||||
}>
|
||||
| undefined;
|
||||
}
|
||||
|
||||
export interface ProbeIngestJobData {
|
||||
jobType: ProbeIngestJobType;
|
||||
ingestionTimestamp: Date;
|
||||
// For probe-response and monitor-test
|
||||
probeMonitorResponse?: JSONObject | undefined;
|
||||
testId?: string | undefined;
|
||||
// For incoming-email
|
||||
incomingEmail?: IncomingEmailJobData | undefined;
|
||||
}
|
||||
|
||||
export interface ServerMonitorIngestJobData {
|
||||
secretKey: string;
|
||||
serverMonitorResponse: JSONObject;
|
||||
ingestionTimestamp: Date;
|
||||
}
|
||||
|
||||
export interface IncomingRequestIngestJobData {
|
||||
secretKey: string;
|
||||
requestHeaders: Dictionary<string>;
|
||||
requestBody: string | JSONObject;
|
||||
requestMethod: string;
|
||||
ingestionTimestamp: Date;
|
||||
}
|
||||
|
||||
export interface TelemetryIngestJobData {
|
||||
type: TelemetryType;
|
||||
projectId: string;
|
||||
requestBody: JSONObject;
|
||||
requestHeaders: Record<string, string>;
|
||||
projectId?: string;
|
||||
requestBody?: JSONObject;
|
||||
requestHeaders?: Record<string, string>;
|
||||
ingestionTimestamp: Date;
|
||||
// ProbeIngest-specific
|
||||
probeIngest?: ProbeIngestJobData;
|
||||
// ServerMonitorIngest-specific
|
||||
serverMonitorIngest?: ServerMonitorIngestJobData;
|
||||
// IncomingRequestIngest-specific
|
||||
incomingRequestIngest?: IncomingRequestIngestJobData;
|
||||
}
|
||||
|
||||
// Legacy interfaces for backward compatibility
|
||||
@@ -86,6 +142,169 @@ export default class TelemetryQueueService {
|
||||
return this.addTelemetryIngestJob(req, TelemetryType.FluentLogs);
|
||||
}
|
||||
|
||||
public static async addProbeIngestJob(data: {
|
||||
probeMonitorResponse: JSONObject;
|
||||
jobType: "probe-response" | "monitor-test";
|
||||
testId?: string;
|
||||
}): Promise<void> {
|
||||
try {
|
||||
const probeData: ProbeIngestJobData = {
|
||||
probeMonitorResponse: data.probeMonitorResponse,
|
||||
jobType: data.jobType,
|
||||
testId: data.testId,
|
||||
ingestionTimestamp: OneUptimeDate.getCurrentDate(),
|
||||
};
|
||||
|
||||
const jobData: TelemetryIngestJobData = {
|
||||
type: TelemetryType.ProbeIngest,
|
||||
ingestionTimestamp: OneUptimeDate.getCurrentDate(),
|
||||
probeIngest: probeData,
|
||||
};
|
||||
|
||||
const jobId: string = `probe-${data.jobType}-${data.testId || "general"}-${OneUptimeDate.getCurrentDateAsUnixNano()}`;
|
||||
|
||||
await Queue.addJob(
|
||||
QueueName.Telemetry,
|
||||
jobId,
|
||||
"ProcessTelemetry",
|
||||
jobData as unknown as JSONObject,
|
||||
);
|
||||
|
||||
logger.debug(`Added probe ingestion job: ${jobId}`);
|
||||
} catch (error) {
|
||||
logger.error(`Error adding probe ingestion job:`);
|
||||
logger.error(error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
public static async addIncomingEmailJob(data: {
|
||||
secretKey: string;
|
||||
emailFrom: string;
|
||||
emailTo: string;
|
||||
emailSubject: string;
|
||||
emailBody: string;
|
||||
emailBodyHtml?: string | undefined;
|
||||
emailHeaders?: Dictionary<string> | undefined;
|
||||
attachments?:
|
||||
| Array<{
|
||||
filename: string;
|
||||
contentType: string;
|
||||
size: number;
|
||||
}>
|
||||
| undefined;
|
||||
}): Promise<void> {
|
||||
try {
|
||||
const probeData: ProbeIngestJobData = {
|
||||
jobType: "incoming-email",
|
||||
ingestionTimestamp: OneUptimeDate.getCurrentDate(),
|
||||
incomingEmail: {
|
||||
secretKey: data.secretKey,
|
||||
emailFrom: data.emailFrom,
|
||||
emailTo: data.emailTo,
|
||||
emailSubject: data.emailSubject,
|
||||
emailBody: data.emailBody,
|
||||
emailBodyHtml: data.emailBodyHtml,
|
||||
emailHeaders: data.emailHeaders,
|
||||
attachments: data.attachments,
|
||||
},
|
||||
};
|
||||
|
||||
const jobData: TelemetryIngestJobData = {
|
||||
type: TelemetryType.ProbeIngest,
|
||||
ingestionTimestamp: OneUptimeDate.getCurrentDate(),
|
||||
probeIngest: probeData,
|
||||
};
|
||||
|
||||
const jobId: string = `incoming-email-${data.secretKey}-${OneUptimeDate.getCurrentDateAsUnixNano()}`;
|
||||
|
||||
await Queue.addJob(
|
||||
QueueName.Telemetry,
|
||||
jobId,
|
||||
"ProcessTelemetry",
|
||||
jobData as unknown as JSONObject,
|
||||
);
|
||||
|
||||
logger.debug(`Added incoming email ingestion job: ${jobId}`);
|
||||
} catch (error) {
|
||||
logger.error(`Error adding incoming email ingestion job:`);
|
||||
logger.error(error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
public static async addServerMonitorIngestJob(data: {
|
||||
secretKey: string;
|
||||
serverMonitorResponse: JSONObject;
|
||||
}): Promise<void> {
|
||||
try {
|
||||
const serverMonitorData: ServerMonitorIngestJobData = {
|
||||
secretKey: data.secretKey,
|
||||
serverMonitorResponse: data.serverMonitorResponse,
|
||||
ingestionTimestamp: OneUptimeDate.getCurrentDate(),
|
||||
};
|
||||
|
||||
const jobData: TelemetryIngestJobData = {
|
||||
type: TelemetryType.ServerMonitorIngest,
|
||||
ingestionTimestamp: OneUptimeDate.getCurrentDate(),
|
||||
serverMonitorIngest: serverMonitorData,
|
||||
};
|
||||
|
||||
const jobId: string = `server-monitor-${data.secretKey}-${OneUptimeDate.getCurrentDateAsUnixNano()}`;
|
||||
|
||||
await Queue.addJob(
|
||||
QueueName.Telemetry,
|
||||
jobId,
|
||||
"ProcessTelemetry",
|
||||
jobData as unknown as JSONObject,
|
||||
);
|
||||
|
||||
logger.debug(`Added server monitor ingestion job: ${jobId}`);
|
||||
} catch (error) {
|
||||
logger.error(`Error adding server monitor ingestion job:`);
|
||||
logger.error(error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
public static async addIncomingRequestIngestJob(data: {
|
||||
secretKey: string;
|
||||
requestHeaders: Dictionary<string>;
|
||||
requestBody: string | JSONObject;
|
||||
requestMethod: string;
|
||||
}): Promise<void> {
|
||||
try {
|
||||
const incomingRequestData: IncomingRequestIngestJobData = {
|
||||
secretKey: data.secretKey,
|
||||
requestHeaders: data.requestHeaders,
|
||||
requestBody: data.requestBody,
|
||||
requestMethod: data.requestMethod,
|
||||
ingestionTimestamp: OneUptimeDate.getCurrentDate(),
|
||||
};
|
||||
|
||||
const jobData: TelemetryIngestJobData = {
|
||||
type: TelemetryType.IncomingRequestIngest,
|
||||
ingestionTimestamp: OneUptimeDate.getCurrentDate(),
|
||||
incomingRequestIngest: incomingRequestData,
|
||||
};
|
||||
|
||||
const jobId: string = `incoming-request-${data.secretKey}-${OneUptimeDate.getCurrentDateAsUnixNano()}`;
|
||||
|
||||
await Queue.addJob(
|
||||
QueueName.Telemetry,
|
||||
jobId,
|
||||
"ProcessTelemetry",
|
||||
jobData as unknown as JSONObject,
|
||||
);
|
||||
|
||||
logger.debug(`Added incoming request ingestion job: ${jobId}`);
|
||||
} catch (error) {
|
||||
logger.error(`Error adding incoming request ingestion job:`);
|
||||
logger.error(error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
public static async getQueueSize(): Promise<number> {
|
||||
return Queue.getQueueSize(QueueName.Telemetry);
|
||||
}
|
||||
|
||||
@@ -92,16 +92,11 @@ REDIS_TLS_CA=
|
||||
REDIS_TLS_SENTINEL_MODE=false
|
||||
|
||||
# Hostnames. Usually does not need to change.
|
||||
PROBE_INGEST_HOSTNAME=probe-ingest:3400
|
||||
INCOMING_REQUEST_INGEST_HOSTNAME=incoming-request-ingest:3402
|
||||
TELEMETRY_HOSTNAME=telemetry:3403
|
||||
|
||||
SERVER_ACCOUNTS_HOSTNAME=accounts
|
||||
SERVER_APP_HOSTNAME=app
|
||||
SERVER_PROBE_INGEST_HOSTNAME=probe-ingest
|
||||
SERVER_SERVER_MONITOR_INGEST_HOSTNAME=server-monitor-ingest
|
||||
SERVER_TELEMETRY_HOSTNAME=telemetry
|
||||
SERVER_INCOMING_REQUEST_INGEST_HOSTNAME=incoming-request-ingest
|
||||
SERVER_TEST_SERVER_HOSTNAME=test-server
|
||||
SERVER_STATUS_PAGE_HOSTNAME=status-page
|
||||
SERVER_DASHBOARD_HOSTNAME=dashboard
|
||||
@@ -114,10 +109,7 @@ SERVER_MCP_HOSTNAME=mcp
|
||||
#Ports. Usually they don't need to change.
|
||||
|
||||
APP_PORT=3002
|
||||
PROBE_INGEST_PORT=3400
|
||||
SERVER_MONITOR_INGEST_PORT=3404
|
||||
TELEMETRY_PORT=3403
|
||||
INCOMING_REQUEST_INGEST_PORT=3402
|
||||
TEST_SERVER_PORT=3800
|
||||
ACCOUNTS_PORT=3003
|
||||
STATUS_PAGE_PORT=3105
|
||||
@@ -252,14 +244,6 @@ TELEMETRY_CONCURRENCY=100
|
||||
# Max number of jobs processed concurrently by Fluent Logs worker
|
||||
FLUENT_LOGS_CONCURRENCY=100
|
||||
|
||||
# Max number of jobs processed concurrently by Incoming Request Ingest worker
|
||||
INCOMING_REQUEST_INGEST_CONCURRENCY=100
|
||||
|
||||
# Max number of jobs processed concurrently by Server Monitor Ingest worker
|
||||
SERVER_MONITOR_INGEST_CONCURRENCY=100
|
||||
|
||||
# Max number of jobs processed concurrently by Probe Ingest worker
|
||||
PROBE_INGEST_CONCURRENCY=100
|
||||
|
||||
# Max number of jobs processed concurrently by Worker service
|
||||
WORKER_CONCURRENCY=100
|
||||
@@ -321,10 +305,8 @@ AI_AGENT_PORT=3876
|
||||
# By default telemetry is disabled for all services in docker compose. If you want to enable telemetry for a service, then set the env var to false.
|
||||
DISABLE_TELEMETRY_FOR_ACCOUNTS=true
|
||||
DISABLE_TELEMETRY_FOR_APP=true
|
||||
DISABLE_TELEMETRY_FOR_PROBE_INGEST=true
|
||||
DISABLE_TELEMETRY_FOR_TELEMETRY=true
|
||||
DISABLE_TELEMETRY_FOR_FLUENT_LOGS=true
|
||||
DISABLE_TELEMETRY_FOR_INCOMING_REQUEST_INGEST=true
|
||||
DISABLE_TELEMETRY_FOR_TEST_SERVER=true
|
||||
DISABLE_TELEMETRY_FOR_STATUS_PAGE=true
|
||||
DISABLE_TELEMETRY_FOR_DASHBOARD=true
|
||||
@@ -334,7 +316,7 @@ DISABLE_TELEMETRY_FOR_OTEL_COLLECTOR=true
|
||||
DISABLE_TELEMETRY_FOR_ISOLATED_VM=true
|
||||
DISABLE_TELEMETRY_FOR_INGRESS=true
|
||||
DISABLE_TELEMETRY_FOR_WORKER=true
|
||||
DISABLE_TELEMETRY_FOR_SERVER_MONITOR_INGEST=true
|
||||
|
||||
DISABLE_TELEMETRY_FOR_MCP=true
|
||||
DISABLE_TELEMETRY_FOR_AI_AGENT=true
|
||||
|
||||
|
||||
@@ -31,9 +31,7 @@ x-common-variables: &common-variables
|
||||
SERVER_ACCOUNTS_HOSTNAME: accounts
|
||||
SERVER_APP_HOSTNAME: app
|
||||
SERVER_ALERT_HOSTNAME: alert
|
||||
SERVER_PROBE_INGEST_HOSTNAME: probe-ingest
|
||||
SERVER_TELEMETRY_HOSTNAME: telemetry
|
||||
SERVER_INCOMING_REQUEST_INGEST_HOSTNAME: incoming-request-ingest
|
||||
SERVER_TEST_SERVER_HOSTNAME: test-server
|
||||
SERVER_STATUS_PAGE_HOSTNAME: status-page
|
||||
SERVER_DASHBOARD_HOSTNAME: dashboard
|
||||
@@ -44,15 +42,12 @@ x-common-variables: &common-variables
|
||||
SERVER_HOME_HOSTNAME: home
|
||||
SERVER_WORKFLOW_HOSTNAME: workflow
|
||||
SERVER_DOCS_HOSTNAME: docs
|
||||
SERVER_SERVER_MONITOR_INGEST_HOSTNAME: server-monitor-ingest
|
||||
SERVER_MCP_HOSTNAME: mcp
|
||||
|
||||
#Ports. Usually they don't need to change.
|
||||
APP_PORT: ${APP_PORT}
|
||||
HOME_PORT: ${HOME_PORT}
|
||||
PROBE_INGEST_PORT: ${PROBE_INGEST_PORT}
|
||||
TELEMETRY_PORT: ${TELEMETRY_PORT}
|
||||
INCOMING_REQUEST_INGEST_PORT: ${INCOMING_REQUEST_INGEST_PORT}
|
||||
TEST_SERVER_PORT: ${TEST_SERVER_PORT}
|
||||
ACCOUNTS_PORT: ${ACCOUNTS_PORT}
|
||||
STATUS_PAGE_PORT: ${STATUS_PAGE_PORT}
|
||||
@@ -62,7 +57,6 @@ x-common-variables: &common-variables
|
||||
WORKER_PORT: ${WORKER_PORT}
|
||||
WORKFLOW_PORT: ${WORKFLOW_PORT}
|
||||
DOCS_PORT: ${DOCS_PORT}
|
||||
SERVER_MONITOR_INGEST_PORT: ${SERVER_MONITOR_INGEST_PORT}
|
||||
MCP_PORT: ${MCP_PORT}
|
||||
|
||||
OPENTELEMETRY_EXPORTER_OTLP_ENDPOINT: ${OPENTELEMETRY_EXPORTER_OTLP_ENDPOINT}
|
||||
@@ -492,35 +486,6 @@ services:
|
||||
options:
|
||||
max-size: "1000m"
|
||||
|
||||
probe-ingest:
|
||||
networks:
|
||||
- oneuptime
|
||||
restart: always
|
||||
environment:
|
||||
<<: *common-runtime-variables
|
||||
PORT: ${PROBE_INGEST_PORT}
|
||||
DISABLE_TELEMETRY: ${DISABLE_TELEMETRY_FOR_PROBE_INGEST}
|
||||
PROBE_INGEST_CONCURRENCY: ${PROBE_INGEST_CONCURRENCY}
|
||||
REGISTER_PROBE_KEY: ${REGISTER_PROBE_KEY}
|
||||
logging:
|
||||
driver: "local"
|
||||
options:
|
||||
max-size: "1000m"
|
||||
|
||||
server-monitor-ingest:
|
||||
networks:
|
||||
- oneuptime
|
||||
restart: always
|
||||
environment:
|
||||
<<: *common-runtime-variables
|
||||
PORT: ${SERVER_MONITOR_INGEST_PORT}
|
||||
DISABLE_TELEMETRY: ${DISABLE_TELEMETRY_FOR_SERVER_MONITOR_INGEST}
|
||||
SERVER_MONITOR_INGEST_CONCURRENCY: ${SERVER_MONITOR_INGEST_CONCURRENCY}
|
||||
logging:
|
||||
driver: "local"
|
||||
options:
|
||||
max-size: "1000m"
|
||||
|
||||
telemetry:
|
||||
networks:
|
||||
- oneuptime
|
||||
@@ -531,20 +496,7 @@ services:
|
||||
DISABLE_TELEMETRY: ${DISABLE_TELEMETRY_FOR_TELEMETRY}
|
||||
# Max concurrent telemetry jobs the worker will process
|
||||
TELEMETRY_CONCURRENCY: ${TELEMETRY_CONCURRENCY}
|
||||
logging:
|
||||
driver: "local"
|
||||
options:
|
||||
max-size: "1000m"
|
||||
|
||||
incoming-request-ingest:
|
||||
networks:
|
||||
- oneuptime
|
||||
restart: always
|
||||
environment:
|
||||
<<: *common-runtime-variables
|
||||
PORT: ${INCOMING_REQUEST_INGEST_PORT}
|
||||
DISABLE_TELEMETRY: ${DISABLE_TELEMETRY_FOR_INCOMING_REQUEST_INGEST}
|
||||
INCOMING_REQUEST_INGEST_CONCURRENCY: ${INCOMING_REQUEST_INGEST_CONCURRENCY}
|
||||
REGISTER_PROBE_KEY: ${REGISTER_PROBE_KEY}
|
||||
logging:
|
||||
driver: "local"
|
||||
options:
|
||||
|
||||
@@ -357,46 +357,6 @@ services:
|
||||
context: .
|
||||
dockerfile: ./IsolatedVM/Dockerfile
|
||||
|
||||
probe-ingest:
|
||||
volumes:
|
||||
- ./ProbeIngest:/usr/src/app:cached
|
||||
# Use node modules of the container and not host system.
|
||||
# https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder
|
||||
- /usr/src/app/node_modules/
|
||||
- ./Common:/usr/src/Common:cached
|
||||
- /usr/src/Common/node_modules/
|
||||
ports:
|
||||
- '9932:9229' # Debugging port.
|
||||
extends:
|
||||
file: ./docker-compose.base.yml
|
||||
service: probe-ingest
|
||||
depends_on:
|
||||
<<: *common-depends-on
|
||||
build:
|
||||
network: host
|
||||
context: .
|
||||
dockerfile: ./ProbeIngest/Dockerfile
|
||||
|
||||
server-monitor-ingest:
|
||||
volumes:
|
||||
- ./ServerMonitorIngest:/usr/src/app:cached
|
||||
# Use node modules of the container and not host system.
|
||||
# https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder
|
||||
- /usr/src/app/node_modules/
|
||||
- ./Common:/usr/src/Common:cached
|
||||
- /usr/src/Common/node_modules/
|
||||
ports:
|
||||
- '9941:9229' # Debugging port.
|
||||
extends:
|
||||
file: ./docker-compose.base.yml
|
||||
service: server-monitor-ingest
|
||||
depends_on:
|
||||
<<: *common-depends-on
|
||||
build:
|
||||
network: host
|
||||
context: .
|
||||
dockerfile: ./ServerMonitorIngest/Dockerfile
|
||||
|
||||
telemetry:
|
||||
volumes:
|
||||
- ./Telemetry:/usr/src/app:cached
|
||||
@@ -417,26 +377,6 @@ services:
|
||||
context: .
|
||||
dockerfile: ./Telemetry/Dockerfile
|
||||
|
||||
incoming-request-ingest:
|
||||
volumes:
|
||||
- ./IncomingRequestIngest:/usr/src/app:cached
|
||||
# Use node modules of the container and not host system.
|
||||
# https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder
|
||||
- /usr/src/app/node_modules/
|
||||
- ./Common:/usr/src/Common:cached
|
||||
- /usr/src/Common/node_modules/
|
||||
ports:
|
||||
- '9933:9229' # Debugging port.
|
||||
extends:
|
||||
file: ./docker-compose.base.yml
|
||||
service: incoming-request-ingest
|
||||
depends_on:
|
||||
<<: *common-depends-on
|
||||
build:
|
||||
network: host
|
||||
context: .
|
||||
dockerfile: ./IncomingRequestIngest/Dockerfile
|
||||
|
||||
mcp:
|
||||
volumes:
|
||||
- ./MCP:/usr/src/app:cached
|
||||
|
||||
@@ -123,22 +123,6 @@ services:
|
||||
depends_on:
|
||||
<<: *common-depends-on
|
||||
|
||||
probe-ingest:
|
||||
image: oneuptime/probe-ingest:${APP_TAG}
|
||||
extends:
|
||||
file: ./docker-compose.base.yml
|
||||
service: probe-ingest
|
||||
depends_on:
|
||||
<<: *common-depends-on
|
||||
|
||||
server-monitor-ingest:
|
||||
image: oneuptime/server-monitor-ingest:${APP_TAG}
|
||||
extends:
|
||||
file: ./docker-compose.base.yml
|
||||
service: server-monitor-ingest
|
||||
depends_on:
|
||||
<<: *common-depends-on
|
||||
|
||||
telemetry:
|
||||
image: oneuptime/telemetry:${APP_TAG}
|
||||
extends:
|
||||
@@ -147,14 +131,6 @@ services:
|
||||
depends_on:
|
||||
<<: *common-depends-on
|
||||
|
||||
incoming-request-ingest:
|
||||
image: oneuptime/incoming-request-ingest:${APP_TAG}
|
||||
extends:
|
||||
file: ./docker-compose.base.yml
|
||||
service: incoming-request-ingest
|
||||
depends_on:
|
||||
<<: *common-depends-on
|
||||
|
||||
isolated-vm:
|
||||
image: oneuptime/isolated-vm:${APP_TAG}
|
||||
extends:
|
||||
|
||||
Reference in New Issue
Block a user