feat: Add telemetry API for probe and server monitor ingestion

- Implemented Probe registration endpoint to handle global probes.
- Created Server Monitor ingestion API with endpoints for secret key verification and response ingestion.
- Developed job processing functions for incoming request and probe ingestion.
- Added middleware for probe authorization and request type definitions.
- Enhanced monitor utility functions to populate secrets in monitor steps.
- Introduced queue stats and failed jobs endpoints for server monitor.

config.example.env

@@ -92,16 +92,11 @@ REDIS_TLS_CA=
 REDIS_TLS_SENTINEL_MODE=false
 
 # Hostnames. Usually does not need to change. 
-PROBE_INGEST_HOSTNAME=probe-ingest:3400
-INCOMING_REQUEST_INGEST_HOSTNAME=incoming-request-ingest:3402
 TELEMETRY_HOSTNAME=telemetry:3403
 
 SERVER_ACCOUNTS_HOSTNAME=accounts
 SERVER_APP_HOSTNAME=app
-SERVER_PROBE_INGEST_HOSTNAME=probe-ingest
-SERVER_SERVER_MONITOR_INGEST_HOSTNAME=server-monitor-ingest
 SERVER_TELEMETRY_HOSTNAME=telemetry
-SERVER_INCOMING_REQUEST_INGEST_HOSTNAME=incoming-request-ingest
 SERVER_TEST_SERVER_HOSTNAME=test-server
 SERVER_STATUS_PAGE_HOSTNAME=status-page
 SERVER_DASHBOARD_HOSTNAME=dashboard
@@ -114,10 +109,7 @@ SERVER_MCP_HOSTNAME=mcp
 #Ports. Usually they don't need to change.
 
 APP_PORT=3002
-PROBE_INGEST_PORT=3400
-SERVER_MONITOR_INGEST_PORT=3404
 TELEMETRY_PORT=3403
-INCOMING_REQUEST_INGEST_PORT=3402
 TEST_SERVER_PORT=3800
 ACCOUNTS_PORT=3003
 STATUS_PAGE_PORT=3105
@@ -252,14 +244,6 @@ TELEMETRY_CONCURRENCY=100
 # Max number of jobs processed concurrently by Fluent Logs worker
 FLUENT_LOGS_CONCURRENCY=100
 
-# Max number of jobs processed concurrently by Incoming Request Ingest worker
-INCOMING_REQUEST_INGEST_CONCURRENCY=100
-
-# Max number of jobs processed concurrently by Server Monitor Ingest worker
-SERVER_MONITOR_INGEST_CONCURRENCY=100
-
-# Max number of jobs processed concurrently by Probe Ingest worker
-PROBE_INGEST_CONCURRENCY=100
 
 # Max number of jobs processed concurrently by Worker service
 WORKER_CONCURRENCY=100
@@ -321,10 +305,8 @@ AI_AGENT_PORT=3876
 # By default telemetry is disabled for all services in docker compose. If you want to enable telemetry for a service, then set the env var to false.
 DISABLE_TELEMETRY_FOR_ACCOUNTS=true
 DISABLE_TELEMETRY_FOR_APP=true
-DISABLE_TELEMETRY_FOR_PROBE_INGEST=true
 DISABLE_TELEMETRY_FOR_TELEMETRY=true
 DISABLE_TELEMETRY_FOR_FLUENT_LOGS=true
-DISABLE_TELEMETRY_FOR_INCOMING_REQUEST_INGEST=true
 DISABLE_TELEMETRY_FOR_TEST_SERVER=true
 DISABLE_TELEMETRY_FOR_STATUS_PAGE=true
 DISABLE_TELEMETRY_FOR_DASHBOARD=true
@@ -334,7 +316,7 @@ DISABLE_TELEMETRY_FOR_OTEL_COLLECTOR=true
 DISABLE_TELEMETRY_FOR_ISOLATED_VM=true
 DISABLE_TELEMETRY_FOR_INGRESS=true
 DISABLE_TELEMETRY_FOR_WORKER=true
-DISABLE_TELEMETRY_FOR_SERVER_MONITOR_INGEST=true
+
 DISABLE_TELEMETRY_FOR_MCP=true
 DISABLE_TELEMETRY_FOR_AI_AGENT=true