Add configuration files for OpenTelemetry Ingest, including .gitignore, .dockerignore, and nodemon.json; refactor code for consistency and clarity

This commit is contained in:
Simon Larsen
2024-11-21 16:42:29 +00:00
parent 945cef653c
commit eec51342de
30 changed files with 5287 additions and 15 deletions

View File

@@ -100,7 +100,6 @@ export const IncomingRequestIngestHostname: Hostname = Hostname.fromString(
}`,
);
export const FluentIngestHostname: Hostname = Hostname.fromString(
`${process.env["SERVER_FLUENT_INGEST_HOSTNAME"] || "localhost"}:${
process.env["FLUENT_INGEST_PORT"] || 80

View File

@@ -100,7 +100,9 @@ export default class ClickhouseDatabase {
public async checkConnnectionStatus(): Promise<boolean> {
// Ping clickhouse to check if the connection is still alive
try {
logger.debug("Checking Clickhouse Connection Status - pinging clickhouse");
logger.debug(
"Checking Clickhouse Connection Status - pinging clickhouse",
);
const result: PingResult | undefined = await this.getDataSource()?.ping();
logger.debug("Clickhouse Connection Status Result");
logger.debug(result);

View File

@@ -28,9 +28,11 @@ export const AdminDashboardRoute: Route = new Route("/admin");
export const IngestorRoute: Route = new Route("/ingestor");
export const IncomingRequestIngestRoute: Route = new Route("/incoming-request-ingest");
export const IncomingRequestIngestRoute: Route = new Route(
"/incoming-request-ingest",
);
export const FluentIngestRoute: Route = new Route("/fluent-ingest");
export const FluentIngestRoute: Route = new Route("/fluent-ingest");
export const RealtimeRoute: Route = new Route("/realtime/socket");

View File

@@ -16,7 +16,7 @@ import {
StatusPageRoute,
WorkflowRoute,
FluentIngestRoute,
IncomingRequestIngestRoute
IncomingRequestIngestRoute,
} from "Common/ServiceRoute";
import Hostname from "Common/Types/API/Hostname";
import Protocol from "Common/Types/API/Protocol";
@@ -69,7 +69,8 @@ export const WORKFLOW_HOSTNAME: Hostname = Hostname.fromString(HOST);
export const INGESTOR_HOSTNAME: Hostname = Hostname.fromString(HOST);
export const INCOMING_REQUEST_INGEST_HOSTNAME: Hostname = Hostname.fromString(HOST);
export const INCOMING_REQUEST_INGEST_HOSTNAME: Hostname =
Hostname.fromString(HOST);
export const FLUENT_INGEST_HOSTNAME: Hostname = Hostname.fromString(HOST);

View File

@@ -3,7 +3,11 @@ import { Page, expect, test } from "@playwright/test";
import URL from "Common/Types/API/URL";
test.describe("check live and health check of the fluent ingest", () => {
test("check if fluent ingest status is ok", async ({ page }: { page: Page }) => {
test("check if fluent ingest status is ok", async ({
page,
}: {
page: Page;
}) => {
await page.goto(
`${URL.fromString(BASE_URL.toString())
.addRoute("/fluent-ingest/status")

View File

@@ -3,7 +3,11 @@ import { Page, expect, test } from "@playwright/test";
import URL from "Common/Types/API/URL";
test.describe("check live and health check of the incoming request ingest", () => {
test("check if incoming request ingest status is ok", async ({ page }: { page: Page }) => {
test("check if incoming request ingest status is ok", async ({
page,
}: {
page: Page;
}) => {
await page.goto(
`${URL.fromString(BASE_URL.toString())
.addRoute("/incoming-request-ingest/status")
@@ -13,7 +17,11 @@ test.describe("check live and health check of the incoming request ingest", () =
expect(content).toContain('{"status":"ok"}');
});
test("check if incoming request ingest is ready", async ({ page }: { page: Page }) => {
test("check if incoming request ingest is ready", async ({
page,
}: {
page: Page;
}) => {
await page.goto(
`${URL.fromString(BASE_URL.toString())
.addRoute("/incoming-request-ingest/status/ready")
@@ -23,7 +31,11 @@ test.describe("check live and health check of the incoming request ingest", () =
expect(content).toContain('{"status":"ok"}');
});
test("check if incoming request ingest is live", async ({ page }: { page: Page }) => {
test("check if incoming request ingest is live", async ({
page,
}: {
page: Page;
}) => {
await page.goto(
`${URL.fromString(BASE_URL.toString())
.addRoute("/incoming-request-ingest/status/live")

View File

@@ -1,4 +1,3 @@
import FluentIngestAPI from "../FluentIngest/API/FluentIngest";
import { PromiseVoidFunction } from "Common/Types/FunctionTypes";

View File

@@ -0,0 +1,56 @@
.git
node_modules
# See https://help.github.com/ignore-files/ for more about ignoring files.
# dependencies
/node_modules
node_modules
.idea
# testing
/coverage
# production
/build
# misc
.DS_Store
env.js
npm-debug.log*
yarn-debug.log*
yarn-error.log*
yarn.lock
Untitled-1
*.local.sh
*.local.yaml
run
stop
nohup.out*
encrypted-credentials.tar
encrypted-credentials/
_README.md
# Important Add production values to gitignore.
values-saas-production.yaml
kubernetes/values-saas-production.yaml
/private
/tls_cert.pem
/tls_key.pem
/keys
temp_readme.md
tests/coverage
settings.json
GoSDK/tester/

1
OpenTelemetryIngest/.gitattributes vendored Normal file
View File

@@ -0,0 +1 @@
*.js text eol=lf

16
OpenTelemetryIngest/.gitignore vendored Normal file
View File

@@ -0,0 +1,16 @@
# See https://help.github.com/ignore-files/ for more about ignoring files.
# dependencies
#/backend/node_modules
/kubernetes
/node_modules
.idea
# misc
.DS_Store
npm-debug.log*
yarn-debug.log*
yarn-error.log*
yarn.lock

View File

@@ -0,0 +1,83 @@
#
# OneUptime-Ingestor Dockerfile
#
# Pull base image nodejs image.
FROM public.ecr.aws/docker/library/node:22.9
RUN mkdir /tmp/npm && chmod 2777 /tmp/npm && chown 1000:1000 /tmp/npm && npm config set cache /tmp/npm --global
RUN npm config set fetch-retries 5
RUN npm config set fetch-retry-mintimeout 100000
RUN npm config set fetch-retry-maxtimeout 600000
ARG GIT_SHA
ARG APP_VERSION
ENV GIT_SHA=${GIT_SHA}
ENV APP_VERSION=${APP_VERSION}
ENV NODE_OPTIONS="--use-openssl-ca"
## Add Intermediate Certs
COPY ./SslCertificates /usr/local/share/ca-certificates
RUN update-ca-certificates
# IF APP_VERSION is not set, set it to 1.0.0
RUN if [ -z "$APP_VERSION" ]; then export APP_VERSION=1.0.0; fi
RUN apt-get update
# Install bash.
RUN apt-get install bash -y && apt-get install curl -y && apt-get install iputils-ping -y
# Install python
RUN apt-get update && apt-get install -y .gyp python3 make g++
# Install playwright dependencies
RUN apt-get install -y libnss3 libnspr4 libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 libxkbcommon0 libxcomposite1 libxdamage1 libxfixes3 libxrandr2 libgbm1 libgtk-3-0 libpango-1.0-0 libcairo2 libgdk-pixbuf2.0-0 libasound2 libatspi2.0-0
#Use bash shell by default
SHELL ["/bin/bash", "-c"]
# Install iputils
RUN apt-get install net-tools -y
RUN mkdir -p /usr/src
WORKDIR /usr/src/Common
COPY ./Common/package*.json /usr/src/Common/
# Set version in ./Common/package.json to the APP_VERSION
RUN sed -i "s/\"version\": \".*\"/\"version\": \"$APP_VERSION\"/g" /usr/src/Common/package.json
RUN npm install
COPY ./Common /usr/src/Common
ENV PRODUCTION=true
WORKDIR /usr/src/app
RUN npx playwright install --with-deps
# Install app dependencies
COPY ./Ingestor/package*.json /usr/src/app/
RUN npm install
# Expose ports.
# - 3400: OneUptime-ingestor
EXPOSE 3400
{{ if eq .Env.ENVIRONMENT "development" }}
#Run the app
CMD [ "npm", "run", "dev" ]
{{ else }}
# Copy app source
COPY ./Ingestor /usr/src/app
# Bundle app source
RUN npm run compile
#Run the app
CMD [ "npm", "start" ]
{{ end }}

View File

@@ -0,0 +1,79 @@
import IncomingRequestAPI from "../IncomingRequestIngest/API/IncomingRequest";
import MonitorAPI from "./API/Monitor";
import OTelIngestAPI from "./API/OTelIngest";
import Ingestor from "./API/Probe";
import RegisterAPI from "./API/Register";
import ServerMonitorAPI from "./API/ServerMonitor";
import { PromiseVoidFunction } from "Common/Types/FunctionTypes";
import { ClickhouseAppInstance } from "Common/Server/Infrastructure/ClickhouseDatabase";
import PostgresAppInstance from "Common/Server/Infrastructure/PostgresDatabase";
import Redis from "Common/Server/Infrastructure/Redis";
import InfrastructureStatus from "Common/Server/Infrastructure/Status";
import Express, { ExpressApplication } from "Common/Server/Utils/Express";
import logger from "Common/Server/Utils/Logger";
import Realtime from "Common/Server/Utils/Realtime";
import App from "Common/Server/Utils/StartServer";
import Telemetry from "Common/Server/Utils/Telemetry";
import "ejs";
const app: ExpressApplication = Express.getExpressApp();
const APP_NAME: string = "ingestor";
app.use([`/${APP_NAME}`, "/"], RegisterAPI);
app.use([`/${APP_NAME}`, "/"], MonitorAPI);
app.use([`/${APP_NAME}`, "/"], Ingestor);
app.use([`/${APP_NAME}`, "/"], IncomingRequestAPI);
app.use([`/${APP_NAME}`, "/"], OTelIngestAPI);
app.use([`/${APP_NAME}`, "/"], ServerMonitorAPI);
const init: PromiseVoidFunction = async (): Promise<void> => {
try {
const statusCheck: PromiseVoidFunction = async (): Promise<void> => {
return await InfrastructureStatus.checkStatus({
checkClickhouseStatus: true,
checkPostgresStatus: true,
checkRedisStatus: true,
});
};
// Initialize telemetry
Telemetry.init({
serviceName: APP_NAME,
});
// init the app
await App.init({
appName: APP_NAME,
statusOptions: {
liveCheck: statusCheck,
readyCheck: statusCheck,
},
});
// connect to the database.
await PostgresAppInstance.connect();
// connect redis
await Redis.connect();
await ClickhouseAppInstance.connect(
ClickhouseAppInstance.getDatasourceOptions(),
);
await Realtime.init();
// add default routes
await App.addDefaultRoutes();
} catch (err) {
logger.error("App Init Failed:");
logger.error(err);
throw err;
}
};
init().catch((err: Error) => {
logger.error(err);
logger.error("Exiting node process");
process.exit(1);
});

View File

@@ -1,4 +1,3 @@
import { ProbeExpressRequest } from "../Types/Request";
import BadRequestException from "Common/Types/Exception/BadRequestException";
import ProductType from "Common/Types/MeteredPlan/ProductType";
import ObjectID from "Common/Types/ObjectID";
@@ -17,7 +16,7 @@ export interface TelemetryRequest extends ExpressRequest {
export default class TelemetryIngest {
public static async isAuthorizedServiceMiddleware(
req: ProbeExpressRequest,
req: ExpressRequest,
_res: ExpressResponse,
next: NextFunction,
): Promise<void> {

View File

@@ -0,0 +1,32 @@
{
"preset": "ts-jest",
"testPathIgnorePatterns": [
"node_modules",
"dist"
],
"verbose": true,
"globals": {
"ts-jest": {
"tsconfig": "tsconfig.json",
"babelConfig": false
}
},
"moduleFileExtensions": ["ts", "js", "json"],
"transform": {
".(ts|tsx)": "ts-jest"
},
"testEnvironment": "node",
"collectCoverage": false,
"coverageReporters": ["text", "lcov"],
"testRegex": "./Tests/(.*).test.ts",
"collectCoverageFrom": ["./**/*.(tsx||ts)"],
"coverageThreshold": {
"global": {
"lines": 0,
"functions": 0,
"branches": 0,
"statements": 0
}
}
}

View File

@@ -0,0 +1,5 @@
{
"watch": ["./","../Common/Server", "../Common/Types", "../Common/Utils", "../Common/Models"],
"ext": "ts,json,tsx,env,js,jsx,hbs",
"exec": "node --inspect=0.0.0.0:9229 --require ts-node/register Index.ts"
}

4836
OpenTelemetryIngest/package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,30 @@
{
"name": "@oneuptime/ingestor",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"start": "node --require ts-node/register Index.ts",
"compile": "tsc",
"clear-modules": "rm -rf node_modules && rm package-lock.json && npm install",
"dev": "npx nodemon",
"audit": "npm audit --audit-level=low",
"dep-check": "npm install -g depcheck && depcheck ./ --skip-missing=true",
"test": "jest --passWithNoTests"
},
"author": "OneUptime <hello@oneuptime.com> (https://oneuptime.com/)",
"license": "Apache-2.0",
"dependencies": {
"Common": "file:../Common",
"ejs": "^3.1.10",
"protobufjs": "^7.3.2",
"ts-node": "^10.9.1"
},
"devDependencies": {
"@types/jest": "^27.5.0",
"@types/node": "^17.0.31",
"jest": "^28.1.0",
"nodemon": "^2.0.20",
"ts-jest": "^28.0.2"
}
}

View File

@@ -0,0 +1,115 @@
{
"ts-node": {
// these options are overrides used only by ts-node
// same as the --compilerOptions flag and the TS_NODE_COMPILER_OPTIONS environment variable
"compilerOptions": {
"module": "commonjs",
"resolveJsonModule": true,
}
},
"compilerOptions": {
/* Visit https://aka.ms/tsconfig.json to read more about this file */
/* Projects */
// "incremental": true, /* Enable incremental compilation */
// "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */
// "tsBuildInfoFile": "./", /* Specify the folder for .tsbuildinfo incremental compilation files. */
// "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects */
// "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */
// "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */
/* Language and Environment */
"target": "es2017" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
// "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */
"jsx": "react" /* Specify what JSX code is generated. */,
"experimentalDecorators": true /* Enable experimental support for TC39 stage 2 draft decorators. */,
"emitDecoratorMetadata": true /* Emit design-type metadata for decorated declarations in source files. */,
// "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h' */
// "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
// "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using `jsx: react-jsx*`.` */
// "reactNamespace": "", /* Specify the object invoked for `createElement`. This only applies when targeting `react` JSX emit. */
// "noLib": true, /* Disable including any library files, including the default lib.d.ts. */
// "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */
/* Modules */
// "module": "es2022" /* Specify what module code is generated. */,
"rootDir": "" /* Specify the root folder within your source files. */,
"moduleResolution": "node" /* Specify how TypeScript looks up a file from a given module specifier. */,
// "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */
// "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */
// "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */
"typeRoots": [
"./node_modules/@types"
] /* Specify multiple folders that act like `./node_modules/@types`. */,
"types": [
"node",
"jest"
] /* Specify type package names to be included without being referenced in a source file. */,
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
// "resolveJsonModule": true, /* Enable importing .json files */
// "noResolve": true, /* Disallow `import`s, `require`s or `<reference>`s from expanding the number of files TypeScript should add to a project. */
/* JavaScript Support */
// "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */
// "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */
// "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from `node_modules`. Only applicable with `allowJs`. */
/* Emit */
// "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
// "declarationMap": true, /* Create sourcemaps for d.ts files. */
// "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */
"sourceMap": true /* Create source map files for emitted JavaScript files. */,
// "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If `declaration` is true, also designates a file that bundles all .d.ts output. */
"outDir": "build/dist" /* Specify an output folder for all emitted files. */,
// "removeComments": true, /* Disable emitting comments. */
// "noEmit": true, /* Disable emitting files from a compilation. */
// "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
// "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types */
// "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
// "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
// "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */
// "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */
// "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
// "newLine": "crlf", /* Set the newline character for emitting files. */
// "stripInternal": true, /* Disable emitting declarations that have `@internal` in their JSDoc comments. */
// "noEmitHelpers": true, /* Disable generating custom helper functions like `__extends` in compiled output. */
// "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */
// "preserveConstEnums": true, /* Disable erasing `const enum` declarations in generated code. */
// "declarationDir": "./", /* Specify the output directory for generated declaration files. */
// "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */
/* Interop Constraints */
// "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */
// "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */
"esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables `allowSyntheticDefaultImports` for type compatibility. */,
// "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
"forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
/* Type Checking */
"strict": true /* Enable all strict type-checking options. */,
"noImplicitAny": true /* Enable error reporting for expressions and declarations with an implied `any` type.. */,
"strictNullChecks": true /* When type checking, take into account `null` and `undefined`. */,
"strictFunctionTypes": true /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */,
"strictBindCallApply": true /* Check that the arguments for `bind`, `call`, and `apply` methods match the original function. */,
"strictPropertyInitialization": true /* Check for class properties that are declared but not set in the constructor. */,
"noImplicitThis": true /* Enable error reporting when `this` is given the type `any`. */,
"useUnknownInCatchVariables": true /* Type catch clause variables as 'unknown' instead of 'any'. */,
"alwaysStrict": true /* Ensure 'use strict' is always emitted. */,
"noUnusedLocals": true /* Enable error reporting when a local variables aren't read. */,
"noUnusedParameters": true /* Raise an error when a function parameter isn't read */,
"exactOptionalPropertyTypes": true /* Interpret optional property types as written, rather than adding 'undefined'. */,
"noImplicitReturns": true /* Enable error reporting for codepaths that do not explicitly return in a function. */,
"noFallthroughCasesInSwitch": true /* Enable error reporting for fallthrough cases in switch statements. */,
"noUncheckedIndexedAccess": true /* Include 'undefined' in index signature results */,
"noImplicitOverride": true /* Ensure overriding members in derived classes are marked with an override modifier. */,
"noPropertyAccessFromIndexSignature": true /* Enforces using indexed accessors for keys declared using an indexed type */,
// "allowUnusedLabels": true, /* Disable error reporting for unused labels. */
// "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */
/* Completeness */
// "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */
"skipLibCheck": true, /* Skip type checking all .d.ts files. */
"resolveJsonModule": true
}
}

View File

@@ -15,9 +15,10 @@ import { JSONObject } from "Common/Types/JSON";
import ProbeStatusReport from "Common/Types/Probe/ProbeStatusReport";
import Sleep from "Common/Types/Sleep";
import API from "Common/Utils/API";
import { ClusterKey, HasClusterKey } from "Common/Server/EnvironmentConfig";
import { HasClusterKey } from "Common/Server/EnvironmentConfig";
import LocalCache from "Common/Server/Infrastructure/LocalCache";
import logger from "Common/Server/Utils/Logger";
import ClusterKeyAuthorization from "Common/Server/Middleware/ClusterKeyAuthorization";
export default class Register {
public static async isPingMonitoringEnabled(): Promise<boolean> {
@@ -127,7 +128,7 @@ export default class Register {
probeKey: PROBE_KEY,
probeName: PROBE_NAME,
probeDescription: PROBE_DESCRIPTION,
clusterKey: ClusterKey.toString(),
clusterKey: ClusterKeyAuthorization.getClusterKey(),
},
);