From 0233919f3b49e785bbe4bee540cc201e96a7b387 Mon Sep 17 00:00:00 2001 From: Simon Larsen Date: Fri, 2 Feb 2024 12:59:23 +0000 Subject: [PATCH] Fix code formatting and add missing imports --- Common/Types/Date.ts | 8 +++ CommonServer/Utils/StartServer.ts | 1 + Fluentd/fluent.conf | 3 + Ingestor/API/FluentIngest.ts | 116 +++++++++++------------------- Ingestor/API/OTelIngest.ts | 1 + Ingestor/Index.ts | 1 - 6 files changed, 53 insertions(+), 77 deletions(-) diff --git a/Common/Types/Date.ts b/Common/Types/Date.ts index 58d3f76d8a..35f6367216 100644 --- a/Common/Types/Date.ts +++ b/Common/Types/Date.ts @@ -8,6 +8,14 @@ import moment from 'moment-timezone'; export const Moment: typeof moment = moment; export default class OneUptimeDate { + public static getCurrentDateAsUnixNano(): number { + return this.toUnixNano(this.getCurrentDate()); + } + + public static toUnixNano(date: Date): number { + return date.getTime() * 1000000; + } + public static moveDateToTheDayOfWeek( date: Date, moveToWeek: Date, diff --git a/CommonServer/Utils/StartServer.ts b/CommonServer/Utils/StartServer.ts index e75c66b614..4422469bb1 100644 --- a/CommonServer/Utils/StartServer.ts +++ b/CommonServer/Utils/StartServer.ts @@ -99,6 +99,7 @@ const setDefaultHeaders: RequestHandler = ( if (typeof req.body === Typeof.String) { req.body = JSONFunctions.parse(req.body); } + res.header('Access-Control-Allow-Credentials', 'true'); res.header('Access-Control-Allow-Origin', req.headers['origin']); res.header('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS'); diff --git a/Fluentd/fluent.conf b/Fluentd/fluent.conf index 8d4bd7a2b9..311c6bcb62 100644 --- a/Fluentd/fluent.conf +++ b/Fluentd/fluent.conf @@ -19,6 +19,9 @@ headers {"x-oneuptime-service-token":"72f87070-c1c0-11ee-938d-f587493d9c6f"} + content_type application/json + json_array true + @type json diff --git a/Ingestor/API/FluentIngest.ts b/Ingestor/API/FluentIngest.ts index 3329658374..38509b2c14 100644 --- a/Ingestor/API/FluentIngest.ts +++ b/Ingestor/API/FluentIngest.ts @@ -1,19 +1,19 @@ -import { ProductType } from "Model/Models/UsageBilling"; +import { ProductType } from 'Model/Models/UsageBilling'; import Express, { ExpressRequest, ExpressResponse, ExpressRouter, NextFunction, } from 'CommonServer/Utils/Express'; -import { TelemetryRequest } from "../Middleware/TelemetryIngest"; -import Response from "CommonServer/Utils/Response"; -import logger from "CommonServer/Utils/Logger"; - -// import { JSONArray, JSONObject } from "Common/Types/JSON"; -// import Log from "Model/AnalyticsModels/Log"; -// import OTelIngestService from "../Service/OTelIngest"; -// import LogService from "CommonServer/Services/LogService"; -// import OneUptimeDate from "Common/Types/Date"; +import TelemetryIngest, { + TelemetryRequest, +} from '../Middleware/TelemetryIngest'; +import Response from 'CommonServer/Utils/Response'; +import logger from 'CommonServer/Utils/Logger'; +import { JSONObject } from 'Common/Types/JSON'; +import Log from 'Model/AnalyticsModels/Log'; +import LogService from 'CommonServer/Services/LogService'; +import OneUptimeDate from 'Common/Types/Date'; export class FluentRequestMiddleware { public static async getProductType( @@ -23,6 +23,7 @@ export class FluentRequestMiddleware { ): Promise { try { (req as TelemetryRequest).productType = ProductType.Logs; + return next(); } catch (err) { return next(err); } @@ -33,8 +34,8 @@ const router: ExpressRouter = Express.getRouter(); router.post( '/fluentd/v1/logs', - // FluentRequestMiddleware.getProductType, - // TelemetryIngest.isAuthorizedServiceMiddleware, + FluentRequestMiddleware.getProductType, + TelemetryIngest.isAuthorizedServiceMiddleware, async ( req: ExpressRequest, res: ExpressResponse, @@ -43,77 +44,40 @@ router.post( try { logger.info('Fluent Ingestor API called'); - logger.info('Request body: '); - logger.info(req.body); + const dbLogs: Array = []; - // const resourceLogs: JSONArray = req.body[ - // 'resourceLogs' - // ] as JSONArray; + const logItems: Array = req.body as Array< + JSONObject | string + >; - // const dbLogs: Array = []; + for (let logItem of logItems) { + const dbLog: Log = new Log(); - // for (const resourceLog of resourceLogs) { - // const scopeLogs: JSONArray = resourceLog[ - // 'scopeLogs' - // ] as JSONArray; + dbLog.projectId = (req as TelemetryRequest).projectId; + dbLog.serviceId = (req as TelemetryRequest).serviceId; + dbLog.severityNumber = '0'; + const currentTimeAndDate: Date = OneUptimeDate.getCurrentDate(); + dbLog.timeUnixNano = + OneUptimeDate.toUnixNano(currentTimeAndDate); + dbLog.time = currentTimeAndDate; - // for (const scopeLog of scopeLogs) { - // const logRecords: JSONArray = scopeLog[ - // 'logRecords' - // ] as JSONArray; + dbLog.severityText = 'Information'; - // for (const log of logRecords) { - // const dbLog: Log = new Log(); + if (typeof logItem !== 'string') { + logItem = JSON.stringify(logItem); + } - // /* - // Example: + dbLog.body = logItem as string; - // { - // "timeUnixNano":"1698069643739368000", - // "severityNumber":"SEVERITY_NUMBER_INFO", - // "severityText":"Information", - // "body":{ - // "stringValue":"Application is shutting down..." - // }, - // "traceId":"", - // "spanId":"", - // "observedTimeUnixNano":"1698069643739368000" - // } - // */ + dbLogs.push(dbLog); + } - // dbLog.projectId = (req as TelemetryRequest).projectId; - // dbLog.serviceId = (req as TelemetryRequest).serviceId; - - // dbLog.timeUnixNano = log['timeUnixNano'] as number; - // dbLog.time = OneUptimeDate.fromUnixNano( - // log['timeUnixNano'] as number - // ); - // dbLog.severityNumber = log['severityNumber'] as string; - // dbLog.severityText = log['severityText'] as string; - - // const logBody: JSONObject = log['body'] as JSONObject; - - // dbLog.body = logBody['stringValue'] as string; - - // dbLog.traceId = log['traceId'] as string; - // dbLog.spanId = log['spanId'] as string; - - // // We need to convert this to date. - // dbLog.attributes = OTelIngestService.getKeyValues( - // log['attributes'] as JSONArray - // ); - - // dbLogs.push(dbLog); - // } - // } - // } - - // await LogService.createMany({ - // items: dbLogs, - // props: { - // isRoot: true, - // }, - // }); + await LogService.createMany({ + items: dbLogs, + props: { + isRoot: true, + }, + }); return Response.sendEmptyResponse(req, res); } catch (err) { @@ -122,4 +86,4 @@ router.post( } ); -export default router; \ No newline at end of file +export default router; diff --git a/Ingestor/API/OTelIngest.ts b/Ingestor/API/OTelIngest.ts index 6ed33953a8..83dfa556da 100644 --- a/Ingestor/API/OTelIngest.ts +++ b/Ingestor/API/OTelIngest.ts @@ -77,6 +77,7 @@ class OpenTelemetryRequestMiddleware { } (req as TelemetryRequest).productType = productType; + next(); } catch (err) { return next(err); } diff --git a/Ingestor/Index.ts b/Ingestor/Index.ts index d17de83705..a6121c448b 100644 --- a/Ingestor/Index.ts +++ b/Ingestor/Index.ts @@ -32,7 +32,6 @@ const init: () => Promise = async (): Promise => { // init the app await App(APP_NAME); - // connect to the database. await PostgresAppInstance.connect( PostgresAppInstance.getDatasourceOptions()