mirror of
https://github.com/OneUptime/oneuptime.git
synced 2026-04-06 00:32:12 +02:00
chore(fluent-ingest): remove FluentIngest service and all related sources, configs and artifacts
This commit is contained in:
@@ -1,56 +0,0 @@
|
||||
.git
|
||||
|
||||
node_modules
|
||||
# See https://help.github.com/ignore-files/ for more about ignoring files.
|
||||
|
||||
# dependencies
|
||||
/node_modules
|
||||
node_modules
|
||||
|
||||
.idea
|
||||
# testing
|
||||
/coverage
|
||||
|
||||
# production
|
||||
/build
|
||||
|
||||
# misc
|
||||
.DS_Store
|
||||
|
||||
env.js
|
||||
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
|
||||
yarn.lock
|
||||
Untitled-1
|
||||
*.local.sh
|
||||
*.local.yaml
|
||||
run
|
||||
stop
|
||||
|
||||
nohup.out*
|
||||
|
||||
encrypted-credentials.tar
|
||||
encrypted-credentials/
|
||||
|
||||
_README.md
|
||||
|
||||
# Important Add production values to gitignore.
|
||||
values-saas-production.yaml
|
||||
kubernetes/values-saas-production.yaml
|
||||
|
||||
/private
|
||||
|
||||
/tls_cert.pem
|
||||
/tls_key.pem
|
||||
/keys
|
||||
|
||||
temp_readme.md
|
||||
|
||||
tests/coverage
|
||||
|
||||
settings.json
|
||||
|
||||
GoSDK/tester/
|
||||
1
FluentIngest/.gitattributes
vendored
1
FluentIngest/.gitattributes
vendored
@@ -1 +0,0 @@
|
||||
*.js text eol=lf
|
||||
16
FluentIngest/.gitignore
vendored
16
FluentIngest/.gitignore
vendored
@@ -1,16 +0,0 @@
|
||||
# See https://help.github.com/ignore-files/ for more about ignoring files.
|
||||
|
||||
# dependencies
|
||||
#/backend/node_modules
|
||||
/kubernetes
|
||||
/node_modules
|
||||
.idea
|
||||
|
||||
# misc
|
||||
.DS_Store
|
||||
|
||||
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
yarn.lock
|
||||
@@ -1,151 +0,0 @@
|
||||
import TelemetryIngest, {
|
||||
TelemetryRequest,
|
||||
} from "Common/Server/Middleware/TelemetryIngest";
|
||||
import ProductType from "Common/Types/MeteredPlan/ProductType";
|
||||
import Express, {
|
||||
ExpressRequest,
|
||||
ExpressResponse,
|
||||
ExpressRouter,
|
||||
NextFunction,
|
||||
} from "Common/Server/Utils/Express";
|
||||
import Response from "Common/Server/Utils/Response";
|
||||
import FluentIngestQueueService from "../Services/Queue/FluentIngestQueueService";
|
||||
import ClusterKeyAuthorization from "Common/Server/Middleware/ClusterKeyAuthorization";
|
||||
import BadRequestException from "Common/Types/Exception/BadRequestException";
|
||||
|
||||
export class FluentRequestMiddleware {
|
||||
public static async getProductType(
|
||||
req: ExpressRequest,
|
||||
_res: ExpressResponse,
|
||||
next: NextFunction,
|
||||
): Promise<void> {
|
||||
try {
|
||||
(req as TelemetryRequest).productType = ProductType.Logs;
|
||||
return next();
|
||||
} catch (err) {
|
||||
return next(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const router: ExpressRouter = Express.getRouter();
|
||||
|
||||
router.post(
|
||||
"/fluentd/v1/logs",
|
||||
FluentRequestMiddleware.getProductType,
|
||||
TelemetryIngest.isAuthorizedServiceMiddleware,
|
||||
async (
|
||||
req: ExpressRequest,
|
||||
res: ExpressResponse,
|
||||
next: NextFunction,
|
||||
): Promise<void> => {
|
||||
try {
|
||||
if (!(req as TelemetryRequest).projectId) {
|
||||
throw new BadRequestException(
|
||||
"Invalid request - projectId not found in request.",
|
||||
);
|
||||
}
|
||||
|
||||
req.body = req.body.toJSON ? req.body.toJSON() : req.body;
|
||||
|
||||
// Return response immediately
|
||||
Response.sendEmptySuccessResponse(req, res);
|
||||
|
||||
// Add to queue for asynchronous processing
|
||||
await FluentIngestQueueService.addFluentIngestJob(
|
||||
req as TelemetryRequest,
|
||||
);
|
||||
|
||||
return;
|
||||
} catch (err) {
|
||||
return next(err);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// Queue stats endpoint
|
||||
router.get(
|
||||
"/fluent/queue/stats",
|
||||
ClusterKeyAuthorization.isAuthorizedServiceMiddleware,
|
||||
async (
|
||||
req: ExpressRequest,
|
||||
res: ExpressResponse,
|
||||
next: NextFunction,
|
||||
): Promise<void> => {
|
||||
try {
|
||||
const stats: {
|
||||
waiting: number;
|
||||
active: number;
|
||||
completed: number;
|
||||
failed: number;
|
||||
delayed: number;
|
||||
total: number;
|
||||
} = await FluentIngestQueueService.getQueueStats();
|
||||
return Response.sendJsonObjectResponse(req, res, stats);
|
||||
} catch (err) {
|
||||
return next(err);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// Queue size endpoint
|
||||
router.get(
|
||||
"/fluent/queue/size",
|
||||
ClusterKeyAuthorization.isAuthorizedServiceMiddleware,
|
||||
async (
|
||||
req: ExpressRequest,
|
||||
res: ExpressResponse,
|
||||
next: NextFunction,
|
||||
): Promise<void> => {
|
||||
try {
|
||||
const size: number = await FluentIngestQueueService.getQueueSize();
|
||||
return Response.sendJsonObjectResponse(req, res, { size });
|
||||
} catch (err) {
|
||||
return next(err);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// Queue failed jobs endpoint
|
||||
router.get(
|
||||
"/fluent/queue/failed",
|
||||
ClusterKeyAuthorization.isAuthorizedServiceMiddleware,
|
||||
async (
|
||||
req: ExpressRequest,
|
||||
res: ExpressResponse,
|
||||
next: NextFunction,
|
||||
): Promise<void> => {
|
||||
try {
|
||||
// Parse pagination parameters from query string
|
||||
const start: number = parseInt(req.query["start"] as string) || 0;
|
||||
const end: number = parseInt(req.query["end"] as string) || 100;
|
||||
|
||||
const failedJobs: Array<{
|
||||
id: string;
|
||||
name: string;
|
||||
data: any;
|
||||
failedReason: string;
|
||||
stackTrace?: string;
|
||||
processedOn: Date | null;
|
||||
finishedOn: Date | null;
|
||||
attemptsMade: number;
|
||||
}> = await FluentIngestQueueService.getFailedJobs({
|
||||
start,
|
||||
end,
|
||||
});
|
||||
|
||||
return Response.sendJsonObjectResponse(req, res, {
|
||||
failedJobs,
|
||||
pagination: {
|
||||
start,
|
||||
end,
|
||||
count: failedJobs.length,
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
return next(err);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
export default router;
|
||||
@@ -1,37 +0,0 @@
|
||||
import Express, {
|
||||
ExpressRequest,
|
||||
ExpressResponse,
|
||||
ExpressRouter,
|
||||
NextFunction,
|
||||
} from "Common/Server/Utils/Express";
|
||||
import FluentIngestQueueService from "../Services/Queue/FluentIngestQueueService";
|
||||
// import ClusterKeyAuthorization from "Common/Server/Middleware/ClusterKeyAuthorization";
|
||||
|
||||
const router: ExpressRouter = Express.getRouter();
|
||||
|
||||
/**
|
||||
* JSON metrics endpoint for KEDA autoscaling
|
||||
* Returns queue size as JSON for KEDA metrics-api scaler
|
||||
*/
|
||||
router.get(
|
||||
"/metrics/queue-size",
|
||||
// ClusterKeyAuthorization.isAuthorizedServiceMiddleware, // Temporarily disabled for KEDA debugging
|
||||
async (
|
||||
_req: ExpressRequest,
|
||||
res: ExpressResponse,
|
||||
next: NextFunction,
|
||||
): Promise<void> => {
|
||||
try {
|
||||
const queueSize: number = await FluentIngestQueueService.getQueueSize();
|
||||
|
||||
res.setHeader("Content-Type", "application/json");
|
||||
res.status(200).json({
|
||||
queueSize: queueSize,
|
||||
});
|
||||
} catch (err) {
|
||||
return next(err);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
export default router;
|
||||
@@ -1,9 +0,0 @@
|
||||
let concurrency: string | number =
|
||||
process.env["FLUENT_INGEST_CONCURRENCY"] || 100;
|
||||
|
||||
if (typeof concurrency === "string") {
|
||||
const parsed: number = parseInt(concurrency, 10);
|
||||
concurrency = !isNaN(parsed) && parsed > 0 ? parsed : 100;
|
||||
}
|
||||
|
||||
export const FLUENT_INGEST_CONCURRENCY: number = concurrency as number;
|
||||
@@ -1,88 +0,0 @@
|
||||
#
|
||||
# OneUptime-FluentIngest Dockerfile
|
||||
#
|
||||
|
||||
# Pull base image nodejs image.
|
||||
FROM public.ecr.aws/docker/library/node:23.8
|
||||
RUN mkdir /tmp/npm && chmod 2777 /tmp/npm && chown 1000:1000 /tmp/npm && npm config set cache /tmp/npm --global
|
||||
|
||||
RUN npm config set fetch-retries 5
|
||||
RUN npm config set fetch-retry-mintimeout 100000
|
||||
RUN npm config set fetch-retry-maxtimeout 600000
|
||||
|
||||
|
||||
ARG GIT_SHA
|
||||
ARG APP_VERSION
|
||||
ARG IS_ENTERPRISE_EDITION=false
|
||||
|
||||
ENV GIT_SHA=${GIT_SHA}
|
||||
ENV APP_VERSION=${APP_VERSION}
|
||||
ENV IS_ENTERPRISE_EDITION=${IS_ENTERPRISE_EDITION}
|
||||
ENV PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1
|
||||
ENV NODE_OPTIONS="--use-openssl-ca"
|
||||
|
||||
## Add Intermediate Certs
|
||||
COPY ./SslCertificates /usr/local/share/ca-certificates
|
||||
RUN update-ca-certificates
|
||||
|
||||
|
||||
# IF APP_VERSION is not set, set it to 1.0.0
|
||||
RUN if [ -z "$APP_VERSION" ]; then export APP_VERSION=1.0.0; fi
|
||||
|
||||
|
||||
RUN apt-get update
|
||||
|
||||
# Install bash.
|
||||
RUN apt-get install bash -y && apt-get install curl -y && apt-get install iputils-ping -y
|
||||
|
||||
# Install python
|
||||
RUN apt-get update && apt-get install -y .gyp python3 make g++
|
||||
|
||||
# Install playwright dependencies
|
||||
RUN apt-get install -y libnss3 libnspr4 libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 libxkbcommon0 libxcomposite1 libxdamage1 libxfixes3 libxrandr2 libgbm1 libgtk-3-0 libpango-1.0-0 libcairo2 libgdk-pixbuf2.0-0 libasound2 libatspi2.0-0
|
||||
|
||||
#Use bash shell by default
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
|
||||
# Install iputils
|
||||
RUN apt-get install net-tools -y
|
||||
|
||||
RUN mkdir -p /usr/src
|
||||
|
||||
WORKDIR /usr/src/Common
|
||||
COPY ./Common/package*.json /usr/src/Common/
|
||||
# Set version in ./Common/package.json to the APP_VERSION
|
||||
RUN sed -i "s/\"version\": \".*\"/\"version\": \"$APP_VERSION\"/g" /usr/src/Common/package.json
|
||||
RUN npm install
|
||||
COPY ./Common /usr/src/Common
|
||||
|
||||
|
||||
|
||||
ENV PRODUCTION=true
|
||||
|
||||
WORKDIR /usr/src/app
|
||||
|
||||
RUN PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=0 npx playwright install --with-deps
|
||||
|
||||
# Install app dependencies
|
||||
COPY ./FluentIngest/package*.json /usr/src/app/
|
||||
RUN npm install
|
||||
|
||||
# Expose ports.
|
||||
# - 3401: OneUptime-fluentd
|
||||
EXPOSE 3401
|
||||
|
||||
{{ if eq .Env.ENVIRONMENT "development" }}
|
||||
#Run the app
|
||||
CMD [ "npm", "run", "dev" ]
|
||||
{{ else }}
|
||||
# Copy app source
|
||||
COPY ./FluentIngest /usr/src/app
|
||||
# Bundle app source
|
||||
RUN npm run compile
|
||||
# Set permission to write logs and cache in case container run as non root
|
||||
RUN chown -R 1000:1000 "/tmp/npm" && chmod -R 2777 "/tmp/npm"
|
||||
#Run the app
|
||||
CMD [ "npm", "start" ]
|
||||
{{ end }}
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
import FluentIngestAPI from "./API/FluentIngest";
|
||||
import MetricsAPI from "./API/Metrics";
|
||||
import { PromiseVoidFunction } from "Common/Types/FunctionTypes";
|
||||
import { ClickhouseAppInstance } from "Common/Server/Infrastructure/ClickhouseDatabase";
|
||||
import PostgresAppInstance from "Common/Server/Infrastructure/PostgresDatabase";
|
||||
import Redis from "Common/Server/Infrastructure/Redis";
|
||||
import InfrastructureStatus from "Common/Server/Infrastructure/Status";
|
||||
import Express, { ExpressApplication } from "Common/Server/Utils/Express";
|
||||
import logger from "Common/Server/Utils/Logger";
|
||||
import Realtime from "Common/Server/Utils/Realtime";
|
||||
import App from "Common/Server/Utils/StartServer";
|
||||
import Telemetry from "Common/Server/Utils/Telemetry";
|
||||
import "./Jobs/FluentIngest/ProcessFluentIngest";
|
||||
import { FLUENT_INGEST_CONCURRENCY } from "./Config";
|
||||
|
||||
const app: ExpressApplication = Express.getExpressApp();
|
||||
|
||||
const APP_NAME: string = "fluent-ingest";
|
||||
|
||||
app.use([`/${APP_NAME}`, "/"], FluentIngestAPI);
|
||||
app.use([`/${APP_NAME}`, "/"], MetricsAPI);
|
||||
|
||||
const init: PromiseVoidFunction = async (): Promise<void> => {
|
||||
try {
|
||||
const statusCheck: PromiseVoidFunction = async (): Promise<void> => {
|
||||
return await InfrastructureStatus.checkStatusWithRetry({
|
||||
checkClickhouseStatus: true,
|
||||
checkPostgresStatus: true,
|
||||
checkRedisStatus: true,
|
||||
retryCount: 3,
|
||||
});
|
||||
};
|
||||
|
||||
// Initialize telemetry
|
||||
Telemetry.init({
|
||||
serviceName: APP_NAME,
|
||||
});
|
||||
|
||||
logger.info(
|
||||
`FluentIngest Service - Queue concurrency: ${FLUENT_INGEST_CONCURRENCY}`,
|
||||
);
|
||||
|
||||
// init the app
|
||||
await App.init({
|
||||
appName: APP_NAME,
|
||||
statusOptions: {
|
||||
liveCheck: statusCheck,
|
||||
readyCheck: statusCheck,
|
||||
},
|
||||
});
|
||||
|
||||
// connect to the database.
|
||||
await PostgresAppInstance.connect();
|
||||
|
||||
// connect redis
|
||||
await Redis.connect();
|
||||
|
||||
await ClickhouseAppInstance.connect(
|
||||
ClickhouseAppInstance.getDatasourceOptions(),
|
||||
);
|
||||
|
||||
await Realtime.init();
|
||||
|
||||
// add default routes
|
||||
await App.addDefaultRoutes();
|
||||
} catch (err) {
|
||||
logger.error("App Init Failed:");
|
||||
logger.error(err);
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
init().catch((err: Error) => {
|
||||
logger.error(err);
|
||||
logger.error("Exiting node process");
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -1,143 +0,0 @@
|
||||
import { FluentIngestJobData } from "../../Services/Queue/FluentIngestQueueService";
|
||||
import logger from "Common/Server/Utils/Logger";
|
||||
import { QueueJob, QueueName } from "Common/Server/Infrastructure/Queue";
|
||||
import QueueWorker from "Common/Server/Infrastructure/QueueWorker";
|
||||
import ObjectID from "Common/Types/ObjectID";
|
||||
import OneUptimeDate from "Common/Types/Date";
|
||||
import { JSONObject } from "Common/Types/JSON";
|
||||
import LogService from "Common/Server/Services/LogService";
|
||||
import LogSeverity from "Common/Types/Log/LogSeverity";
|
||||
import OTelIngestService from "Common/Server/Services/OpenTelemetryIngestService";
|
||||
import { FLUENT_INGEST_CONCURRENCY } from "../../Config";
|
||||
|
||||
interface FluentIngestProcessData {
|
||||
projectId: ObjectID;
|
||||
requestBody: JSONObject;
|
||||
requestHeaders: JSONObject;
|
||||
}
|
||||
|
||||
const FLUENT_INGEST_LOG_FLUSH_BATCH_SIZE: number = 500;
|
||||
|
||||
// Set up the worker for processing fluent ingest queue
|
||||
QueueWorker.getWorker(
|
||||
QueueName.FluentIngest,
|
||||
async (job: QueueJob): Promise<void> => {
|
||||
logger.debug(`Processing fluent ingestion job: ${job.name}`);
|
||||
|
||||
try {
|
||||
const jobData: FluentIngestJobData = job.data as FluentIngestJobData;
|
||||
|
||||
// Pass job data directly to processing function
|
||||
await processFluentIngestFromQueue({
|
||||
projectId: new ObjectID(jobData.projectId),
|
||||
requestBody: jobData.requestBody,
|
||||
requestHeaders: jobData.requestHeaders,
|
||||
});
|
||||
|
||||
logger.debug(`Successfully processed fluent ingestion job: ${job.name}`);
|
||||
} catch (error) {
|
||||
logger.error(`Error processing fluent ingestion job:`);
|
||||
logger.error(error);
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
{ concurrency: FLUENT_INGEST_CONCURRENCY },
|
||||
);
|
||||
|
||||
async function processFluentIngestFromQueue(
|
||||
data: FluentIngestProcessData,
|
||||
): Promise<void> {
|
||||
const dbLogs: Array<JSONObject> = [];
|
||||
|
||||
let logItems: Array<JSONObject | string> | JSONObject = data.requestBody as
|
||||
| Array<JSONObject | string>
|
||||
| JSONObject;
|
||||
|
||||
let oneuptimeServiceName: string | string[] | undefined = data.requestHeaders[
|
||||
"x-oneuptime-service-name"
|
||||
] as string | string[] | undefined;
|
||||
|
||||
if (!oneuptimeServiceName) {
|
||||
oneuptimeServiceName = "Unknown Service";
|
||||
}
|
||||
|
||||
const telemetryService: {
|
||||
serviceId: ObjectID;
|
||||
dataRententionInDays: number;
|
||||
} = await OTelIngestService.telemetryServiceFromName({
|
||||
serviceName: oneuptimeServiceName as string,
|
||||
projectId: data.projectId,
|
||||
});
|
||||
|
||||
if (
|
||||
logItems &&
|
||||
typeof logItems === "object" &&
|
||||
(logItems as JSONObject)["json"]
|
||||
) {
|
||||
logItems = (logItems as JSONObject)["json"] as
|
||||
| Array<JSONObject | string>
|
||||
| JSONObject;
|
||||
}
|
||||
|
||||
if (!Array.isArray(logItems)) {
|
||||
logItems = [logItems];
|
||||
}
|
||||
|
||||
for (const logItem of logItems) {
|
||||
const logBody: string =
|
||||
typeof logItem === "string" ? logItem : JSON.stringify(logItem);
|
||||
|
||||
const ingestionDate: Date = OneUptimeDate.getCurrentDate();
|
||||
const ingestionIso: string = OneUptimeDate.toString(ingestionDate);
|
||||
const timeUnixNano: number = OneUptimeDate.toUnixNano(ingestionDate);
|
||||
|
||||
const logRow: JSONObject = {
|
||||
_id: ObjectID.generate().toString(),
|
||||
createdAt: ingestionIso,
|
||||
updatedAt: ingestionIso,
|
||||
projectId: data.projectId.toString(),
|
||||
serviceId: telemetryService.serviceId.toString(),
|
||||
time: ingestionIso,
|
||||
timeUnixNano: Math.trunc(timeUnixNano).toString(),
|
||||
severityNumber: 0,
|
||||
severityText: LogSeverity.Unspecified,
|
||||
attributes: {},
|
||||
attributeKeys: [],
|
||||
traceId: "",
|
||||
spanId: "",
|
||||
body: logBody,
|
||||
};
|
||||
|
||||
dbLogs.push(logRow);
|
||||
|
||||
if (dbLogs.length >= FLUENT_INGEST_LOG_FLUSH_BATCH_SIZE) {
|
||||
await flushLogBuffer(dbLogs);
|
||||
}
|
||||
}
|
||||
|
||||
await flushLogBuffer(dbLogs, true);
|
||||
}
|
||||
|
||||
logger.debug("Fluent ingest worker initialized");
|
||||
|
||||
async function flushLogBuffer(
|
||||
logs: Array<JSONObject>,
|
||||
force: boolean = false,
|
||||
): Promise<void> {
|
||||
while (
|
||||
logs.length >= FLUENT_INGEST_LOG_FLUSH_BATCH_SIZE ||
|
||||
(force && logs.length > 0)
|
||||
) {
|
||||
const batchSize: number = Math.min(
|
||||
logs.length,
|
||||
FLUENT_INGEST_LOG_FLUSH_BATCH_SIZE,
|
||||
);
|
||||
const batch: Array<JSONObject> = logs.splice(0, batchSize);
|
||||
|
||||
if (batch.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
await LogService.insertJsonRows(batch);
|
||||
}
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
import { TelemetryRequest } from "Common/Server/Middleware/TelemetryIngest";
|
||||
import Queue, { QueueName } from "Common/Server/Infrastructure/Queue";
|
||||
import { JSONObject } from "Common/Types/JSON";
|
||||
import OneUptimeDate from "Common/Types/Date";
|
||||
import logger from "Common/Server/Utils/Logger";
|
||||
|
||||
export interface FluentIngestJobData {
|
||||
projectId: string;
|
||||
requestBody: JSONObject;
|
||||
requestHeaders: Record<string, string>;
|
||||
ingestionTimestamp: Date;
|
||||
}
|
||||
|
||||
export default class FluentIngestQueueService {
|
||||
public static async addFluentIngestJob(req: TelemetryRequest): Promise<void> {
|
||||
try {
|
||||
const jobData: FluentIngestJobData = {
|
||||
projectId: req.projectId.toString(),
|
||||
requestBody: req.body,
|
||||
requestHeaders: req.headers as Record<string, string>,
|
||||
ingestionTimestamp: OneUptimeDate.getCurrentDate(),
|
||||
};
|
||||
|
||||
const jobId: string = `fluent-${req.projectId?.toString()}-${OneUptimeDate.getCurrentDateAsUnixNano()}`;
|
||||
|
||||
await Queue.addJob(
|
||||
QueueName.FluentIngest,
|
||||
jobId,
|
||||
"ProcessFluentIngest",
|
||||
jobData as unknown as JSONObject,
|
||||
);
|
||||
|
||||
logger.debug(`Added fluent ingestion job: ${jobId}`);
|
||||
} catch (error) {
|
||||
logger.error(`Error adding fluent ingestion job:`);
|
||||
logger.error(error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
public static async getQueueSize(): Promise<number> {
|
||||
return Queue.getQueueSize(QueueName.FluentIngest);
|
||||
}
|
||||
|
||||
public static async getQueueStats(): Promise<{
|
||||
waiting: number;
|
||||
active: number;
|
||||
completed: number;
|
||||
failed: number;
|
||||
delayed: number;
|
||||
total: number;
|
||||
}> {
|
||||
return Queue.getQueueStats(QueueName.FluentIngest);
|
||||
}
|
||||
|
||||
public static getFailedJobs(options?: {
|
||||
start?: number;
|
||||
end?: number;
|
||||
}): Promise<
|
||||
Array<{
|
||||
id: string;
|
||||
name: string;
|
||||
data: JSONObject;
|
||||
failedReason: string;
|
||||
stackTrace?: string;
|
||||
processedOn: Date | null;
|
||||
finishedOn: Date | null;
|
||||
attemptsMade: number;
|
||||
}>
|
||||
> {
|
||||
return Queue.getFailedJobs(QueueName.FluentIngest, options);
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
{
|
||||
|
||||
"preset": "ts-jest",
|
||||
"testPathIgnorePatterns": [
|
||||
"node_modules",
|
||||
"dist"
|
||||
],
|
||||
"verbose": true,
|
||||
"globals": {
|
||||
"ts-jest": {
|
||||
"tsconfig": "tsconfig.json",
|
||||
"babelConfig": false
|
||||
}
|
||||
},
|
||||
"moduleFileExtensions": ["ts", "js", "json"],
|
||||
"transform": {
|
||||
".(ts|tsx)": "ts-jest"
|
||||
},
|
||||
"testEnvironment": "node",
|
||||
"collectCoverage": false,
|
||||
"coverageReporters": ["text", "lcov"],
|
||||
"testRegex": "./Tests/(.*).test.ts",
|
||||
"collectCoverageFrom": ["./**/*.(tsx||ts)"],
|
||||
"coverageThreshold": {
|
||||
"global": {
|
||||
"lines": 0,
|
||||
"functions": 0,
|
||||
"branches": 0,
|
||||
"statements": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
{
|
||||
"watch": ["./","../Common/Server", "../Common/Types", "../Common/Utils", "../Common/Models"],
|
||||
"ext": "ts,tsx",
|
||||
"ignore": ["./node_modules/**", "./public/**", "./bin/**", "./build/**"],
|
||||
"watchOptions": {"useFsEvents": false, "interval": 500},
|
||||
"env": {"TS_NODE_TRANSPILE_ONLY": "1", "TS_NODE_FILES": "false"},
|
||||
"exec": "node -r ts-node/register/transpile-only Index.ts"
|
||||
}
|
||||
4560
FluentIngest/package-lock.json
generated
4560
FluentIngest/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -1,28 +0,0 @@
|
||||
{
|
||||
"name": "@oneuptime/fluent-ingest",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"start": "export NODE_OPTIONS='--max-old-space-size=8096' && node --require ts-node/register Index.ts",
|
||||
"compile": "tsc",
|
||||
"clear-modules": "rm -rf node_modules && rm package-lock.json && npm install",
|
||||
"dev": "npx nodemon",
|
||||
"audit": "npm audit --audit-level=low",
|
||||
"dep-check": "npm install -g depcheck && depcheck ./ --skip-missing=true",
|
||||
"test": "jest --passWithNoTests"
|
||||
},
|
||||
"author": "OneUptime <hello@oneuptime.com> (https://oneuptime.com/)",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"Common": "file:../Common",
|
||||
"ts-node": "^10.9.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/jest": "^27.5.0",
|
||||
"@types/node": "^17.0.31",
|
||||
"jest": "^28.1.0",
|
||||
"nodemon": "^2.0.20",
|
||||
"ts-jest": "^28.0.2"
|
||||
}
|
||||
}
|
||||
@@ -1,115 +0,0 @@
|
||||
{
|
||||
"ts-node": {
|
||||
// these options are overrides used only by ts-node
|
||||
// same as the --compilerOptions flag and the TS_NODE_COMPILER_OPTIONS environment variable
|
||||
"compilerOptions": {
|
||||
"module": "commonjs",
|
||||
"resolveJsonModule": true,
|
||||
}
|
||||
},
|
||||
"compilerOptions": {
|
||||
/* Visit https://aka.ms/tsconfig.json to read more about this file */
|
||||
|
||||
/* Projects */
|
||||
// "incremental": true, /* Enable incremental compilation */
|
||||
// "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */
|
||||
// "tsBuildInfoFile": "./", /* Specify the folder for .tsbuildinfo incremental compilation files. */
|
||||
// "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects */
|
||||
// "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */
|
||||
// "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */
|
||||
|
||||
/* Language and Environment */
|
||||
"target": "es2017" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
|
||||
// "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */
|
||||
"jsx": "react" /* Specify what JSX code is generated. */,
|
||||
"experimentalDecorators": true /* Enable experimental support for TC39 stage 2 draft decorators. */,
|
||||
"emitDecoratorMetadata": true /* Emit design-type metadata for decorated declarations in source files. */,
|
||||
// "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h' */
|
||||
// "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
|
||||
// "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using `jsx: react-jsx*`.` */
|
||||
// "reactNamespace": "", /* Specify the object invoked for `createElement`. This only applies when targeting `react` JSX emit. */
|
||||
// "noLib": true, /* Disable including any library files, including the default lib.d.ts. */
|
||||
// "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */
|
||||
|
||||
/* Modules */
|
||||
// "module": "es2022" /* Specify what module code is generated. */,
|
||||
"rootDir": "" /* Specify the root folder within your source files. */,
|
||||
"moduleResolution": "node" /* Specify how TypeScript looks up a file from a given module specifier. */,
|
||||
// "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */
|
||||
// "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */
|
||||
// "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */
|
||||
"typeRoots": [
|
||||
"./node_modules/@types"
|
||||
] /* Specify multiple folders that act like `./node_modules/@types`. */,
|
||||
"types": [
|
||||
"node",
|
||||
"jest"
|
||||
] /* Specify type package names to be included without being referenced in a source file. */,
|
||||
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
|
||||
// "resolveJsonModule": true, /* Enable importing .json files */
|
||||
// "noResolve": true, /* Disallow `import`s, `require`s or `<reference>`s from expanding the number of files TypeScript should add to a project. */
|
||||
|
||||
/* JavaScript Support */
|
||||
// "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */
|
||||
// "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */
|
||||
// "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from `node_modules`. Only applicable with `allowJs`. */
|
||||
|
||||
/* Emit */
|
||||
// "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
|
||||
// "declarationMap": true, /* Create sourcemaps for d.ts files. */
|
||||
// "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */
|
||||
"sourceMap": true /* Create source map files for emitted JavaScript files. */,
|
||||
// "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If `declaration` is true, also designates a file that bundles all .d.ts output. */
|
||||
"outDir": "build/dist" /* Specify an output folder for all emitted files. */,
|
||||
// "removeComments": true, /* Disable emitting comments. */
|
||||
// "noEmit": true, /* Disable emitting files from a compilation. */
|
||||
// "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
|
||||
// "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types */
|
||||
// "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
|
||||
// "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */
|
||||
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
|
||||
// "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */
|
||||
// "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */
|
||||
// "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
|
||||
// "newLine": "crlf", /* Set the newline character for emitting files. */
|
||||
// "stripInternal": true, /* Disable emitting declarations that have `@internal` in their JSDoc comments. */
|
||||
// "noEmitHelpers": true, /* Disable generating custom helper functions like `__extends` in compiled output. */
|
||||
// "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */
|
||||
// "preserveConstEnums": true, /* Disable erasing `const enum` declarations in generated code. */
|
||||
// "declarationDir": "./", /* Specify the output directory for generated declaration files. */
|
||||
// "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */
|
||||
|
||||
/* Interop Constraints */
|
||||
// "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */
|
||||
// "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */
|
||||
"esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables `allowSyntheticDefaultImports` for type compatibility. */,
|
||||
// "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
|
||||
"forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
|
||||
|
||||
/* Type Checking */
|
||||
"strict": true /* Enable all strict type-checking options. */,
|
||||
"noImplicitAny": true /* Enable error reporting for expressions and declarations with an implied `any` type.. */,
|
||||
"strictNullChecks": true /* When type checking, take into account `null` and `undefined`. */,
|
||||
"strictFunctionTypes": true /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */,
|
||||
"strictBindCallApply": true /* Check that the arguments for `bind`, `call`, and `apply` methods match the original function. */,
|
||||
"strictPropertyInitialization": true /* Check for class properties that are declared but not set in the constructor. */,
|
||||
"noImplicitThis": true /* Enable error reporting when `this` is given the type `any`. */,
|
||||
"useUnknownInCatchVariables": true /* Type catch clause variables as 'unknown' instead of 'any'. */,
|
||||
"alwaysStrict": true /* Ensure 'use strict' is always emitted. */,
|
||||
"noUnusedLocals": true /* Enable error reporting when a local variables aren't read. */,
|
||||
"noUnusedParameters": true /* Raise an error when a function parameter isn't read */,
|
||||
"exactOptionalPropertyTypes": true /* Interpret optional property types as written, rather than adding 'undefined'. */,
|
||||
"noImplicitReturns": true /* Enable error reporting for codepaths that do not explicitly return in a function. */,
|
||||
"noFallthroughCasesInSwitch": true /* Enable error reporting for fallthrough cases in switch statements. */,
|
||||
"noUncheckedIndexedAccess": true /* Include 'undefined' in index signature results */,
|
||||
"noImplicitOverride": true /* Ensure overriding members in derived classes are marked with an override modifier. */,
|
||||
"noPropertyAccessFromIndexSignature": true /* Enforces using indexed accessors for keys declared using an indexed type */,
|
||||
// "allowUnusedLabels": true, /* Disable error reporting for unused labels. */
|
||||
// "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */
|
||||
|
||||
/* Completeness */
|
||||
// "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */
|
||||
"skipLibCheck": true, /* Skip type checking all .d.ts files. */
|
||||
"resolveJsonModule": true
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user