github/oneuptime
1
1
Fork 0
mirror of https://github.com/OneUptime/oneuptime.git synced 2026-04-06 00:32:12 +02:00
Code Issues 193 Packages Projects Releases Wiki Activity

Merge pull request #1074 from OneUptime/otel

Browse Source 
Otel
This commit is contained in:
Simon Larsen
2023-12-26 11:32:29 +00:00
committed by GitHub
parent 937679d544 8584726d22
commit 0fd7d5f3d1
20 changed files with 208 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
.vscode/launch.json vendored
View File

@@ -181,48 +181,6 @@
"restart": true,
"autoAttachChildProcesses": true
},
{
"address": "127.0.0.1",
"localRoot": "${workspaceFolder}/Alert",
"name": "Alert: Debug with Docker",
"port": 9133,
"remoteRoot": "/usr/src/app",
"request": "attach",
"skipFiles": [
"<node_internals>/**"
],
"type": "node",
"restart": true,
"autoAttachChildProcesses": true
},
{
"address": "127.0.0.1",
"localRoot": "${workspaceFolder}/Alert",
"name": "Integration: Debug with Docker",
"port": 9134,
"remoteRoot": "/usr/src/app",
"request": "attach",
"skipFiles": [
"<node_internals>/**"
],
"type": "node",
"restart": true,
"autoAttachChildProcesses": true
},
{
"address": "127.0.0.1",
"localRoot": "${workspaceFolder}/licensing",
"name": "Licensing: Debug with Docker",
"port": 9233,
"remoteRoot": "/usr/src/app",
"request": "attach",
"skipFiles": [
"<node_internals>/**"
],
"type": "node",
"restart": true,
"autoAttachChildProcesses": true
},
{
"address": "127.0.0.1",
"localRoot": "${workspaceFolder}/HttpTestServer",

34
Common/Types/Currency.ts Normal file
View File

@@ -0,0 +1,34 @@
import BadDataException from './Exception/BadDataException';
export default class Currency {
public static convertToDecimalPlaces(
value: number,
decimalPlaces: number = 2
): number {
if (decimalPlaces < 0) {
throw new BadDataException(
'decimalPlaces must be greater than or equal to 0.'
);
}
if (typeof value === 'string') {
value = parseFloat(value);
}
if (decimalPlaces === 0) {
return Math.ceil(value);
}
value = value * Math.pow(10, decimalPlaces);
// convert to int.
value = Math.round(value);
// convert back to float.
value = value / Math.pow(10, decimalPlaces);
return value;
}
}

46
Common/Types/DiskSize.ts Normal file
View File

@@ -0,0 +1,46 @@
import BadDataException from './Exception/BadDataException';
export default class DiskSize {
public static convertToDecimalPlaces(
value: number,
decimalPlaces: number = 2
): number {
if (decimalPlaces < 0) {
throw new BadDataException(
'decimalPlaces must be greater than or equal to 0.'
);
}
if (typeof value === 'string') {
value = parseFloat(value);
}
if (decimalPlaces === 0) {
return Math.ceil(value);
}
value = value * Math.pow(10, decimalPlaces);
// convert to int.
value = Math.round(value);
// convert back to float.
value = value / Math.pow(10, decimalPlaces);
return value;
}
public static byteSizeToGB(byteSize: number): number {
return byteSize / 1024 / 1024 / 1024;
}
public static byteSizeToMB(byteSize: number): number {
return byteSize / 1024 / 1024;
}
public static byteSizeToKB(byteSize: number): number {
return byteSize / 1024;
}
}

4
Common/Types/ObjectID.ts
View File

@@ -24,6 +24,10 @@ export default class ObjectID extends DatabaseProperty {
this.id = id;
}
public get value(): string {
return this._id.toString();
}
public equals(other: ObjectID): boolean {
return this.id.toString() === other.id.toString();
}

10
CommonServer/BillingConfig.ts
View File

@@ -1,13 +1,9 @@
const IsBillingEnabled: boolean =
process.env['BILLING_ENABLED'] === 'true';
const IsBillingEnabled: boolean = process.env['BILLING_ENABLED'] === 'true';
const BillingPublicKey: string = process.env['BILLING_PUBLIC_KEY'] || '';
const BillingPrivateKey: string =
process.env['BILLING_PRIVATE_KEY'] || '';
const BillingPrivateKey: string = process.env['BILLING_PRIVATE_KEY'] || '';
export default {
IsBillingEnabled,
BillingPublicKey,
BillingPrivateKey,
}
};

29
CommonServer/EnvironmentConfig.ts
View File

@@ -10,9 +10,8 @@ export const getAllEnvVars: () => JSONObject = (): JSONObject => {
};
export const IsBillingEnabled: boolean = BillingConfig.IsBillingEnabled;
export const BillingPublicKey: string = BillingConfig.BillingPublicKey;;
export const BillingPrivateKey: string =
BillingConfig.BillingPrivateKey;
export const BillingPublicKey: string = BillingConfig.BillingPublicKey;
export const BillingPrivateKey: string = BillingConfig.BillingPrivateKey;
export const DatabaseHost: Hostname = Hostname.fromString(
process.env['DATABASE_HOST'] || 'postgres'
@@ -62,42 +61,50 @@ export const ClusterKey: ObjectID = new ObjectID(
export const HasClusterKey: boolean = Boolean(process.env['ONEUPTIME_SECRET']);
export const RealtimeHostname: Hostname = Hostname.fromString(
`${process.env['SERVER_REALTIME_HOSTNAME'] || 'localhost'}:${process.env['REALTIME_PORT'] || 80
`${process.env['SERVER_REALTIME_HOSTNAME'] || 'localhost'}:${
process.env['REALTIME_PORT'] || 80
}`
);
export const WorkerHostname: Hostname = Hostname.fromString(
`${process.env['SERVER_WORKERS_HOSTNAME'] || 'localhost'}:${process.env['WORKERS_PORT'] || 80
`${process.env['SERVER_WORKERS_HOSTNAME'] || 'localhost'}:${
process.env['WORKERS_PORT'] || 80
}`
);
export const WorkflowHostname: Hostname = Hostname.fromString(
`${process.env['SERVER_WORKFLOW_HOSTNAME'] || 'localhost'}:${process.env['WORKFLOW_PORT'] || 80
`${process.env['SERVER_WORKFLOW_HOSTNAME'] || 'localhost'}:${
process.env['WORKFLOW_PORT'] || 80
}`
);
export const DashboardApiHostname: Hostname = Hostname.fromString(
`${process.env['SERVER_DASHBOARD_API_HOSTNAME'] || 'localhost'}:${process.env['DASHBOARD_API_PORT'] || 80
`${process.env['SERVER_DASHBOARD_API_HOSTNAME'] || 'localhost'}:${
process.env['DASHBOARD_API_PORT'] || 80
}`
);
export const IngestorHostname: Hostname = Hostname.fromString(
`${process.env['SERVER_INGESTOR_HOSTNAME'] || 'localhost'}:${process.env['INGESTOR_PORT'] || 80
`${process.env['SERVER_INGESTOR_HOSTNAME'] || 'localhost'}:${
process.env['INGESTOR_PORT'] || 80
}`
);
export const AccountsHostname: Hostname = Hostname.fromString(
`${process.env['SERVER_ACCOUNTS_HOSTNAME'] || 'localhost'}:${process.env['ACCOUNTS_PORT'] || 80
`${process.env['SERVER_ACCOUNTS_HOSTNAME'] || 'localhost'}:${
process.env['ACCOUNTS_PORT'] || 80
}`
);
export const HomeHostname: Hostname = Hostname.fromString(
`${process.env['SERVER_HOME_HOSTNAME'] || 'localhost'}:${process.env['HOME_PORT'] || 80
`${process.env['SERVER_HOME_HOSTNAME'] || 'localhost'}:${
process.env['HOME_PORT'] || 80
}`
);
export const DashboardHostname: Hostname = Hostname.fromString(
`${process.env['SERVER_DASHBOARD_HOSTNAME'] || 'localhost'}:${process.env['DASHBOARD_PORT'] || 80
`${process.env['SERVER_DASHBOARD_HOSTNAME'] || 'localhost'}:${
process.env['DASHBOARD_PORT'] || 80
}`
);

9
CommonServer/Infrastructure/PostgresConfig.ts
View File

@@ -10,10 +10,12 @@ import {
DatabaseSslCert,
DatabaseRejectUnauthorized,
ShouldDatabaseSslEnable,
Env,
} from '../EnvironmentConfig';
import Entities from 'Model/Models/Index';
import Migrations from 'Model/Migrations/Index';
import DatabaseType from 'Common/Types/DatabaseType';
import AppEnvironment from 'Common/Types/AppEnvironment';
import Faker from 'Common/Utils/Faker';
export const dataSourceOptions: DataSourceOptions = {
@@ -43,11 +45,12 @@ export const datasource: DataSource = new DataSource(dataSourceOptions);
export const testDataSourceOptions: DataSourceOptions = {
type: DatabaseType.Postgres,
host: 'localhost',
port: 5400,
host: DatabaseHost.toString(),
port: DatabasePort.toNumber(),
username: DatabaseUsername,
password: DatabasePassword,
database: DatabaseName + Faker.randomNumbers(16),
entities: Entities,
synchronize: true,
synchronize:
Env === AppEnvironment.Test || Env === AppEnvironment.Development,
};

6
CommonServer/Services/BillingService.ts
View File

@@ -892,7 +892,7 @@ export class BillingService extends BaseService {
priceId: this.getMeteredPlanPriceId(data.productType),
pricePerUnitInUSD:
0.1 * dataRetentionDays * dataRetentionMultiplier,
unitName: `per GB for ${dataRetentionDays} days data retention.`,
unitName: `GB (${dataRetentionDays} days data retention)`,
});
}
@@ -901,7 +901,7 @@ export class BillingService extends BaseService {
priceId: this.getMeteredPlanPriceId(data.productType),
pricePerUnitInUSD:
0.1 * dataRetentionDays * dataRetentionMultiplier,
unitName: `per GB for ${dataRetentionDays} days data retention.`,
unitName: `GB (${dataRetentionDays} days data retention)`,
});
}
@@ -910,7 +910,7 @@ export class BillingService extends BaseService {
priceId: this.getMeteredPlanPriceId(data.productType),
pricePerUnitInUSD:
0.1 * dataRetentionDays * dataRetentionMultiplier,
unitName: `per GB for ${dataRetentionDays} days data retention.`,
unitName: `GB (${dataRetentionDays} days data retention)`,
});
}

1
CommonServer/Services/UsageBillingService.ts
View File

@@ -37,6 +37,7 @@ export class Service extends DatabaseService<Model> {
select: {
_id: true,
usageCount: true,
totalCostInUSD: true,
},
props: {
isRoot: true,

9
CommonServer/Tests/Services/BillingService.test.ts
View File

@@ -34,7 +34,6 @@ import {
Subscription,
} from '../TestingUtils/Services/Types';
import { ActiveMonitoringMeteredPlan } from '../../Types/Billing/MeteredPlan/AllMeteredPlans';
import Database from '../TestingUtils/Database';
describe('BillingService', () => {
let billingService: BillingService;
@@ -43,22 +42,14 @@ describe('BillingService', () => {
customer.id.toString()
);
let database!: Database;
beforeEach(
async () => {
jest.clearAllMocks();
billingService = mockIsBillingEnabled(true);
database = new Database();
await database.createAndConnect();
},
10 * 1000 // 10 second timeout because setting up the DB is slow
);
afterEach(async () => {
await database.disconnectAndDropDatabase();
});
describe('Customer Management', () => {
describe('createCustomer', () => {
it('should create a customer when valid data is provided', async () => {

2
CommonServer/Tests/TestingUtils/Init.ts
View File

@@ -11,10 +11,8 @@ process.env['BILLING_ENABLED'] = 'true';
process.env['DATABASE_HOST'] = 'localhost';
process.env['DATABASE_PORT'] = '5400';
process.env['DATABASE_PASSWORD'] = 'please-change-this-to-random-value';
process.env['REDIS_HOST'] = 'localhost';
process.env['REDIS_PORT'] = '6379';
process.env['REDIS_DB'] = '0';
process.env['REDIS_USERNAME'] = 'default';
process.env['REDIS_PASSWORD'] = 'please-change-this-to-random-value';

7
CommonServer/Types/Billing/MeteredPlan/ServerMeteredPlan.ts
View File

@@ -24,8 +24,11 @@ export default class ServerMeteredPlan {
return meteredPlan.getPricePerUnit() * quantity;
}
public async getMeteredPlan(_projectId: ObjectID): Promise<MeteredPlan> {
throw new NotImplementedException();
public async getMeteredPlan(projectId: ObjectID): Promise<MeteredPlan> {
return await BillingService.getMeteredPlan({
projectId: projectId,
productType: this.getProductType(),
});
}
public async reportQuantityToBillingProvider(

45
CommonServer/Types/Billing/MeteredPlan/TelemetryMeteredPlan.ts
View File

@@ -43,6 +43,31 @@ export default class TelemetryMeteredPlan extends ServerMeteredPlan {
return;
}
// calculate all the total usage count and report it to billing provider.
let totalCostInUSD: number = 0;
for (const usageBilling of usageBillings) {
if (
usageBilling?.totalCostInUSD &&
usageBilling?.totalCostInUSD > 0
) {
totalCostInUSD += usageBilling.totalCostInUSD;
}
}
if (totalCostInUSD < 1) {
return; // too low to report.
}
// convert USD to cents.
let totalCostInCents: number = totalCostInUSD * 100;
// convert this to integer.
totalCostInCents = Math.ceil(totalCostInCents);
// update this count in project as well.
const project: Project | null = await ProjectService.findOneById({
id: projectId,
@@ -61,19 +86,15 @@ export default class TelemetryMeteredPlan extends ServerMeteredPlan {
project.paymentProviderMeteredSubscriptionId) &&
project.paymentProviderPlanId
) {
for (const usageBilling of usageBillings) {
if (
usageBilling?.usageCount &&
usageBilling?.usageCount > 0 &&
usageBilling.id
) {
await BillingService.addOrUpdateMeteredPricingOnSubscription(
(options?.meteredPlanSubscriptionId as string) ||
(project.paymentProviderMeteredSubscriptionId as string),
this,
usageBilling.usageCount
);
await BillingService.addOrUpdateMeteredPricingOnSubscription(
(options?.meteredPlanSubscriptionId as string) ||
(project.paymentProviderMeteredSubscriptionId as string),
this,
totalCostInCents
);
for (const usageBilling of usageBillings) {
if (usageBilling.id) {
// now mark it as reported.
await UsageBillingService.updateOneById({

13
CommonServer/Utils/AnalyticsDatabase/Statement.ts
View File

@@ -2,6 +2,7 @@ import { BaseQueryParams } from '@clickhouse/client';
import { integer } from '@elastic/elasticsearch/lib/api/types';
import { RecordValue } from 'Common/AnalyticsModels/CommonModel';
import TableColumnType from 'Common/Types/AnalyticsDatabase/TableColumnType';
import ObjectID from 'Common/Types/ObjectID';
import { inspect } from 'util';
/**
@@ -48,7 +49,17 @@ export class Statement implements BaseQueryParams {
public get query_params(): Record<string, unknown> {
return Object.fromEntries(
this.values.map((v: StatementParameter | string, i: integer) => {
return [`p${i}`, typeof v === 'string' ? v : v.value];
let finalValue: any = v;
if (typeof v === 'string') {
finalValue = v;
} else if (v.value instanceof ObjectID) {
finalValue = v.value.toString();
} else {
finalValue = v.value;
}
return [`p${i}`, finalValue];
})
);
}

4
Dashboard/src/Components/NavBar/NavBar.tsx
View File

@@ -134,7 +134,7 @@ const DashboardNavbar: FunctionComponent<ComponentProps> = (
),
}}
>
{/* <NavBarMenuItem
<NavBarMenuItem
title="Telemetry"
description="Logs, Traces, Metrics and more."
route={RouteUtil.populateRouteParams(
@@ -144,7 +144,7 @@ const DashboardNavbar: FunctionComponent<ComponentProps> = (
onClick={() => {
forceHideMoreMenu();
}}
/> */}
/>
<NavBarMenuItem
title="On-Call Duty"

17
Dashboard/src/Pages/Settings/UsageHistory.tsx
View File

@@ -10,6 +10,8 @@ import DashboardSideMenu from './SideMenu';
import UsageBilling from 'Model/Models/UsageBilling';
import FieldType from 'CommonUI/src/Components/Types/FieldType';
import DashboardNavigation from '../../Utils/Navigation';
import Currency from 'Common/Types/Currency';
import DiskSize from 'Common/Types/DiskSize';
export interface ComponentProps extends PageComponentProps {}
@@ -51,7 +53,8 @@ const Settings: FunctionComponent<ComponentProps> = (
isViewable={false}
cardProps={{
title: 'Usage History',
description: 'Here is the usage history for this project.',
description:
'Here is the usage history for this project. Please refer to the pricing page for more details.',
}}
noItemsMessage={
'No usage history found. Maybe you have not used Telemetry features yet?'
@@ -89,9 +92,9 @@ const Settings: FunctionComponent<ComponentProps> = (
type: FieldType.Text,
getElement: (item: JSONObject) => {
return (
<div>{`${item['usageCount'] as string} ${
item['usageUnitName'] as string
}`}</div>
<div>{`${DiskSize.convertToDecimalPlaces(
item['usageCount'] as number
)} ${item['usageUnitName'] as string}`}</div>
);
},
},
@@ -103,9 +106,9 @@ const Settings: FunctionComponent<ComponentProps> = (
type: FieldType.Text,
getElement: (item: JSONObject) => {
return (
<div>{`${
item['totalCostInUSD'] as string
} USD`}</div>
<div>{`${Currency.convertToDecimalPlaces(
item['totalCostInUSD'] as number
)} USD`}</div>
);
},
},

1
Examples/otel-dotnet/obj/staticwebassets.pack.sentinel
View File

@@ -33,3 +33,4 @@
2.0
2.0
2.0
2.0

23
Ingestor/API/OTelIngest.ts
View File

@@ -25,6 +25,9 @@ import OTelIngestService from '../Service/OTelIngest';
import GlobalCache from 'CommonServer/Infrastructure/GlobalCache';
import TelemetryServiceService from 'CommonServer/Services/TelemetryServiceService';
import TelemetryService from 'Model/Models/TelemetryService';
import DiskSize from 'Common/Types/DiskSize';
import UsageBillingService from 'CommonServer/Services/UsageBillingService';
import { ProductType } from 'Model/Models/UsageBilling';
// Load proto file for OTel
@@ -60,12 +63,24 @@ router.use(
'/otel/*',
async (req: ExpressRequest, _res: ExpressResponse, next: NextFunction) => {
try {
// size of req.body in bytes.
const sizeInBytes: number = Buffer.byteLength(
JSON.stringify(req.body)
);
let productType: ProductType;
const sizeToGb: number = DiskSize.byteSizeToGB(sizeInBytes);
if (req.baseUrl === '/otel/v1/traces') {
req.body = TracesData.decode(req.body);
productType = ProductType.Traces;
} else if (req.baseUrl === '/otel/v1/logs') {
req.body = LogsData.decode(req.body);
productType = ProductType.Logs;
} else if (req.baseUrl === '/otel/v1/metrics') {
req.body = MetricsData.decode(req.body);
productType = ProductType.Metrics;
} else {
throw new BadRequestException('Invalid URL: ' + req.baseUrl);
}
@@ -136,6 +151,14 @@ router.use(
);
// report to Usage Service.
UsageBillingService.updateUsageBilling({
projectId: (req as OtelRequest).projectId,
productType: productType,
usageCount: sizeToGb,
}).catch((err: Error) => {
logger.error('Failed to update usage billing for OTel');
logger.error(err);
});
next();
} catch (err) {

4
Model/Models/UsageBilling.ts
View File

@@ -169,7 +169,7 @@ export default class UsageBilling extends AccessControlModel {
})
@Column({
nullable: false,
type: ColumnType.Number,
type: ColumnType.Decimal,
})
public usageCount?: number = undefined;
@@ -215,7 +215,7 @@ export default class UsageBilling extends AccessControlModel {
})
@Column({
nullable: false,
type: ColumnType.Number,
type: ColumnType.Decimal,
})
public totalCostInUSD?: number = undefined;

7
Workers/Jobs/MeteredPlan/ReportTelemetryMeteredPlan.ts
View File

@@ -3,7 +3,7 @@ import {
IsDevelopment,
} from 'CommonServer/EnvironmentConfig';
import RunCron from '../../Utils/Cron';
import { EVERY_DAY, EVERY_MINUTE } from 'Common/Utils/CronTime';
import { EVERY_DAY, EVERY_FIVE_MINUTE } from 'Common/Utils/CronTime';
import LIMIT_MAX from 'Common/Types/Database/LimitMax';
import logger from 'CommonServer/Utils/Logger';
import Project from 'Model/Models/Project';
@@ -17,7 +17,10 @@ import Sleep from 'Common/Types/Sleep';
RunCron(
'MeteredPlan:ReportTelemetryMeteredPlan',
{ schedule: IsDevelopment ? EVERY_MINUTE : EVERY_DAY, runOnStartup: false },
{
schedule: IsDevelopment ? EVERY_FIVE_MINUTE : EVERY_DAY,
runOnStartup: true,
},
async () => {
if (!IsBillingEnabled) {
logger.info(