From 4dc77412893f603adda492897e1d3e808a59d3dc Mon Sep 17 00:00:00 2001 From: Simon Larsen Date: Tue, 22 Aug 2023 09:37:39 +0100 Subject: [PATCH] fix phone number --- Common/Models/AnalyticsBaseModel.ts | 37 +++-- .../AnalyticsDatabase/AnalyticsTableEngine.ts | 4 +- Common/Utils/Faker.ts | 2 +- CommonServer/Config.ts | 2 +- .../Infrastructure/ClickhouseDatabase.ts | 11 +- .../Services/AnalyticsDatabaseService.ts | 155 +++++++++++++++--- CommonServer/Services/Index.ts | 6 +- CommonServer/Services/LogService.ts | 2 - .../Types/AnalyticsDatabase/CreateBy.ts | 7 + DashboardAPI/Index.ts | 1 - Model/Models/Index.ts | 5 +- Model/Models/Log.ts | 16 +- Workers/Index.ts | 2 +- .../AnalyticsDatabase/TableManegement.ts | 9 +- docker-compose.base.yml | 6 + docker-compose.dev.yml | 3 + 16 files changed, 198 insertions(+), 70 deletions(-) create mode 100644 CommonServer/Types/AnalyticsDatabase/CreateBy.ts diff --git a/Common/Models/AnalyticsBaseModel.ts b/Common/Models/AnalyticsBaseModel.ts index e27b5f34ab..bcce0fc522 100644 --- a/Common/Models/AnalyticsBaseModel.ts +++ b/Common/Models/AnalyticsBaseModel.ts @@ -20,24 +20,21 @@ export default class AnalyticsDataModel { this._tableName = v; } - - private _tableEngine : AnalyticsTableEngine = AnalyticsTableEngine.MergeTree; - public get tableEngine() : AnalyticsTableEngine { + private _tableEngine: AnalyticsTableEngine = AnalyticsTableEngine.MergeTree; + public get tableEngine(): AnalyticsTableEngine { return this._tableEngine; } - public set tableEngine(v : AnalyticsTableEngine) { + public set tableEngine(v: AnalyticsTableEngine) { this._tableEngine = v; } - - private _primaryKeys : Array = []; - public get primaryKeys() : Array { + private _primaryKeys: Array = []; + public get primaryKeys(): Array { return this._primaryKeys; } - public set primaryKeys(v : Array) { + public set primaryKeys(v: Array) { this._primaryKeys = v; } - public constructor(data: { tableName: string; @@ -45,13 +42,12 @@ export default class AnalyticsDataModel { tableColumns: Array; primaryKeys: Array; // this should be the subset of tableColumns }) { - - let columns: Array = [...data.tableColumns]; + const columns: Array = [...data.tableColumns]; this.tableName = data.tableName; - if(data.tableEngine){ - this.tableEngine = data.tableEngine; + if (data.tableEngine) { + this.tableEngine = data.tableEngine; } columns.push( @@ -84,8 +80,8 @@ export default class AnalyticsDataModel { }) ); - if(!data.primaryKeys || data.primaryKeys.length === 0){ - throw new BadDataException("Primary keys are required"); + if (!data.primaryKeys || data.primaryKeys.length === 0) { + throw new BadDataException('Primary keys are required'); } // check if primary keys are subset of tableColumns @@ -93,13 +89,18 @@ export default class AnalyticsDataModel { console.log(columns); data.primaryKeys.forEach((primaryKey) => { - if(!columns.find((column) => column.key === primaryKey)){ - throw new BadDataException("Primary key "+primaryKey+" is not part of tableColumns"); + if ( + !columns.find((column) => { + return column.key === primaryKey; + }) + ) { + throw new BadDataException( + 'Primary key ' + primaryKey + ' is not part of tableColumns' + ); } }); this.primaryKeys = data.primaryKeys; this.tableColumns = columns; - } } diff --git a/Common/Types/AnalyticsDatabase/AnalyticsTableEngine.ts b/Common/Types/AnalyticsDatabase/AnalyticsTableEngine.ts index b1539f8e07..55c909e0c8 100644 --- a/Common/Types/AnalyticsDatabase/AnalyticsTableEngine.ts +++ b/Common/Types/AnalyticsDatabase/AnalyticsTableEngine.ts @@ -1,5 +1,5 @@ enum AnalyticsTableEngine { - MergeTree = 'MergeTree' + MergeTree = 'MergeTree', } -export default AnalyticsTableEngine; \ No newline at end of file +export default AnalyticsTableEngine; diff --git a/Common/Utils/Faker.ts b/Common/Utils/Faker.ts index 56059c8edd..1cfb07c445 100644 --- a/Common/Utils/Faker.ts +++ b/Common/Utils/Faker.ts @@ -29,6 +29,6 @@ export default class Faker { } public static generatePhone(): Phone { - return new Phone(faker.phone.number()); + return new Phone(faker.phone.number("+1##########")); } } diff --git a/CommonServer/Config.ts b/CommonServer/Config.ts index 318f42fe3e..3a17e61a13 100644 --- a/CommonServer/Config.ts +++ b/CommonServer/Config.ts @@ -217,7 +217,7 @@ export const ClickhousePort: Port = new Port( ); export const ClickhouseUsername: string = - process.env['CLICKHOUSE_USERNAME'] || 'default'; + process.env['CLICKHOUSE_USER'] || 'default'; export const ClickhousePassword: string = process.env['CLICKHOUSE_PASSWORD'] || 'password'; diff --git a/CommonServer/Infrastructure/ClickhouseDatabase.ts b/CommonServer/Infrastructure/ClickhouseDatabase.ts index 418f20ce5f..7b83a6613b 100644 --- a/CommonServer/Infrastructure/ClickhouseDatabase.ts +++ b/CommonServer/Infrastructure/ClickhouseDatabase.ts @@ -36,7 +36,6 @@ export default class ClickhouseDatabase { let retry: number = 0; try { - const connectToDatabase: Function = async (): Promise => { try { @@ -46,11 +45,15 @@ export default class ClickhouseDatabase { const result = await clickhouseClient.ping(); - if(result.success === false){ - throw new DatabaseNotConnectedException("Clickhouse Database is not connected") + if (result.success === false) { + throw new DatabaseNotConnectedException( + 'Clickhouse Database is not connected' + ); } - logger.info(`Clickhouse Database Connected: ${dataSourceOptions.host?.toString()}`); + logger.info( + `Clickhouse Database Connected: ${dataSourceOptions.host?.toString()}` + ); return clickhouseClient; } catch (err) { diff --git a/CommonServer/Services/AnalyticsDatabaseService.ts b/CommonServer/Services/AnalyticsDatabaseService.ts index 7c705e001c..e071fa3fde 100644 --- a/CommonServer/Services/AnalyticsDatabaseService.ts +++ b/CommonServer/Services/AnalyticsDatabaseService.ts @@ -1,18 +1,24 @@ -import TableColumnType from "Common/Types/BaseDatabase/TableColumnType"; -import ClickhouseDatabase, { ClickhouseAppInstance, ClickhouseClient } from "../Infrastructure/ClickhouseDatabase"; -import BaseService from "./BaseService"; -import AnalyticsBaseModel from "Common/Models/AnalyticsBaseModel"; -import BadDataException from "Common/Types/Exception/BadDataException"; +import TableColumnType from 'Common/Types/BaseDatabase/TableColumnType'; +import ClickhouseDatabase, { + ClickhouseAppInstance, + ClickhouseClient, +} from '../Infrastructure/ClickhouseDatabase'; +import BaseService from './BaseService'; +import AnalyticsBaseModel from 'Common/Models/AnalyticsBaseModel'; +import BadDataException from 'Common/Types/Exception/BadDataException'; +import logger from '../Utils/Logger'; +// import CreateBy from "../Types/AnalyticsDatabase/CreateBy"; -export default class AnalyticsDatabaseService extends BaseService { - - public modelType!: { new(): TBaseModel }; +export default class AnalyticsDatabaseService< + TBaseModel extends AnalyticsBaseModel +> extends BaseService { + public modelType!: { new (): TBaseModel }; public database!: ClickhouseDatabase; public model!: TBaseModel; public databaseClient!: ClickhouseClient; public constructor(data: { - modelType: { new(): TBaseModel }, + modelType: { new (): TBaseModel }; database?: ClickhouseDatabase | undefined; }) { super(); @@ -20,18 +26,36 @@ export default class AnalyticsDatabaseService key).join(', ')} - ` + PRIMARY KEY ( + ${this.model.primaryKeys + .map(key => { + return key; + }) + .join(', ')} + ) + `; + + logger.info(`${this.model.tableName} Table Create Statement`); + logger.info(statement); + + return statement; } public useDefaultDatabase(): void { @@ -40,22 +64,116 @@ export default class AnalyticsDatabaseService { - - if(!this.databaseClient){ + if (!this.databaseClient) { this.useDefaultDatabase(); } await this.databaseClient.exec({ query: query, - }) + }); } + // public async create(createBy: CreateBy): Promise { + + // const onCreate: OnCreate = createBy.props.ignoreHooks + // ? { createBy, carryForward: [] } + // : await this._onBeforeCreate(createBy); + + // let _createdBy: CreateBy = onCreate.createBy; + + // const carryForward: any = onCreate.carryForward; + + // _createdBy = this.generateSlug(_createdBy); + + // let data: TBaseModel = _createdBy.data; + + // // add tenantId if present. + // const tenantColumnName: string | null = data.getTenantColumn(); + + // if (tenantColumnName && _createdBy.props.tenantId) { + // data.setColumnValue(tenantColumnName, _createdBy.props.tenantId); + // } + + // data = this.generateDefaultValues(data); + // data = this.checkRequiredFields(data); + + // if (!this.isValid(data)) { + // throw new BadDataException('Data is not valid'); + // } + + // // check total items by. + + // await this.checkTotalItemsBy(_createdBy); + + // // Encrypt data + // data = this.encrypt(data); + + // // hash data + // data = await this.hash(data); + + // ModelPermission.checkCreatePermissions( + // this.entityType, + // data, + // _createdBy.props + // ); + + // createBy.data = data; + + // // check uniqueColumns by: + // createBy = await this.checkUniqueColumnBy(createBy); + + // // serialize. + // createBy.data = (await this.sanitizeCreateOrUpdate( + // createBy.data, + // createBy.props + // )) as TBaseModel; + + // try { + // createBy.data = await this.getRepository().save(createBy.data); + + // if (!createBy.props.ignoreHooks) { + // createBy.data = await this.onCreateSuccess( + // { + // createBy, + // carryForward, + // }, + // createBy.data + // ); + // } + + // // hit workflow.; + // if (this.getModel().enableWorkflowOn?.create) { + // let tenantId: ObjectID | undefined = createBy.props.tenantId; + + // if (!tenantId && this.getModel().getTenantColumn()) { + // tenantId = createBy.data.getValue( + // this.getModel().getTenantColumn()! + // ); + // } + + // if (tenantId) { + // await this.onTrigger( + // createBy.data.id!, + // tenantId, + // 'on-create' + // ); + // } + // } + + // return createBy.data; + // } catch (error) { + // await this.onCreateError(error as Exception); + // throw this.getException(error as Exception); + // } + // } public toColumnsCreateStatement(): string { let columns: string = ''; this.model.tableColumns.forEach((column) => { - columns += `${column.key} ${this.toColumnType(column.type)} ${column.required ? 'NOT NULL' : ' NULL'},\n`; + columns += `${column.key} ${this.toColumnType(column.type)} ${ + column.required ? 'NOT NULL' : ' NULL' + },\n`; }); return columns; @@ -86,7 +204,6 @@ export default class AnalyticsDatabaseService = [ WorkflowVariablesService, ]; -export const AnalyticsServices: Array> = [ - LogService -] +export const AnalyticsServices: Array< + AnalyticsDatabaseService +> = [LogService]; export default services; diff --git a/CommonServer/Services/LogService.ts b/CommonServer/Services/LogService.ts index 5a0f764b1e..33985b1a06 100644 --- a/CommonServer/Services/LogService.ts +++ b/CommonServer/Services/LogService.ts @@ -1,4 +1,3 @@ - import Log from 'Model/Models/Log'; import AnalyticsDatabaseService from './AnalyticsDatabaseService'; import ClickhouseDatabase from '../Infrastructure/ClickhouseDatabase'; @@ -7,6 +6,5 @@ export class Service extends AnalyticsDatabaseService { public constructor(clickhouseDatabase?: ClickhouseDatabase | undefined) { super({ modelType: Log, database: clickhouseDatabase }); } - } export default new Service(); diff --git a/CommonServer/Types/AnalyticsDatabase/CreateBy.ts b/CommonServer/Types/AnalyticsDatabase/CreateBy.ts new file mode 100644 index 0000000000..9b0800f484 --- /dev/null +++ b/CommonServer/Types/AnalyticsDatabase/CreateBy.ts @@ -0,0 +1,7 @@ +import AnalyticsBaseModel from 'Common/Models/AnalyticsBaseModel'; +import DatabaseCommonInteractionProps from 'Common/Types/Database/DatabaseCommonInteractionProps'; + +export default interface CreateBy { + data: TBaseModel; + props: DatabaseCommonInteractionProps; +} diff --git a/DashboardAPI/Index.ts b/DashboardAPI/Index.ts index be53c1dd2d..6aa4c92da3 100755 --- a/DashboardAPI/Index.ts +++ b/DashboardAPI/Index.ts @@ -963,7 +963,6 @@ const init: () => Promise = async (): Promise => { await ClickhouseAppInstance.connect( ClickhouseAppInstance.getDatasourceOptions() ); - } catch (err) { logger.error('App Init Failed:'); logger.error(err); diff --git a/Model/Models/Index.ts b/Model/Models/Index.ts index 74df0a9e41..cfa140bc92 100644 --- a/Model/Models/Index.ts +++ b/Model/Models/Index.ts @@ -223,7 +223,4 @@ export default [ IncidentTemplateOwnerUser, ]; - -export const AnalyticsModels: Array = [ - Log -]; \ No newline at end of file +export const AnalyticsModels: Array = [Log]; diff --git a/Model/Models/Log.ts b/Model/Models/Log.ts index de4ad1e144..188885948b 100644 --- a/Model/Models/Log.ts +++ b/Model/Models/Log.ts @@ -1,7 +1,7 @@ -import AnalyticsBaseModel from "Common/Models/AnalyticsBaseModel"; -import AnalyticsTableColumn from "Common/Types/AnalyticsDatabase/TableColumn"; -import TableColumnType from "Common/Types/BaseDatabase/TableColumnType"; -import AnalyticsTableEngine from "Common/Types/AnalyticsDatabase/AnalyticsTableEngine"; +import AnalyticsBaseModel from 'Common/Models/AnalyticsBaseModel'; +import AnalyticsTableColumn from 'Common/Types/AnalyticsDatabase/TableColumn'; +import TableColumnType from 'Common/Types/BaseDatabase/TableColumnType'; +import AnalyticsTableEngine from 'Common/Types/AnalyticsDatabase/AnalyticsTableEngine'; export default class Log extends AnalyticsBaseModel { public constructor() { @@ -31,9 +31,7 @@ export default class Log extends AnalyticsBaseModel { type: TableColumnType.VeryLongText, }), ], - primaryKeys: [ - 'projectId', 'logContainerId', 'createdAt' - ] - }) + primaryKeys: ['projectId', 'logContainerId', 'createdAt'], + }); } -} \ No newline at end of file +} diff --git a/Workers/Index.ts b/Workers/Index.ts index 2e3750cb74..e8426eb7b1 100644 --- a/Workers/Index.ts +++ b/Workers/Index.ts @@ -97,7 +97,7 @@ const init: () => Promise = async (): Promise => { ); await RunDatabaseMigrations(); - + // create tables in analytics database await AnalyticsTableManagement.createTables(); diff --git a/Workers/Utils/AnalyticsDatabase/TableManegement.ts b/Workers/Utils/AnalyticsDatabase/TableManegement.ts index e929b913e2..446cb34fe2 100644 --- a/Workers/Utils/AnalyticsDatabase/TableManegement.ts +++ b/Workers/Utils/AnalyticsDatabase/TableManegement.ts @@ -1,11 +1,10 @@ -import { AnalyticsServices } from "CommonServer/Services/Index"; +import { AnalyticsServices } from 'CommonServer/Services/Index'; export default class AnalyticsTableManagement { - public static async createTables(): Promise { - for(const service of AnalyticsServices){ - + public static async createTables(): Promise { + for (const service of AnalyticsServices) { // create a table if it does not exist await service.execute(service.toTableCreateStatement()); } } -} \ No newline at end of file +} diff --git a/docker-compose.base.yml b/docker-compose.base.yml index 70b17b9962..c5cbc48fa8 100644 --- a/docker-compose.base.yml +++ b/docker-compose.base.yml @@ -88,6 +88,12 @@ x-common-server-variables: &common-server-variables WORKFLOW_ROUTE: ${WORKFLOW_ROUTE} STATUS_PAGE_ROUTE: ${STATUS_PAGE_ROUTE} + CLICKHOUSE_USER: ${CLICKHOUSE_USER} + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD} + CLICKHOUSE_DATABASE: ${CLICKHOUSE_DATABASE} + CLICKHOUSE_HOST: ${CLICKHOUSE_HOST} + CLICKHOUSE_PORT: ${CLICKHOUSE_PORT} + IS_SERVER: true DISABLE_AUTOMATIC_INCIDENT_CREATION: ${DISABLE_AUTOMATIC_INCIDENT_CREATION} diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 53686f7220..8c9c28dd78 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -27,6 +27,9 @@ services: service: redis clickhouse: + ports: + - '9034:9000' + - '8189:8123' extends: file: ./docker-compose.base.yml service: clickhouse