diff --git a/Clickhouse/config.xml b/Clickhouse/config.xml
new file mode 100644
index 0000000000..43d871f40b
--- /dev/null
+++ b/Clickhouse/config.xml
@@ -0,0 +1,1778 @@
+
+
+
+
+ trace
+ /var/log/clickhouse-server/clickhouse-server.log
+ /var/log/clickhouse-server/clickhouse-server.err.log
+
+ 1000M
+ 10
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ https://{bucket}.s3.amazonaws.com
+
+
+ https://storage.googleapis.com/{bucket}
+
+
+ https://{bucket}.oss.aliyuncs.com
+
+
+
+
+
+
+ Access-Control-Allow-Origin
+ *
+
+
+ Access-Control-Allow-Headers
+ origin, x-requested-with, x-clickhouse-format, x-clickhouse-user, x-clickhouse-key, Authorization
+
+
+ Access-Control-Allow-Methods
+ POST, GET, OPTIONS
+
+
+ Access-Control-Max-Age
+ 86400
+
+
+
+
+
+
+
+ 8123
+
+
+ 9000
+
+
+
+
+
+ 9004
+
+
+ 9005
+
+
+
+
+
+
+
+
+
+
+
+ 9009
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10
+
+
+
+
+ false
+
+
+ /path/to/ssl_cert_file
+ /path/to/ssl_key_file
+
+
+ false
+
+
+ /path/to/ssl_ca_cert_file
+
+
+ none
+
+
+ 0
+
+
+ -1
+ -1
+
+
+ false
+
+
+
+
+
+
+
+
+
+ none
+ true
+ true
+ sslv2,sslv3
+ true
+
+
+
+ RejectCertificateHandler
+
+
+
+
+ true
+ true
+ sslv2,sslv3
+ true
+
+
+
+ RejectCertificateHandler
+
+
+
+
+
+
+
+
+ 0
+ 2
+
+
+ 1000
+
+
+ 0
+
+
+
+ 10000
+
+
+
+
+
+ true
+
+
+ 0.9
+
+
+ 4194304
+
+
+ 0
+
+
+
+
+
+ 8589934592
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ /var/lib/clickhouse/caches/
+
+ false
+
+
+ /var/lib/clickhouse/
+
+
+
+
+
+ /var/lib/clickhouse/tmp/
+
+
+ 1
+ 1
+ 1
+
+
+ sha256_password
+
+
+ 12
+
+
+
+
+
+
+
+
+ /var/lib/clickhouse/user_files/
+
+
+
+
+
+
+
+
+
+
+
+
+ users.xml
+
+
+
+ /var/lib/clickhouse/access/
+
+
+
+
+
+
+
+ true
+
+
+ true
+
+
+ true
+
+
+ true
+
+
+ true
+
+
+ false
+
+
+ 600
+
+
+
+ default
+
+
+ SQL_
+
+
+
+
+
+
+
+
+ default
+
+
+
+
+
+
+
+
+ true
+
+
+ false
+
+ ' | sed -e 's|.*>\(.*\)<.*|\1|')
+ wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb
+ apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb
+ clickhouse-jdbc-bridge &
+
+ * [CentOS/RHEL]
+ export MVN_URL=https://repo1.maven.org/maven2/com/clickhouse/clickhouse-jdbc-bridge/
+ export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|')
+ wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm
+ yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm
+ clickhouse-jdbc-bridge &
+
+ Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information.
+ ]]>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ localhost
+ 9000
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 3600
+
+
+
+ 3600
+
+
+ 60
+
+
+
+
+
+
+
+
+
+
+
+
+ system
+
+
+ toYYYYMM(event_date)
+
+
+
+
+
+
+
+ 7500
+
+ 1048576
+
+ 8192
+
+ 524288
+
+ false
+
+
+
+
+
+
+
+ system
+
+
+ toYYYYMM(event_date)
+ 7500
+ 1048576
+ 8192
+ 524288
+
+ false
+ true
+
+
+
+
+ system
+
+ toYYYYMM(event_date)
+ 7500
+ 1048576
+ 8192
+ 524288
+ false
+
+
+
+
+ system
+
+ toYYYYMM(event_date)
+ 7500
+
+
+
+
+ system
+
+ toYYYYMM(event_date)
+ 7500
+ 1048576
+ 8192
+ 524288
+ false
+
+
+
+
+ system
+
+ 7500
+ 1048576
+ 8192
+ 524288
+ false
+ trace
+
+
+
+
+ system
+
+ 7500
+ 1048576
+ 8192
+ 524288
+ 1000
+ false
+
+
+
+
+ system
+
+ 7500
+ 1048576
+ 8192
+ 524288
+ 1000
+ false
+
+
+
+
+ system
+
+ 7500
+ 1048576
+ 8192
+ 524288
+ 1000
+ false
+
+
+
+
+ system
+
+ 7000
+ 1048576
+ 8192
+ 524288
+ false
+
+
+
+
+
+
+ engine MergeTree
+ partition by toYYYYMM(finish_date)
+ order by (finish_date, finish_time_us, trace_id)
+
+ system
+
+ 7500
+ 1048576
+ 8192
+ 524288
+ false
+
+
+
+
+
+ system
+
+
+
+ 1000
+ 1024
+ 1024
+ 512
+ true
+
+
+
+
+
+
+
+ system
+
+
+ toYYYYMM(event_date)
+ 7500
+ 1048576
+ 8192
+ 524288
+ false
+
+
+
+
+ system
+
+
+ 7500
+ 1048576
+ 8192
+ 524288
+ false
+ event_date
+ event_date + INTERVAL 3 DAY
+
+
+
+
+ system
+
+ toYYYYMM(event_date)
+ 7500
+
+
+
+
+ system
+
+ toYYYYMM(event_date)
+ 7500
+
+
+
+
+ system
+
+ toYYYYMM(event_date)
+ 7500
+ event_date + INTERVAL 30 DAY
+
+
+
+
+
+
+
+
+
+
+
+
+
+ *_dictionary.*ml
+
+
+ true
+
+
+ true
+
+
+ *_function.*ml
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ /clickhouse/task_queue/ddl
+
+ /clickhouse/task_queue/replicas
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ click_cost
+ any
+
+ 0
+ 3600
+
+
+ 86400
+ 60
+
+
+
+ max
+
+ 0
+ 60
+
+
+ 3600
+ 300
+
+
+ 86400
+ 3600
+
+
+
+
+
+ /var/lib/clickhouse/format_schemas/
+
+
+ /usr/share/clickhouse/protos/
+
+
+
+
+
+
+
+
+
+ false
+
+ false
+
+
+ https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277
+
+ false
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ backups
+
+
+ true
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/Common/Server/Services/AnalyticsDatabaseService.ts b/Common/Server/Services/AnalyticsDatabaseService.ts
index 9cb0ba63bd..120dec7f61 100644
--- a/Common/Server/Services/AnalyticsDatabaseService.ts
+++ b/Common/Server/Services/AnalyticsDatabaseService.ts
@@ -29,7 +29,7 @@ import logger from "../Utils/Logger";
import Realtime from "../Utils/Realtime";
import StreamUtil from "../Utils/Stream";
import BaseService from "./BaseService";
-import { ExecResult } from "@clickhouse/client";
+import { ExecResult, ResponseJSON, ResultSet } from "@clickhouse/client";
import AnalyticsBaseModel from "Common/Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel";
import { WorkflowRoute } from "Common/ServiceRoute";
import Protocol from "../../Types/API/Protocol";
@@ -148,19 +148,23 @@ export default class AnalyticsDatabaseService<
const countStatement: Statement = this.toCountStatement(countBy);
- const dbResult: ExecResult = await this.execute(countStatement);
- const strResult: string = await StreamUtil.convertStreamToText(
- dbResult.stream,
+ const dbResult: ResultSet<"JSON"> = await this.executeQuery(
+ countStatement,
);
- let countPositive: PositiveNumber = new PositiveNumber(strResult || 0);
+ logger.debug(`${this.model.tableName} Count Statement executed`);
+ logger.debug(countStatement);
- if (countBy.groupBy && Object.keys(countBy.groupBy).length > 0) {
- // this usually happens when group by is used. In this case we count the total number of groups and not rows in those groups.
- countPositive = new PositiveNumber(strResult.split("\n").length - 1); // -1 because the last line is empty.
+ const resultInJSON: ResponseJSON = (await dbResult.json());
+ let countPositive: PositiveNumber = new PositiveNumber(0);
+ if (resultInJSON.data && resultInJSON.data[0] && resultInJSON.data[0]["count()"] && typeof resultInJSON.data[0]["count()"] === "string") {
+ countPositive = new PositiveNumber(resultInJSON.data[0]["count()"] as string);
}
+ logger.debug(`Result: `);
+ logger.debug(countPositive.toNumber());
+
countPositive = await this.onCountSuccess(countPositive);
return countPositive;
} catch (error) {
@@ -314,16 +318,6 @@ export default class AnalyticsDatabaseService<
}
}
- public async executeQuery(query: string): Promise {
- const dbResult: ExecResult = await this.execute(query);
-
- const strResult: string = await StreamUtil.convertStreamToText(
- dbResult.stream,
- );
-
- return strResult;
- }
-
private async _findBy(
findBy: FindBy,
): Promise> {
@@ -379,21 +373,16 @@ export default class AnalyticsDatabaseService<
columns: Array;
} = this.toFindStatement(onBeforeFind);
- const dbResult: ExecResult = await this.execute(
+ const dbResult: ResultSet<"JSON"> = await this.executeQuery(
findStatement.statement,
);
logger.debug(`${this.model.tableName} Find Statement executed`);
logger.debug(findStatement.statement);
- const strResult: string = await StreamUtil.convertStreamToText(
- dbResult.stream,
- );
+ const responseJSON: ResponseJSON = (await dbResult.json())
- const jsonItems: Array = this.convertSelectReturnedDataToJson(
- strResult,
- findStatement.columns,
- );
+ const jsonItems: Array = responseJSON.data;
let items: Array =
AnalyticsBaseModel.fromJSONArray(jsonItems, this.modelType);
@@ -815,6 +804,24 @@ export default class AnalyticsDatabaseService<
) as ExecResult;
}
+
+ public async executeQuery(
+ statement: Statement | string,
+ ): Promise> {
+ if (!this.databaseClient) {
+ this.useDefaultDatabase();
+ }
+
+ const query: string = statement instanceof Statement ? statement.query : statement;
+ const queryParams: Record | undefined = statement instanceof Statement ? statement.query_params : undefined;
+
+ return await this.databaseClient.query({
+ query: query,
+ format: "JSON",
+ query_params: queryParams || undefined as any, // undefined is not specified in the type for query_params, but its ok to pass undefined.
+ })
+ }
+
protected async onUpdateSuccess(
onUpdate: OnUpdate,
_updatedItemIds: Array,
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 4163c857d9..159f995674 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -26,6 +26,8 @@ services:
extends:
file: ./docker-compose.base.yml
service: clickhouse
+ volumes:
+ - ./Clickhouse/config.xml:/etc/clickhouse-server/config.xml
postgres:
ports: