diff --git a/Common/Types/Workflow/Component.ts b/Common/Types/Workflow/Component.ts index 62fbae633e..0c8f11e5ca 100644 --- a/Common/Types/Workflow/Component.ts +++ b/Common/Types/Workflow/Component.ts @@ -1,4 +1,5 @@ import IconProp from '../Icon/IconProp'; +import { JSONObject } from '../JSON'; export enum ComponentInputType { Text = 'Text', @@ -27,6 +28,25 @@ export enum ComponentType { Component = 'Component', } +export enum NodeType { + Node = 'Node', + PlaceholderNode = 'PlaceholderNode', +} + +export interface NodeDataProp { + error: string; + id: string; + nodeType: NodeType; + onClick?: (node: NodeDataProp) => void | undefined; + isPreview?: boolean | undefined; // is this used to show in the components modal? + metadata: ComponentMetadata; + metadataId: string; + internalId: string; + arguments: JSONObject; + returnValues: JSONObject; + componentType: ComponentType; +} + export interface Port { title: string; description: string; diff --git a/CommonServer/Infrastructure/Queue.ts b/CommonServer/Infrastructure/Queue.ts new file mode 100644 index 0000000000..d1c985c852 --- /dev/null +++ b/CommonServer/Infrastructure/Queue.ts @@ -0,0 +1,21 @@ +import { Queue as BullQueue } from 'bullmq'; +import { JSONObject } from 'Common/Types/JSON'; +import ObjectID from 'Common/Types/ObjectID'; +import { RedisHostname, RedisPort } from '../Config'; + +export enum QueueName { + Workflow ="Workflow" +} + +export default class Queue { + public static getQueue(queueName: QueueName): BullQueue { + return new BullQueue(queueName, { connection: { + host: RedisHostname.toString(), + port: RedisPort.toNumber() + }}); + } + + public static async addJob(queueName: QueueName, jobId: ObjectID, jobName: string, data: JSONObject){ + await this.getQueue(queueName).add(jobName, data, {jobId: jobId.toString()}); + } +} \ No newline at end of file diff --git a/CommonServer/Infrastructure/QueueWorker.ts b/CommonServer/Infrastructure/QueueWorker.ts new file mode 100644 index 0000000000..87176b3fc0 --- /dev/null +++ b/CommonServer/Infrastructure/QueueWorker.ts @@ -0,0 +1,11 @@ +import { Worker } from 'bullmq'; +import { RedisHostname, RedisPort } from '../Config'; + +export default class QueueWorker { + public static getWorker(queueName: string, onJobInQueue: any){ + return new Worker(queueName, onJobInQueue, { connection: { + host: RedisHostname.toString(), + port: RedisPort.toNumber() + }}); + } +} \ No newline at end of file diff --git a/CommonServer/Services/WorkflowService.ts b/CommonServer/Services/WorkflowService.ts index 1398864f11..998786c5ed 100644 --- a/CommonServer/Services/WorkflowService.ts +++ b/CommonServer/Services/WorkflowService.ts @@ -1,10 +1,48 @@ import PostgresDatabase from '../Infrastructure/PostgresDatabase'; import Model from 'Model/Models/Workflow'; -import DatabaseService from './DatabaseService'; +import DatabaseService, { OnUpdate } from './DatabaseService'; +import ObjectID from 'Common/Types/ObjectID'; +import { JSONObject } from 'Common/Types/JSON'; +import { ComponentType, NodeDataProp, NodeType } from "Common/Types/Workflow/Component"; + export class Service extends DatabaseService { public constructor(postgresDatabase?: PostgresDatabase) { super(Model, postgresDatabase); } + + + protected override async onUpdateSuccess(onUpdate: OnUpdate, _updatedItemIds: ObjectID[]): Promise> { + + + /// save trigger and trigger args. + + let trigger: NodeDataProp | null = null; + + if (onUpdate.updateBy.data && (onUpdate.updateBy.data as any).graph && ((onUpdate.updateBy.data as any).graph as any)['nodes'] as Array) { + // check if it has a trigger node. + for (const node of ((onUpdate.updateBy.data as any).graph as any)['nodes'] as Array) { + const nodeData = node["data"] as NodeDataProp; + if(nodeData.componentType === ComponentType.Trigger && nodeData.nodeType === NodeType.Node){ + // found the trigger; + trigger = nodeData; + } + } + } + + await this.updateOneById({ + id: new ObjectID(onUpdate.updateBy.query._id! as any), + data: { + triggerId: trigger?.metadataId!, + triggerArguments: trigger?.arguments || {} + } as any, + props: { + isRoot: true, + ignoreHooks: true, + } + }); + + return onUpdate; + } } export default new Service(); diff --git a/CommonServer/package-lock.json b/CommonServer/package-lock.json index 68e8568303..ac2052e36e 100644 --- a/CommonServer/package-lock.json +++ b/CommonServer/package-lock.json @@ -17,6 +17,7 @@ "@types/gridfs-stream": "^0.5.35", "@types/json2csv": "^5.0.3", "airtable": "^0.11.3", + "bullmq": "^3.6.6", "Common": "file:../Common", "cors": "^2.8.5", "dotenv": "^16.0.0", @@ -1891,6 +1892,11 @@ "@hapi/topo": "^5.0.0" } }, + "node_modules/@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" + }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -2203,6 +2209,78 @@ "@jridgewell/sourcemap-codec": "1.4.14" } }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.0.tgz", + "integrity": "sha512-5qpnNHUyyEj9H3sm/4Um/bnx1lrQGhe8iqry/1d+cQYCRd/gzYA0YLeq0ezlk4hKx4vO+dsEsNyeowqRqslwQA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.0.tgz", + "integrity": "sha512-ZphTFFd6SFweNAMKD+QJCrWpgkjf4qBuHltiMkKkD6FFrB3NOTRVmetAGTkJ57pa+s6J0yCH06LujWB9rZe94g==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.0.tgz", + "integrity": "sha512-ztKVV1dO/sSZyGse0PBCq3Pk1PkYjsA/dsEWE7lfrGoAK3i9HpS2o7XjGQ7V4va6nX+xPPOiuYpQwa4Bi6vlww==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.0.tgz", + "integrity": "sha512-NEX6hdSvP4BmVyegaIbrGxvHzHvTzzsPaxXCsUt0mbLbPpEftsvNwaEVKOowXnLoeuGeD4MaqSwL3BUK2elsUA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.0.tgz", + "integrity": "sha512-9uvdAkZMOPCY7SPRxZLW8XGqBOVNVEhqlgffenN8shA1XR9FWVsSM13nr/oHtNgXg6iVyML7RwWPyqUeThlwxg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.0.tgz", + "integrity": "sha512-Wg0+9615kHKlr9iLVcG5I+/CHnf6w3x5UADRv8Ad16yA0Bu5l9eVOROjV7aHPG6uC8ZPFIVVaoSjDChD+Y0pzg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@opentelemetry/api": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.1.0.tgz", @@ -4359,6 +4437,66 @@ "node": ">=4" } }, + "node_modules/bullmq": { + "version": "3.6.6", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.6.6.tgz", + "integrity": "sha512-W71jXrcTdcT3Y5tzMyTx22Cd8O3dTML7vl6KG3YdGVGrO3+UmKRLYfGLn1QwIhIoTQJVvIrSB4qfGs1hgqYRVw==", + "dependencies": { + "cron-parser": "^4.6.0", + "glob": "^8.0.3", + "ioredis": "^5.3.0", + "lodash": "^4.17.21", + "msgpackr": "^1.6.2", + "semver": "^7.3.7", + "tslib": "^2.0.0", + "uuid": "^9.0.0" + } + }, + "node_modules/bullmq/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/bullmq/node_modules/glob": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^5.0.1", + "once": "^1.3.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/bullmq/node_modules/minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/bullmq/node_modules/uuid": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/busboy": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", @@ -4664,6 +4802,17 @@ "node": ">= 0.10" } }, + "node_modules/cron-parser": { + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.7.1.tgz", + "integrity": "sha512-WguFaoQ0hQ61SgsCZLHUcNbAvlK0lypKXu62ARguefYmjzaOXIVRNrAmyXzabTwUn4sQvQLkk6bjH+ipGfw8bA==", + "dependencies": { + "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -5878,6 +6027,37 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "node_modules/ioredis": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.1.tgz", + "integrity": "sha512-C+IBcMysM6v52pTLItYMeV4Hz7uriGtoJdz7SSBDX6u+zwSYGirLdQh3L7t/OItWITcw3gTFMjJReYUwS4zihg==", + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/ioredis/node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/ip": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz", @@ -6982,11 +7162,21 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" + }, "node_modules/lodash.get": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==" }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" + }, "node_modules/lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", @@ -7021,6 +7211,14 @@ "node": ">=10" } }, + "node_modules/luxon": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.2.1.tgz", + "integrity": "sha512-QrwPArQCNLAKGO/C+ZIilgIuDnEnKx5QYODdDtbFaxzsbZcc/a7WFq7MhsVYgRlwawLtvOUESTlfJ+hc/USqPg==", + "engines": { + "node": ">=12" + } + }, "node_modules/make-dir": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", @@ -7279,6 +7477,35 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, + "node_modules/msgpackr": { + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.8.3.tgz", + "integrity": "sha512-m2JefwcKNzoHYXkH/5jzHRxAw7XLWsAdvu0FOJ+OLwwozwOV/J6UA62iLkfIMbg7G8+dIuRwgg6oz+QoQ4YkoA==", + "optionalDependencies": { + "msgpackr-extract": "^3.0.0" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.0.tgz", + "integrity": "sha512-oy6KCk1+X4Bn5m6Ycq5N1EWl9npqG/cLrE8ga8NX7ZqfqYUUBS08beCQaGq80fjbKBySur0E6x//yZjzNJDt3A==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.0.7" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.0", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.0", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.0", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.0", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.0", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.0" + } + }, "node_modules/mz": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", @@ -7374,6 +7601,17 @@ "webidl-conversions": "^3.0.0" } }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.7.tgz", + "integrity": "sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==", + "optional": true, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -8041,6 +8279,25 @@ "@redis/time-series": "1.0.4" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/reflect-metadata": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz", @@ -8541,6 +8798,11 @@ "node": ">=10" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -11090,6 +11352,11 @@ "@hapi/topo": "^5.0.0" } }, + "@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" + }, "@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -11338,6 +11605,42 @@ "@jridgewell/sourcemap-codec": "1.4.14" } }, + "@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.0.tgz", + "integrity": "sha512-5qpnNHUyyEj9H3sm/4Um/bnx1lrQGhe8iqry/1d+cQYCRd/gzYA0YLeq0ezlk4hKx4vO+dsEsNyeowqRqslwQA==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.0.tgz", + "integrity": "sha512-ZphTFFd6SFweNAMKD+QJCrWpgkjf4qBuHltiMkKkD6FFrB3NOTRVmetAGTkJ57pa+s6J0yCH06LujWB9rZe94g==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.0.tgz", + "integrity": "sha512-ztKVV1dO/sSZyGse0PBCq3Pk1PkYjsA/dsEWE7lfrGoAK3i9HpS2o7XjGQ7V4va6nX+xPPOiuYpQwa4Bi6vlww==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.0.tgz", + "integrity": "sha512-NEX6hdSvP4BmVyegaIbrGxvHzHvTzzsPaxXCsUt0mbLbPpEftsvNwaEVKOowXnLoeuGeD4MaqSwL3BUK2elsUA==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.0.tgz", + "integrity": "sha512-9uvdAkZMOPCY7SPRxZLW8XGqBOVNVEhqlgffenN8shA1XR9FWVsSM13nr/oHtNgXg6iVyML7RwWPyqUeThlwxg==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.0.tgz", + "integrity": "sha512-Wg0+9615kHKlr9iLVcG5I+/CHnf6w3x5UADRv8Ad16yA0Bu5l9eVOROjV7aHPG6uC8ZPFIVVaoSjDChD+Y0pzg==", + "optional": true + }, "@opentelemetry/api": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.1.0.tgz", @@ -12999,6 +13302,56 @@ "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==" }, + "bullmq": { + "version": "3.6.6", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.6.6.tgz", + "integrity": "sha512-W71jXrcTdcT3Y5tzMyTx22Cd8O3dTML7vl6KG3YdGVGrO3+UmKRLYfGLn1QwIhIoTQJVvIrSB4qfGs1hgqYRVw==", + "requires": { + "cron-parser": "^4.6.0", + "glob": "^8.0.3", + "ioredis": "^5.3.0", + "lodash": "^4.17.21", + "msgpackr": "^1.6.2", + "semver": "^7.3.7", + "tslib": "^2.0.0", + "uuid": "^9.0.0" + }, + "dependencies": { + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "requires": { + "balanced-match": "^1.0.0" + } + }, + "glob": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^5.0.1", + "once": "^1.3.0" + } + }, + "minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "requires": { + "brace-expansion": "^2.0.1" + } + }, + "uuid": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==" + } + } + }, "busboy": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", @@ -13257,6 +13610,14 @@ "vary": "^1" } }, + "cron-parser": { + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.7.1.tgz", + "integrity": "sha512-WguFaoQ0hQ61SgsCZLHUcNbAvlK0lypKXu62ARguefYmjzaOXIVRNrAmyXzabTwUn4sQvQLkk6bjH+ipGfw8bA==", + "requires": { + "luxon": "^3.2.1" + } + }, "cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -14182,6 +14543,29 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "ioredis": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.1.tgz", + "integrity": "sha512-C+IBcMysM6v52pTLItYMeV4Hz7uriGtoJdz7SSBDX6u+zwSYGirLdQh3L7t/OItWITcw3gTFMjJReYUwS4zihg==", + "requires": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "dependencies": { + "denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==" + } + } + }, "ip": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz", @@ -15048,11 +15432,21 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" + }, "lodash.get": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==" }, + "lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" + }, "lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", @@ -15084,6 +15478,11 @@ "yallist": "^4.0.0" } }, + "luxon": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.2.1.tgz", + "integrity": "sha512-QrwPArQCNLAKGO/C+ZIilgIuDnEnKx5QYODdDtbFaxzsbZcc/a7WFq7MhsVYgRlwawLtvOUESTlfJ+hc/USqPg==" + }, "make-dir": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", @@ -15273,6 +15672,29 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, + "msgpackr": { + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.8.3.tgz", + "integrity": "sha512-m2JefwcKNzoHYXkH/5jzHRxAw7XLWsAdvu0FOJ+OLwwozwOV/J6UA62iLkfIMbg7G8+dIuRwgg6oz+QoQ4YkoA==", + "requires": { + "msgpackr-extract": "^3.0.0" + } + }, + "msgpackr-extract": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.0.tgz", + "integrity": "sha512-oy6KCk1+X4Bn5m6Ycq5N1EWl9npqG/cLrE8ga8NX7ZqfqYUUBS08beCQaGq80fjbKBySur0E6x//yZjzNJDt3A==", + "optional": true, + "requires": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.0", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.0", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.0", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.0", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.0", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.0", + "node-gyp-build-optional-packages": "5.0.7" + } + }, "mz": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", @@ -15351,6 +15773,12 @@ } } }, + "node-gyp-build-optional-packages": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.7.tgz", + "integrity": "sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==", + "optional": true + }, "node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -15860,6 +16288,19 @@ "@redis/time-series": "1.0.4" } }, + "redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==" + }, + "redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "requires": { + "redis-errors": "^1.0.0" + } + }, "reflect-metadata": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz", @@ -16250,6 +16691,11 @@ "escape-string-regexp": "^2.0.0" } }, + "standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", diff --git a/CommonServer/package.json b/CommonServer/package.json index 3ac26e982c..a0f7a57358 100644 --- a/CommonServer/package.json +++ b/CommonServer/package.json @@ -19,6 +19,7 @@ "@types/gridfs-stream": "^0.5.35", "@types/json2csv": "^5.0.3", "airtable": "^0.11.3", + "bullmq": "^3.6.6", "Common": "file:../Common", "cors": "^2.8.5", "dotenv": "^16.0.0", diff --git a/CommonUI/src/Components/Workflow/ArgumentsForm.tsx b/CommonUI/src/Components/Workflow/ArgumentsForm.tsx index 10cb6ffd29..53f62b9c8d 100644 --- a/CommonUI/src/Components/Workflow/ArgumentsForm.tsx +++ b/CommonUI/src/Components/Workflow/ArgumentsForm.tsx @@ -5,11 +5,10 @@ import React, { useRef, useState, } from 'react'; -import { Argument } from 'Common/Types/Workflow/Component'; +import { Argument, NodeDataProp } from 'Common/Types/Workflow/Component'; import ErrorMessage from '../ErrorMessage/ErrorMessage'; import BasicForm from '../Forms/BasicForm'; import FormValues from '../Forms/Types/FormValues'; -import { NodeDataProp } from './Component'; import { JSONObject } from 'Common/Types/JSON'; import Dictionary from 'Common/Types/Dictionary'; import { componentInputTypeToFormFieldType } from './Utils'; diff --git a/CommonUI/src/Components/Workflow/Component.tsx b/CommonUI/src/Components/Workflow/Component.tsx index ab9931be69..c5897b0935 100644 --- a/CommonUI/src/Components/Workflow/Component.tsx +++ b/CommonUI/src/Components/Workflow/Component.tsx @@ -1,33 +1,19 @@ -import { JSONObject } from 'Common/Types/JSON'; + import React, { FunctionComponent, useState } from 'react'; import { Handle, Position, Connection } from 'reactflow'; import Icon, { ThickProp } from '../Icon/Icon'; import IconProp from 'Common/Types/Icon/IconProp'; -import ComponentMetadata, { +import { ComponentType, + NodeDataProp, + NodeType, Port, } from 'Common/Types/Workflow/Component'; import Tooltip from '../Tooltip/Toolip'; import Pill from '../Pill/Pill'; import { Green } from 'Common/Types/BrandColors'; -export enum NodeType { - Node = 'Node', - PlaceholderNode = 'PlaceholderNode', -} -export interface NodeDataProp { - error: string; - id: string; - nodeType: NodeType; - onClick?: (node: NodeDataProp) => void | undefined; - isPreview?: boolean | undefined; // is this used to show in the components modal? - metadata: ComponentMetadata; - metadataId: string; - internalId: string; - arguments: JSONObject; - returnValues: JSONObject; -} export interface ComponentProps { data: NodeDataProp; diff --git a/CommonUI/src/Components/Workflow/ComponentModal.tsx b/CommonUI/src/Components/Workflow/ComponentModal.tsx index 5c70cbd826..feb022bd56 100644 --- a/CommonUI/src/Components/Workflow/ComponentModal.tsx +++ b/CommonUI/src/Components/Workflow/ComponentModal.tsx @@ -9,9 +9,9 @@ import React, { import ComponentMetadata, { ComponentType, ComponentCategory, + NodeType } from 'Common/Types/Workflow/Component'; - -import ComponentElement, { NodeType } from './Component'; +import ComponentElement from './Component'; import Input from '../Input/Input'; import ErrorMessage from '../ErrorMessage/ErrorMessage'; import Icon from '../Icon/Icon'; @@ -201,6 +201,7 @@ const ComponentsModal: FunctionComponent = ( '', nodeType: NodeType.Node, + componentType:componentMetadata.componentType, returnValues: {}, isPreview: diff --git a/CommonUI/src/Components/Workflow/ComponentSettingsModal.tsx b/CommonUI/src/Components/Workflow/ComponentSettingsModal.tsx index 2cd004ba2c..896e260596 100644 --- a/CommonUI/src/Components/Workflow/ComponentSettingsModal.tsx +++ b/CommonUI/src/Components/Workflow/ComponentSettingsModal.tsx @@ -9,11 +9,11 @@ import FormFieldSchemaType from '../Forms/Types/FormFieldSchemaType'; import FormValues from '../Forms/Types/FormValues'; import ConfirmModal from '../Modal/ConfirmModal'; import SideOver from '../SideOver/SideOver'; -import { NodeDataProp } from './Component'; import ComponentPortViewer from './ComponentPortViewer'; import ComponentReturnValueViewer from './ComponentReturnValueViewer'; import ArgumentsForm from './ArgumentsForm'; import ObjectID from 'Common/Types/ObjectID'; +import { NodeDataProp } from 'Common/Types/Workflow/Component'; export interface ComponentProps { title: string; diff --git a/CommonUI/src/Components/Workflow/ComponentValuePickerModal.tsx b/CommonUI/src/Components/Workflow/ComponentValuePickerModal.tsx index 0646f20c6a..77b06106a7 100644 --- a/CommonUI/src/Components/Workflow/ComponentValuePickerModal.tsx +++ b/CommonUI/src/Components/Workflow/ComponentValuePickerModal.tsx @@ -1,10 +1,9 @@ import { Black } from 'Common/Types/BrandColors'; -import { ReturnValue } from 'Common/Types/Workflow/Component'; +import { NodeDataProp, ReturnValue } from 'Common/Types/Workflow/Component'; import React, { useState, FunctionComponent, ReactElement } from 'react'; import ErrorMessage from '../ErrorMessage/ErrorMessage'; import Modal, { ModalWidth } from '../Modal/Modal'; import Pill from '../Pill/Pill'; -import { NodeDataProp } from './Component'; export interface ComponentProps { onClose: () => void; diff --git a/CommonUI/src/Components/Workflow/Edge.tsx b/CommonUI/src/Components/Workflow/Edge.tsx deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/CommonUI/src/Components/Workflow/RunForm.tsx b/CommonUI/src/Components/Workflow/RunForm.tsx index 42af3174ff..e9bf2fc47f 100644 --- a/CommonUI/src/Components/Workflow/RunForm.tsx +++ b/CommonUI/src/Components/Workflow/RunForm.tsx @@ -5,11 +5,10 @@ import React, { useRef, useState, } from 'react'; -import { ReturnValue } from 'Common/Types/Workflow/Component'; +import { NodeDataProp, ReturnValue } from 'Common/Types/Workflow/Component'; import ErrorMessage from '../ErrorMessage/ErrorMessage'; import BasicForm from '../Forms/BasicForm'; import FormValues from '../Forms/Types/FormValues'; -import { NodeDataProp } from './Component'; import { JSONObject } from 'Common/Types/JSON'; import Dictionary from 'Common/Types/Dictionary'; import { componentInputTypeToFormFieldType } from './Utils'; diff --git a/CommonUI/src/Components/Workflow/RunModal.tsx b/CommonUI/src/Components/Workflow/RunModal.tsx index 0f9a217ac3..331ba68568 100644 --- a/CommonUI/src/Components/Workflow/RunModal.tsx +++ b/CommonUI/src/Components/Workflow/RunModal.tsx @@ -1,10 +1,10 @@ import Dictionary from 'Common/Types/Dictionary'; +import { NodeDataProp, NodeType } from 'Common/Types/Workflow/Component'; import React, { FunctionComponent, ReactElement, useState } from 'react'; import { ButtonStyleType } from '../Button/Button'; import ErrorMessage from '../ErrorMessage/ErrorMessage'; import ConfirmModal from '../Modal/ConfirmModal'; import SideOver from '../SideOver/SideOver'; -import { NodeDataProp, NodeType } from './Component'; import RunForm from './RunForm'; export interface ComponentProps { diff --git a/CommonUI/src/Components/Workflow/Workflow.tsx b/CommonUI/src/Components/Workflow/Workflow.tsx index 320e747057..d3fefe3534 100644 --- a/CommonUI/src/Components/Workflow/Workflow.tsx +++ b/CommonUI/src/Components/Workflow/Workflow.tsx @@ -24,12 +24,13 @@ import ReactFlow, { } from 'reactflow'; // 👇 you need to import the reactflow styles import 'reactflow/dist/style.css'; -import WorkflowComponent, { NodeDataProp, NodeType } from './Component'; +import WorkflowComponent from './Component'; import ObjectID from 'Common/Types/ObjectID'; import IconProp from 'Common/Types/Icon/IconProp'; import ComponentMetadata, { ComponentCategory, ComponentType, + NodeDataProp, NodeType } from 'Common/Types/Workflow/Component'; import ComponentsModal from './ComponentModal'; import { JSONObject } from 'Common/Types/JSON'; @@ -165,7 +166,7 @@ const Workflow: FunctionComponent = (props: ComponentProps) => { return node.data.id !== id; }); - if (nodeToUpdate.length === 0) { + if (nodeToUpdate.filter((n: Node)=> (n.data as NodeDataProp).componentType === ComponentType.Trigger && (n.data as NodeDataProp).nodeType === NodeType.Node).length === 0) { nodeToUpdate = nodeToUpdate.concat(getPlaceholderTriggerNode()); } @@ -311,7 +312,8 @@ const Workflow: FunctionComponent = (props: ComponentProps) => { metadata: { ...componentMetadata }, metadataId: componentMetadata.id, internalId: ObjectID.generate().toString(), // runner id - }, + componentType: componentMetadata.componentType + } as NodeDataProp, }; if (componentMetadata.componentType === ComponentType.Trigger) { diff --git a/Dashboard/src/Pages/Workflow/View/Builder.tsx b/Dashboard/src/Pages/Workflow/View/Builder.tsx index cc505c97c0..cdc1771e3c 100644 --- a/Dashboard/src/Pages/Workflow/View/Builder.tsx +++ b/Dashboard/src/Pages/Workflow/View/Builder.tsx @@ -31,10 +31,7 @@ import ComponentMetadata, { ComponentCategory, } from 'Common/Types/Workflow/Component'; import BadDataException from 'Common/Types/Exception/BadDataException'; -import { - NodeDataProp, - NodeType, -} from 'CommonUI/src/Components/Workflow/Component'; +import { NodeDataProp, NodeType } from 'Common/Types/Workflow/Component'; const Delete: FunctionComponent = ( _props: PageComponentProps diff --git a/Model/Models/Workflow.ts b/Model/Models/Workflow.ts index 6b0820daa1..11d7a038fb 100644 --- a/Model/Models/Workflow.ts +++ b/Model/Models/Workflow.ts @@ -398,4 +398,45 @@ export default class Workflow extends BaseModel { }, }) public labels?: Array