feat: Implement fluent ingest worker for processing queue jobs

- Introduced KEDA autoscaling configuration in values.yaml for probeIngest, fluentIngest, incomingRequestIngest, and serverMonitorIngest.
- Added endpoints for queue statistics, size, and failed jobs in IncomingRequestIngest and ProbeIngest APIs.
- Implemented asynchronous processing of incoming requests and probes using job queues.
- Created Metrics API for KEDA metrics integration in IncomingRequestIngest, ProbeIngest, and ServerMonitorIngest.
- Refactored IncomingRequest and Probe APIs to utilize queue services for processing.
- Added job processing logic for incoming requests and probes in respective job files.
- Implemented queue service classes for managing job addition and retrieval of queue statistics.

.github/workflows/build.yml

@@ -209,22 +209,6 @@ jobs:
       - name: build docker image
         run: sudo docker build -f ./Dashboard/Dockerfile .
 
-  docker-build-haraka:
-    runs-on: ubuntu-latest
-    env:
-      CI_PIPELINE_ID: ${{github.run_number}}
-    steps:
-      - name: Checkout 
-        uses: actions/checkout@v4
-
-      - name: Preinstall 
-        run: npm run prerun
-        
-      # build images
-      - name: build docker image
-        run: sudo docker build -f ./Haraka/Dockerfile .
-
-
   docker-build-probe:
     runs-on: ubuntu-latest
     env:

.github/workflows/release.yml

@@ -70,7 +70,7 @@ jobs:
 
   publish-mcp-server:
     runs-on: ubuntu-latest
-    needs: [generate-build-number]
+    needs: [generate-build-number, publish-npm-packages]
     env:
       CI_PIPELINE_ID: ${{ github.run_number }}
       NPM_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }}
@@ -138,6 +138,7 @@ jobs:
       - name: Install dependencies
         run: |
           cd MCP
+          npm update @oneuptime/common
           npm install
 
       - name: Build MCP server
@@ -1052,67 +1053,6 @@ jobs:
             GIT_SHA=${{ github.sha }}
             APP_VERSION=7.0.${{needs.generate-build-number.outputs.build_number}}
 
-
-  haraka-docker-image-deploy:
-    needs: [generate-build-number]
-    runs-on: ubuntu-latest
-    steps:
-      - name: Docker Meta
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: |
-            oneuptime/haraka
-            ghcr.io/oneuptime/haraka
-          tags: |
-            type=raw,value=release,enable=true
-            type=semver,value=7.0.${{needs.generate-build-number.outputs.build_number}},pattern={{version}},enable=true
-
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.ref }}
-
-      - uses: actions/setup-node@v4
-        with:
-          node-version: latest
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Generate Dockerfile from Dockerfile.tpl
-        run: npm run prerun
-
-      # Build and deploy haraka.
-
-      - name: Login to Docker Hub 
-        uses: docker/login-action@v2.2.0
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PASSWORD }}
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2.2.0
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build and push
-        uses: docker/build-push-action@v4
-        with:
-          file: ./Haraka/Dockerfile
-          context: .
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          build-args: |
-            GIT_SHA=${{ github.sha }}
-            APP_VERSION=7.0.${{needs.generate-build-number.outputs.build_number}}
-
   admin-dashboard-docker-image-deploy:
     needs: [generate-build-number]
     runs-on: ubuntu-latest
@@ -1838,7 +1778,7 @@ jobs:
 
   test-e2e-release-saas: 
     runs-on: ubuntu-latest
-    needs: [open-telemetry-ingest-docker-image-deploy, copilot-docker-image-deploy, fluent-ingest-docker-image-deploy, docs-docker-image-deploy, api-reference-docker-image-deploy, workflow-docker-image-deploy, llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, haraka-docker-image-deploy, probe-ingest-docker-image-deploy, server-monitor-ingest-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, nginx-docker-image-deploy, incoming-request-ingest-docker-image-deploy]
+    needs: [open-telemetry-ingest-docker-image-deploy, copilot-docker-image-deploy, fluent-ingest-docker-image-deploy, docs-docker-image-deploy, api-reference-docker-image-deploy, workflow-docker-image-deploy, llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, probe-ingest-docker-image-deploy, server-monitor-ingest-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, nginx-docker-image-deploy, incoming-request-ingest-docker-image-deploy]
     env:
       CI_PIPELINE_ID: ${{github.run_number}}
     steps:
@@ -1891,7 +1831,7 @@ jobs:
   test-e2e-release-self-hosted: 
     runs-on: ubuntu-latest
     # After all the jobs runs
-    needs: [open-telemetry-ingest-docker-image-deploy, publish-mcp-server, copilot-docker-image-deploy, incoming-request-ingest-docker-image-deploy, fluent-ingest-docker-image-deploy, docs-docker-image-deploy, api-reference-docker-image-deploy, workflow-docker-image-deploy, llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, haraka-docker-image-deploy, probe-ingest-docker-image-deploy, server-monitor-ingest-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, nginx-docker-image-deploy]
+    needs: [open-telemetry-ingest-docker-image-deploy, publish-mcp-server, copilot-docker-image-deploy, incoming-request-ingest-docker-image-deploy, fluent-ingest-docker-image-deploy, docs-docker-image-deploy, api-reference-docker-image-deploy, workflow-docker-image-deploy, llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, probe-ingest-docker-image-deploy, server-monitor-ingest-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, nginx-docker-image-deploy]
     env:
       CI_PIPELINE_ID: ${{github.run_number}}
     steps:

.github/workflows/test-release.yaml

@@ -144,6 +144,7 @@ jobs:
       - name: Install dependencies and build
         run: |
           cd MCP
+          npm update @oneuptime/common
           npm install
           npm run build
 
@@ -1146,67 +1147,6 @@ jobs:
             GIT_SHA=${{ github.sha }}
             APP_VERSION=7.0.${{needs.generate-build-number.outputs.build_number}}
 
-  haraka-docker-image-deploy:
-    needs: generate-build-number
-    runs-on: ubuntu-latest
-    steps:
-      - name: Docker Meta
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: |
-            oneuptime/haraka
-            ghcr.io/oneuptime/haraka
-          tags: |
-            type=raw,value=test,enable=true
-            type=semver,value=7.0.${{needs.generate-build-number.outputs.build_number}}-test,pattern={{version}},enable=true
-            
-
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.ref }}
-
-      - uses: actions/setup-node@v4
-        with:
-          node-version: latest
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Generate Dockerfile from Dockerfile.tpl
-        run: npm run prerun
-
-      # Build and deploy haraka.
-
-      - name: Login to Docker Hub 
-        uses: docker/login-action@v2.2.0
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PASSWORD }}
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2.2.0
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build and push
-        uses: docker/build-push-action@v4
-        with:
-          file: ./Haraka/Dockerfile
-          context: .
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          build-args: |
-            GIT_SHA=${{ github.sha }}
-            APP_VERSION=7.0.${{needs.generate-build-number.outputs.build_number}}
-
   dashboard-docker-image-deploy:
     needs: generate-build-number
     runs-on: ubuntu-latest
@@ -1768,7 +1708,7 @@ jobs:
 
   test-helm-chart:
     runs-on: ubuntu-latest
-    needs: [infrastructure-agent-deploy, publish-mcp-server, llm-docker-image-deploy, publish-terraform-provider, open-telemetry-ingest-docker-image-deploy, copilot-docker-image-deploy, docs-docker-image-deploy, worker-docker-image-deploy,  workflow-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, api-reference-docker-image-deploy,  test-server-docker-image-deploy, test-docker-image-deploy, probe-ingest-docker-image-deploy, server-monitor-ingest-docker-image-deploy, probe-docker-image-deploy, haraka-docker-image-deploy, dashboard-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, accounts-docker-image-deploy, otel-collector-docker-image-deploy, status-page-docker-image-deploy, nginx-docker-image-deploy, e2e-docker-image-deploy, fluent-ingest-docker-image-deploy, incoming-request-ingest-docker-image-deploy]
+    needs: [infrastructure-agent-deploy, publish-mcp-server, llm-docker-image-deploy, publish-terraform-provider, open-telemetry-ingest-docker-image-deploy, copilot-docker-image-deploy, docs-docker-image-deploy, worker-docker-image-deploy,  workflow-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, api-reference-docker-image-deploy,  test-server-docker-image-deploy, test-docker-image-deploy, probe-ingest-docker-image-deploy, server-monitor-ingest-docker-image-deploy, probe-docker-image-deploy, dashboard-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, accounts-docker-image-deploy, otel-collector-docker-image-deploy, status-page-docker-image-deploy, nginx-docker-image-deploy, e2e-docker-image-deploy, fluent-ingest-docker-image-deploy, incoming-request-ingest-docker-image-deploy]
     env:
       CI_PIPELINE_ID: ${{github.run_number}}
     steps:

.gitignore

@@ -86,9 +86,6 @@ Backups/*.tar
 
 .env
 
-Haraka/dkim/keys/private_base64.txt
-Haraka/dkim/keys/public_base64.txt
-
 .eslintcache
 
 HelmChart/Values/*.values.yaml
@@ -129,3 +126,4 @@ terraform-provider-example/**
 MCP/build/
 MCP/.env
 MCP/node_modules 
+Dashboard/public/sw.js

APIReference/CodeExamples/DataTypes/Includes.md

@@ -0,0 +1,11 @@
+{
+    "query": {
+        "labels": {
+            "_type": "Includes",
+            "value": [
+                "aaa00000-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+                "bbb00000-bbbb-bbbb-bbbb-bbbbbbbbbbbb"
+            ]
+        }
+    }
+}

APIReference/Service/DataType.ts

@@ -88,6 +88,16 @@ export default class ServiceHandler {
       },
     );
 
+    pageData.includesCode = await LocalCache.getOrSetString(
+      "data-type",
+      "includes",
+      async () => {
+        return await LocalFile.read(
+          `${CodeExamplesPath}/DataTypes/Includes.md`,
+        );
+      },
+    );
+
     pageData.lessThanOrNullCode = await LocalCache.getOrSetString(
       "data-type",
       "less-than-or-equal",

APIReference/package-lock.json

@@ -55,6 +55,7 @@
         "@types/react-highlight": "^0.12.8",
         "@types/react-syntax-highlighter": "^15.5.13",
         "@types/uuid": "^8.3.4",
+        "@types/web-push": "^3.6.4",
         "acme-client": "^5.3.0",
         "airtable": "^0.12.2",
         "axios": "^1.7.2",
@@ -74,7 +75,6 @@
         "json5": "^2.2.3",
         "jsonwebtoken": "^9.0.0",
         "jwt-decode": "^4.0.0",
-        "lodash": "^4.17.21",
         "marked": "^12.0.2",
         "moment": "^2.30.1",
         "moment-timezone": "^0.5.45",
@@ -118,6 +118,7 @@
         "universal-cookie": "^7.2.1",
         "use-async-effect": "^2.2.6",
         "uuid": "^8.3.2",
+        "web-push": "^3.6.7",
         "zod": "^3.25.30"
       },
       "devDependencies": {
@@ -132,7 +133,6 @@
         "@types/jest": "^28.1.4",
         "@types/json2csv": "^5.0.3",
         "@types/jsonwebtoken": "^8.5.9",
-        "@types/lodash": "^4.14.202",
         "@types/node": "^17.0.45",
         "@types/node-cron": "^3.0.7",
         "@types/nodemailer": "^6.4.7",

APIReference/views/main/data-types.ejs

@@ -395,5 +395,44 @@
             </div>
         </div>
 
+        <h3 id="example-using-cursors" class="scroll-mt-24">
+            Inlcudes
+        </h3>
+
+        <div class="grid grid-cols-1 items-start gap-x-16 gap-y-10 xl:max-w-none xl:grid-cols-2">
+            <div class="[&amp;>:first-child]:mt-0 [&amp;>:last-child]:mb-0">
+                <p> 
+                    Includes will get objects that match any of the values in the array. It is used to
+                    filter objects that have a field that matches any of the values in the array. For example, if you
+                    want to get all objects that have a label with ID `aaa00000-aaaa-aaaa-aaaa-aaaaaaaaaaaa` or
+                    `bbb00000-bbbb-bbbb-bbbb-bbbbbbbbbbbb`, you can use the `includes` query type.
+                </p>
+
+                <div class="my-6">
+                    <ul role="list"
+                        class="m-0 max-w-[calc(theme(maxWidth.lg)-theme(spacing.8))] list-none divide-y divide-zinc-900/5 p-0 ">
+                        <li class="m-0 px-0 py-4 first:pt-0 last:pb-0">
+                            <dl class="m-0 flex flex-wrap items-center gap-x-3 gap-y-2">
+                                <dt class="sr-only">Query</dt>
+                                <dd><code class="inline-code">query</code></dd>
+                                <dt class="sr-only">Type</dt>
+                                <dd class="font-mono text-xs text-zinc-400 ">Query</dd>
+                                <dt class="sr-only">Description</dt>
+                                <dd class="w-full flex-none [&amp;>:first-child]:mt-0 [&amp;>:last-child]:mb-0">
+                                    <p>Here is an example of a less than or equal query</p>
+                                </dd>
+                            </dl>
+                        </li>
+
+                    </ul>
+                </div>
+            </div>
+            <div class="[&amp;>:first-child]:mt-0 [&amp;>:last-child]:mb-0">
+                <%- include('../partials/code', {title: "Example Not EqualTo Request Body" , requestUrl: "" , requestType: "" ,
+                    code: pageData.includesCode }) -%>
+
+            </div>
+        </div>
+
     </article>
 </main>

Accounts/package-lock.json

@@ -59,6 +59,7 @@
                 "@types/react-highlight": "^0.12.8",
                 "@types/react-syntax-highlighter": "^15.5.13",
                 "@types/uuid": "^8.3.4",
+                "@types/web-push": "^3.6.4",
                 "acme-client": "^5.3.0",
                 "airtable": "^0.12.2",
                 "axios": "^1.7.2",
@@ -78,7 +79,6 @@
                 "json5": "^2.2.3",
                 "jsonwebtoken": "^9.0.0",
                 "jwt-decode": "^4.0.0",
-                "lodash": "^4.17.21",
                 "marked": "^12.0.2",
                 "moment": "^2.30.1",
                 "moment-timezone": "^0.5.45",
@@ -122,6 +122,7 @@
                 "universal-cookie": "^7.2.1",
                 "use-async-effect": "^2.2.6",
                 "uuid": "^8.3.2",
+                "web-push": "^3.6.7",
                 "zod": "^3.25.30"
             },
             "devDependencies": {
@@ -136,7 +137,6 @@
                 "@types/jest": "^28.1.4",
                 "@types/json2csv": "^5.0.3",
                 "@types/jsonwebtoken": "^8.5.9",
-                "@types/lodash": "^4.14.202",
                 "@types/node": "^17.0.45",
                 "@types/node-cron": "^3.0.7",
                 "@types/nodemailer": "^6.4.7",

AdminDashboard/package-lock.json

@@ -58,6 +58,7 @@
                 "@types/react-highlight": "^0.12.8",
                 "@types/react-syntax-highlighter": "^15.5.13",
                 "@types/uuid": "^8.3.4",
+                "@types/web-push": "^3.6.4",
                 "acme-client": "^5.3.0",
                 "airtable": "^0.12.2",
                 "axios": "^1.7.2",
@@ -77,7 +78,6 @@
                 "json5": "^2.2.3",
                 "jsonwebtoken": "^9.0.0",
                 "jwt-decode": "^4.0.0",
-                "lodash": "^4.17.21",
                 "marked": "^12.0.2",
                 "moment": "^2.30.1",
                 "moment-timezone": "^0.5.45",
@@ -121,6 +121,7 @@
                 "universal-cookie": "^7.2.1",
                 "use-async-effect": "^2.2.6",
                 "uuid": "^8.3.2",
+                "web-push": "^3.6.7",
                 "zod": "^3.25.30"
             },
             "devDependencies": {
@@ -135,7 +136,6 @@
                 "@types/jest": "^28.1.4",
                 "@types/json2csv": "^5.0.3",
                 "@types/jsonwebtoken": "^8.5.9",
-                "@types/lodash": "^4.14.202",
                 "@types/node": "^17.0.45",
                 "@types/node-cron": "^3.0.7",
                 "@types/nodemailer": "^6.4.7",

AdminDashboard/src/Components/Footer/Footer.tsx

@@ -67,7 +67,7 @@ const DashboardFooter: () => JSX.Element = () => {
   return (
     <>
       <Footer
-        className="bg-white h-16 inset-x-0 bottom-0 px-8"
+        className="bg-white px-8"
         copyright="HackerBay, Inc."
         links={[
           {

AdminDashboard/src/Components/NavBar/NavBar.tsx

@@ -2,36 +2,33 @@ import PageMap from "../../Utils/PageMap";
 import RouteMap, { RouteUtil } from "../../Utils/RouteMap";
 import Route from "Common/Types/API/Route";
 import IconProp from "Common/Types/Icon/IconProp";
-import NavBar from "Common/UI/Components/Navbar/NavBar";
-import NavBarItem from "Common/UI/Components/Navbar/NavBarItem";
+import NavBar, { NavItem } from "Common/UI/Components/Navbar/NavBar";
 import React, { FunctionComponent, ReactElement } from "react";
 
 const DashboardNavbar: FunctionComponent = (): ReactElement => {
-  return (
-    <NavBar>
-      <NavBarItem
-        title="Users"
-        icon={IconProp.User}
-        route={RouteUtil.populateRouteParams(RouteMap[PageMap.USERS] as Route)}
-      ></NavBarItem>
+  // Build the navigation items
+  const navItems: NavItem[] = [
+    {
+      id: "users-nav-bar-item",
+      title: "Users",
+      icon: IconProp.User,
+      route: RouteUtil.populateRouteParams(RouteMap[PageMap.USERS] as Route),
+    },
+    {
+      id: "projects-nav-bar-item",
+      title: "Projects",
+      icon: IconProp.Folder,
+      route: RouteUtil.populateRouteParams(RouteMap[PageMap.PROJECTS] as Route),
+    },
+    {
+      id: "settings-nav-bar-item",
+      title: "Settings",
+      icon: IconProp.Settings,
+      route: RouteUtil.populateRouteParams(RouteMap[PageMap.SETTINGS] as Route),
+    },
+  ];
 
-      <NavBarItem
-        title="Projects"
-        icon={IconProp.Folder}
-        route={RouteUtil.populateRouteParams(
-          RouteMap[PageMap.PROJECTS] as Route,
-        )}
-      ></NavBarItem>
-
-      <NavBarItem
-        title="Settings"
-        icon={IconProp.Settings}
-        route={RouteUtil.populateRouteParams(
-          RouteMap[PageMap.SETTINGS] as Route,
-        )}
-      ></NavBarItem>
-    </NavBar>
-  );
+  return <NavBar items={navItems} />;
 };
 
 export default DashboardNavbar;

AdminDashboard/src/Pages/Projects/Index.tsx

@@ -241,6 +241,7 @@ const Projects: FunctionComponent = (): ReactElement => {
             },
             title: "Created At",
             type: FieldType.DateTime,
+            hideOnMobile: true,
           },
         ]}
         userPreferencesKey="admin-projects-table"

AdminDashboard/src/Pages/Settings/Email/Index.tsx

@@ -21,7 +21,7 @@ import React, { FunctionComponent, ReactElement, useEffect } from "react";
 
 const Settings: FunctionComponent = (): ReactElement => {
   const [emailServerType, setemailServerType] = React.useState<EmailServerType>(
-    EmailServerType.Internal,
+    EmailServerType.CustomSMTP,
   );
 
   const [isLoading, setIsLoading] = React.useState<boolean>(true);
@@ -43,7 +43,7 @@ const Settings: FunctionComponent = (): ReactElement => {
 
     if (globalConfig) {
       setemailServerType(
-        globalConfig.emailServerType || EmailServerType.Internal,
+        globalConfig.emailServerType || EmailServerType.CustomSMTP,
       );
     }
 
@@ -127,7 +127,7 @@ const Settings: FunctionComponent = (): ReactElement => {
       />
 
       <CardModelDetail
-        name="Internal SMTP Settings"
+        name="Email Server Settings"
         cardProps={{
           title: "Email Server Settings",
           description:
@@ -172,7 +172,7 @@ const Settings: FunctionComponent = (): ReactElement => {
           cardProps={{
             title: "Custom Email and SMTP Settings",
             description:
-              "If you have not enabled Internal SMTP server to send emails. Please configure your SMTP server here.",
+              "Please configure your SMTP server here to send emails.",
           }}
           isEditable={true}
           editButtonText="Edit SMTP Config"

AdminDashboard/src/Pages/Settings/Probes/Index.tsx

@@ -54,6 +54,7 @@ const Settings: FunctionComponent = (): ReactElement => {
         title="Need help with setting up Global Probes?"
         description="Here is a guide which will help you get set up"
         link={Route.fromString("/docs/probe/custom-probe")}
+        hideOnMobile={true}
       />
 
       <ModelTable<Probe>
@@ -174,6 +175,7 @@ const Settings: FunctionComponent = (): ReactElement => {
             noValueMessage: "-",
             title: "Description",
             type: FieldType.LongText,
+            hideOnMobile: true,
           },
           {
             field: {
@@ -181,6 +183,7 @@ const Settings: FunctionComponent = (): ReactElement => {
             },
             title: "Status",
             type: FieldType.Text,
+
             getElement: (item: Probe): ReactElement => {
               if (
                 item &&

AdminDashboard/src/Pages/Users/Index.tsx

@@ -116,6 +116,7 @@ const Users: FunctionComponent = (): ReactElement => {
             },
             title: "Email Verified",
             type: FieldType.Boolean,
+            hideOnMobile: true,
           },
           {
             field: {
@@ -123,6 +124,7 @@ const Users: FunctionComponent = (): ReactElement => {
             },
             title: "Created At",
             type: FieldType.DateTime,
+            hideOnMobile: true,
           },
         ]}
       />

App/FeatureSet/BaseAPI/Index.ts

@@ -29,6 +29,7 @@ import MonitorTest from "Common/Models/DatabaseModels/MonitorTest";
 import UserEmailAPI from "Common/Server/API/UserEmailAPI";
 import UserNotificationLogTimelineAPI from "Common/Server/API/UserOnCallLogTimelineAPI";
 import UserSMSAPI from "Common/Server/API/UserSmsAPI";
+import UserPushAPI from "Common/Server/API/UserPushAPI";
 import ApiKeyPermissionService, {
   Service as ApiKeyPermissionServiceType,
 } from "Common/Server/Services/ApiKeyPermissionService";
@@ -1608,6 +1609,7 @@ const BaseAPIFeatureSet: FeatureSet = {
     );
     app.use(`/${APP_NAME.toLocaleLowerCase()}`, new UserEmailAPI().getRouter());
     app.use(`/${APP_NAME.toLocaleLowerCase()}`, new UserSMSAPI().getRouter());
+    app.use(`/${APP_NAME.toLocaleLowerCase()}`, new UserPushAPI().getRouter());
     app.use(`/${APP_NAME.toLocaleLowerCase()}`, new ProbeAPI().getRouter());
 
     app.use(

App/FeatureSet/Notification/Config.ts

@@ -1,10 +1,8 @@
-import Hostname from "Common/Types/API/Hostname";
 import TwilioConfig from "Common/Types/CallAndSMS/TwilioConfig";
 import Email from "Common/Types/Email";
 import EmailServer from "Common/Types/Email/EmailServer";
 import BadDataException from "Common/Types/Exception/BadDataException";
 import ObjectID from "Common/Types/ObjectID";
-import Port from "Common/Types/Port";
 import { AdminDashboardClientURL } from "Common/Server/EnvironmentConfig";
 import GlobalConfigService from "Common/Server/Services/GlobalConfigService";
 import GlobalConfig, {
@@ -12,24 +10,6 @@ import GlobalConfig, {
 } from "Common/Models/DatabaseModels/GlobalConfig";
 import Phone from "Common/Types/Phone";
 
-export const InternalSmtpPassword: string =
-  process.env["INTERNAL_SMTP_PASSWORD"] || "";
-
-export const InternalSmtpHost: Hostname = new Hostname(
-  process.env["INTERNAL_SMTP_HOST"] || "haraka",
-);
-
-export const InternalSmtpPort: Port = new Port(2525);
-
-export const InternalSmtpSecure: boolean = false;
-
-export const InternalSmtpEmail: Email = new Email(
-  process.env["INTERNAL_SMTP_EMAIL"] || "noreply@oneuptime.com",
-);
-
-export const InternalSmtpFromName: string =
-  process.env["INTERNAL_SMTP_FROM_NAME"] || "OneUptime";
-
 type GetGlobalSMTPConfig = () => Promise<EmailServer | null>;
 
 export const getGlobalSMTPConfig: GetGlobalSMTPConfig =
@@ -132,10 +112,10 @@ export const getEmailServerType: GetEmailServerTypeFunction =
       });
 
     if (!globalConfig) {
-      return EmailServerType.Internal;
+      return EmailServerType.CustomSMTP;
     }
 
-    return globalConfig.emailServerType || EmailServerType.Internal;
+    return globalConfig.emailServerType || EmailServerType.CustomSMTP;
   };
 
 export interface SendGridConfig {

App/FeatureSet/Notification/Services/MailService.ts

@@ -1,10 +1,4 @@
 import {
-  InternalSmtpEmail,
-  InternalSmtpFromName,
-  InternalSmtpHost,
-  InternalSmtpPassword,
-  InternalSmtpPort,
-  InternalSmtpSecure,
   SendGridConfig,
   getEmailServerType,
   getGlobalSMTPConfig,
@@ -37,6 +31,98 @@ import nodemailer, { Transporter } from "nodemailer";
 import Path from "path";
 import * as tls from "tls";
 
+// Connection pool for email transporters
+class TransporterPool {
+  private static pools: Map<string, Transporter> = new Map();
+  private static semaphore: Map<string, number> = new Map();
+  private static readonly MAX_CONCURRENT_CONNECTIONS = 100;
+
+  public static getTransporter(
+    emailServer: EmailServer,
+    options: { timeout?: number | undefined },
+  ): Transporter {
+    const key: string = `${emailServer.host.toString()}:${emailServer.port.toNumber()}:${emailServer.username || "noauth"}`;
+
+    if (!this.pools.has(key)) {
+      const transporter: Transporter = this.createTransporter(
+        emailServer,
+        options,
+      );
+      this.pools.set(key, transporter);
+      this.semaphore.set(key, 0);
+    }
+
+    return this.pools.get(key)!;
+  }
+
+  private static createTransporter(
+    emailServer: EmailServer,
+    options: { timeout?: number | undefined },
+  ): Transporter {
+    let tlsOptions: tls.ConnectionOptions | undefined = undefined;
+
+    if (!emailServer.secure) {
+      tlsOptions = {
+        rejectUnauthorized: false,
+      };
+    }
+
+    return nodemailer.createTransport({
+      host: emailServer.host.toString(),
+      port: emailServer.port.toNumber(),
+      secure: emailServer.secure,
+      tls: tlsOptions,
+      auth:
+        emailServer.username && emailServer.password
+          ? {
+              user: emailServer.username,
+              pass: emailServer.password,
+            }
+          : undefined,
+      connectionTimeout: options.timeout || 60000,
+      pool: true, // Enable connection pooling
+      maxConnections: this.MAX_CONCURRENT_CONNECTIONS,
+    });
+  }
+
+  public static async acquireConnection(
+    emailServer: EmailServer,
+  ): Promise<void> {
+    const key: string = `${emailServer.host.toString()}:${emailServer.port.toNumber()}:${emailServer.username || "noauth"}`;
+
+    while ((this.semaphore.get(key) || 0) >= this.MAX_CONCURRENT_CONNECTIONS) {
+      await new Promise<void>((resolve: () => void) => {
+        setTimeout(resolve, 100);
+      });
+    }
+
+    this.semaphore.set(key, (this.semaphore.get(key) || 0) + 1);
+  }
+
+  public static releaseConnection(emailServer: EmailServer): void {
+    const key: string = `${emailServer.host.toString()}:${emailServer.port.toNumber()}:${emailServer.username || "noauth"}`;
+    const current: number = this.semaphore.get(key) || 0;
+    this.semaphore.set(key, Math.max(0, current - 1));
+  }
+
+  public static async cleanup(): Promise<void> {
+    const closePromises: Promise<void>[] = [];
+
+    for (const [, transporter] of this.pools) {
+      closePromises.push(
+        new Promise<void>((resolve: () => void) => {
+          transporter.close();
+          resolve();
+        }),
+      );
+    }
+
+    await Promise.all(closePromises);
+    this.pools.clear();
+    this.semaphore.clear();
+  }
+}
+
 export default class MailService {
   public static isSMTPConfigValid(obj: JSONObject): boolean {
     if (!obj["SMTP_USERNAME"]) {
@@ -110,19 +196,6 @@ export default class MailService {
     };
   }
 
-  public static getInternalEmailServer(): EmailServer {
-    return {
-      id: undefined,
-      username: InternalSmtpEmail.toString(),
-      password: InternalSmtpPassword,
-      host: InternalSmtpHost,
-      port: InternalSmtpPort,
-      fromEmail: InternalSmtpEmail,
-      fromName: InternalSmtpFromName,
-      secure: InternalSmtpSecure,
-    };
-  }
-
   public static async getGlobalFromEmail(): Promise<Email> {
     const emailServer: EmailServer | null = await this.getGlobalSmtpSettings();
 
@@ -205,30 +278,7 @@ export default class MailService {
       timeout?: number | undefined;
     },
   ): Transporter {
-    let tlsOptions: tls.ConnectionOptions | undefined = undefined;
-
-    if (!emailServer.secure) {
-      tlsOptions = {
-        rejectUnauthorized: false,
-      };
-    }
-
-    const privateMailer: Transporter = nodemailer.createTransport({
-      host: emailServer.host.toString(),
-      port: emailServer.port.toNumber(),
-      secure: emailServer.secure,
-      tls: tlsOptions,
-      auth:
-        emailServer.username && emailServer.password
-          ? {
-              user: emailServer.username,
-              pass: emailServer.password,
-            }
-          : undefined,
-      connectionTimeout: options.timeout || undefined,
-    });
-
-    return privateMailer;
+    return TransporterPool.getTransporter(emailServer, options);
   }
 
   private static async transportMail(
@@ -242,12 +292,49 @@ export default class MailService {
     const mailer: Transporter = this.createMailer(options.emailServer, {
       timeout: options.timeout,
     });
-    await mailer.sendMail({
-      from: `${options.emailServer.fromName.toString()} <${options.emailServer.fromEmail.toString()}>`,
-      to: mail.toEmail.toString(),
-      subject: mail.subject,
-      html: mail.body,
-    });
+
+    let lastError: any;
+    const maxRetries: number = 3;
+
+    // Acquire connection slot to prevent overwhelming the server
+    await TransporterPool.acquireConnection(options.emailServer);
+
+    try {
+      for (let attempt: number = 1; attempt <= maxRetries; attempt++) {
+        try {
+          await mailer.sendMail({
+            from: `${options.emailServer.fromName.toString()} <${options.emailServer.fromEmail.toString()}>`,
+            to: mail.toEmail.toString(),
+            subject: mail.subject,
+            html: mail.body,
+          });
+          return; // Success, exit the function
+        } catch (error) {
+          lastError = error;
+          logger.error(`Email send attempt ${attempt} failed:`);
+          logger.error(error);
+
+          if (attempt === maxRetries) {
+            break; // Don't wait after the last attempt
+          }
+
+          // Wait before retrying with jitter to prevent thundering herd
+          const baseWaitTime: number = Math.pow(2, attempt - 1) * 1000;
+          const jitter: number = Math.random() * 1000; // Add up to 1 second of jitter
+          const waitTime: number = baseWaitTime + jitter;
+
+          await new Promise<void>((resolve: (value: void) => void) => {
+            setTimeout(resolve, waitTime);
+          });
+        }
+      }
+
+      // If we reach here, all retries failed
+      throw lastError;
+    } finally {
+      // Always release the connection slot
+      TransporterPool.releaseConnection(options.emailServer);
+    }
   }
 
   public static async send(
@@ -434,17 +521,6 @@ export default class MailService {
         options.emailServer = globalEmailServer;
       }
 
-      if (
-        emailServerType === EmailServerType.Internal &&
-        (!options || !options.emailServer)
-      ) {
-        if (!options) {
-          options = {};
-        }
-
-        options.emailServer = this.getInternalEmailServer();
-      }
-
       if (options && options.emailServer && emailLog) {
         emailLog.fromEmail = options.emailServer.fromEmail;
       }
@@ -518,4 +594,8 @@ export default class MailService {
       throw err;
     }
   }
+
+  public static async cleanup(): Promise<void> {
+    await TransporterPool.cleanup();
+  }
 }

App/package-lock.json

@@ -65,6 +65,7 @@
         "@types/react-highlight": "^0.12.8",
         "@types/react-syntax-highlighter": "^15.5.13",
         "@types/uuid": "^8.3.4",
+        "@types/web-push": "^3.6.4",
         "acme-client": "^5.3.0",
         "airtable": "^0.12.2",
         "axios": "^1.7.2",
@@ -84,7 +85,6 @@
         "json5": "^2.2.3",
         "jsonwebtoken": "^9.0.0",
         "jwt-decode": "^4.0.0",
-        "lodash": "^4.17.21",
         "marked": "^12.0.2",
         "moment": "^2.30.1",
         "moment-timezone": "^0.5.45",
@@ -128,6 +128,7 @@
         "universal-cookie": "^7.2.1",
         "use-async-effect": "^2.2.6",
         "uuid": "^8.3.2",
+        "web-push": "^3.6.7",
         "zod": "^3.25.30"
       },
       "devDependencies": {
@@ -142,7 +143,6 @@
         "@types/jest": "^28.1.4",
         "@types/json2csv": "^5.0.3",
         "@types/jsonwebtoken": "^8.5.9",
-        "@types/lodash": "^4.14.202",
         "@types/node": "^17.0.45",
         "@types/node-cron": "^3.0.7",
         "@types/nodemailer": "^6.4.7",

Common/Models/DatabaseModels/AlertOwnerTeam.ts

@@ -64,6 +64,7 @@ import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 @Entity({
   name: "AlertOwnerTeam",
 })
+@Index(["alertId", "teamId", "projectId"])
 export default class AlertOwnerTeam extends BaseModel {
   @ColumnAccessControl({
     create: [

Common/Models/DatabaseModels/AlertOwnerUser.ts

@@ -63,6 +63,7 @@ import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 @Entity({
   name: "AlertOwnerUser",
 })
+@Index(["alertId", "userId", "projectId"])
 export default class AlertOwnerUser extends BaseModel {
   @ColumnAccessControl({
     create: [

Common/Models/DatabaseModels/AlertSeverity.ts

@@ -76,6 +76,7 @@ import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 @Entity({
   name: "AlertSeverity",
 })
+@Index(["projectId", "order"])
 export default class AlertSeverity extends BaseModel {
   @ColumnAccessControl({
     create: [

Common/Models/DatabaseModels/AlertState.ts

@@ -76,6 +76,10 @@ import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 @Entity({
   name: "AlertState",
 })
+@Index(["projectId", "isCreatedState"])
+@Index(["projectId", "isResolvedState"])
+@Index(["projectId", "isAcknowledgedState"])
+@Index(["projectId", "order"])
 export default class AlertState extends BaseModel {
   @ColumnAccessControl({
     create: [

Common/Models/DatabaseModels/AlertStateTimeline.ts

@@ -60,6 +60,7 @@ import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 @Entity({
   name: "AlertStateTimeline",
 })
+@Index(["alertId", "startsAt"])
 @TableMetadata({
   tableName: "AlertStateTimeline",
   singularName: "Alert State Timeline",

Common/Models/DatabaseModels/GlobalConfig.ts

@@ -17,7 +17,6 @@ import Port from "../../Types/Port";
 import { Column, Entity } from "typeorm";
 
 export enum EmailServerType {
-  Internal = "Internal",
   Sendgrid = "Sendgrid",
   CustomSMTP = "Custom SMTP",
 }

Common/Models/DatabaseModels/IncidentOwnerTeam.ts

@@ -64,6 +64,7 @@ import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 @Entity({
   name: "IncidentOwnerTeam",
 })
+@Index(["incidentId", "teamId", "projectId"])
 export default class IncidentOwnerTeam extends BaseModel {
   @ColumnAccessControl({
     create: [

Common/Models/DatabaseModels/IncidentOwnerUser.ts

@@ -63,6 +63,7 @@ import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 @Entity({
   name: "IncidentOwnerUser",
 })
+@Index(["incidentId", "userId", "projectId"])
 export default class IncidentOwnerUser extends BaseModel {
   @ColumnAccessControl({
     create: [

Common/Models/DatabaseModels/IncidentSeverity.ts

@@ -76,6 +76,7 @@ import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 @Entity({
   name: "IncidentSeverity",
 })
+@Index(["projectId", "order"])
 export default class IncidentSeverity extends BaseModel {
   @ColumnAccessControl({
     create: [

Common/Models/DatabaseModels/IncidentState.ts

@@ -76,6 +76,9 @@ import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 @Entity({
   name: "IncidentState",
 })
+@Index(["projectId", "isCreatedState"])
+@Index(["projectId", "isResolvedState"])
+@Index(["projectId", "order"])
 export default class IncidentState extends BaseModel {
   @ColumnAccessControl({
     create: [

Common/Models/DatabaseModels/IncidentStateTimeline.ts

@@ -24,6 +24,8 @@ import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 @EnableDocumentation()
 @CanAccessIfCanReadOn("incident")
 @TenantColumn("projectId")
+@Index(["incidentId", "startsAt"]) // Composite index for efficient incident timeline queries
+@Index(["incidentId", "projectId", "startsAt"]) // Alternative composite index including project
 @TableAccessControl({
   create: [
     Permission.ProjectOwner,
@@ -60,6 +62,7 @@ import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 @Entity({
   name: "IncidentStateTimeline",
 })
+@Index(["incidentId", "startsAt"])
 @TableMetadata({
   tableName: "IncidentStateTimeline",
   singularName: "Incident State Timeline",

Common/Models/DatabaseModels/Index.ts

@@ -126,6 +126,7 @@ import User from "./User";
 import UserCall from "./UserCall";
 // Notification Methods
 import UserEmail from "./UserEmail";
+import UserPush from "./UserPush";
 // User Notification Rules
 import UserNotificationRule from "./UserNotificationRule";
 import UserNotificationSetting from "./UserNotificationSetting";
@@ -294,6 +295,7 @@ const AllModelTypes: Array<{
   UserEmail,
   UserSms,
   UserCall,
+  UserPush,
 
   UserNotificationRule,
   UserOnCallLog,

Common/Models/DatabaseModels/MetricType.ts

@@ -57,10 +57,10 @@ import TelemetryService from "./TelemetryService";
   ],
 })
 @EnableWorkflow({
-  create: true,
-  delete: true,
-  update: true,
-  read: true,
+  create: false,
+  delete: false,
+  update: false,
+  read: false,
 })
 @CrudApiEndpoint(new Route("/metric-type"))
 @SlugifyColumn("name", "slug")

Common/Models/DatabaseModels/MonitorOwnerTeam.ts

@@ -72,6 +72,7 @@ import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 @Entity({
   name: "MonitorOwnerTeam",
 })
+@Index(["monitorId", "teamId", "projectId"])
 export default class MonitorOwnerTeam extends BaseModel {
   @ColumnAccessControl({
     create: [

Common/Models/DatabaseModels/MonitorOwnerUser.ts

@@ -71,6 +71,7 @@ import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 @Entity({
   name: "MonitorOwnerUser",
 })
+@Index(["monitorId", "userId", "projectId"])
 export default class MonitorOwnerUser extends BaseModel {
   @ColumnAccessControl({
     create: [

Common/Models/DatabaseModels/MonitorProbe.ts

@@ -24,6 +24,8 @@ export type MonitorStepProbeResponse = Dictionary<ProbeMonitorResponse>;
 
 @EnableDocumentation()
 @TenantColumn("projectId")
+@Index(["monitorId", "probeId"]) // Composite index for efficient monitor-probe relationship queries
+@Index(["monitorId", "projectId"]) // Alternative index for monitor queries within project
 @TableAccessControl({
   create: [
     Permission.ProjectOwner,

Common/Models/DatabaseModels/MonitorStatus.ts

@@ -76,6 +76,8 @@ import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 @Entity({
   name: "MonitorStatus",
 })
+@Index(["projectId", "isOperationalState"])
+@Index(["projectId", "isOfflineState"])
 export default class MonitorStatus extends BaseModel {
   @ColumnAccessControl({
     create: [

Common/Models/DatabaseModels/MonitorStatusTimeline.ts

@@ -25,6 +25,8 @@ import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 @EnableDocumentation()
 @CanAccessIfCanReadOn("monitor")
 @TenantColumn("projectId")
+@Index(["monitorId", "projectId", "startsAt"]) // Composite index for efficient timeline queries
+@Index(["monitorId", "startsAt"]) // Alternative index for monitor-specific timeline queries
 @TableAccessControl({
   create: [
     Permission.ProjectOwner,
@@ -62,6 +64,7 @@ import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 @Entity({
   name: "MonitorStatusTimeline",
 })
+@Index(["monitorId", "startsAt"])
 @TableMetadata({
   tableName: "MonitorStatusTimeline",
   singularName: "Monitor Status Event",

Common/Models/DatabaseModels/OnCallDutyPolicyExecutionLogTimeline.ts

@@ -51,6 +51,9 @@ import Alert from "./Alert";
 @Entity({
   name: "OnCallDutyPolicyExecutionLogTimeline",
 })
+@Index(["onCallDutyPolicyExecutionLogId", "createdAt"])
+@Index(["projectId", "createdAt"])
+@Index(["alertSentToUserId", "projectId"])
 @TableMetadata({
   tableName: "OnCallDutyPolicyExecutionLogTimeline",
   singularName: "On-Call Duty Execution Log Timeline",

Common/Models/DatabaseModels/OnCallDutyPolicyOwnerTeam.ts

@@ -72,6 +72,7 @@ import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 @Entity({
   name: "OnCallDutyPolicyOwnerTeam",
 })
+@Index(["onCallDutyPolicyId", "teamId", "projectId"])
 export default class OnCallDutyPolicyOwnerTeam extends BaseModel {
   @ColumnAccessControl({
     create: [

Common/Models/DatabaseModels/OnCallDutyPolicyOwnerUser.ts

@@ -71,6 +71,7 @@ import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 @Entity({
   name: "OnCallDutyPolicyOwnerUser",
 })
+@Index(["onCallDutyPolicyId", "userId", "projectId"])
 export default class OnCallDutyPolicyOwnerUser extends BaseModel {
   @ColumnAccessControl({
     create: [

Common/Models/DatabaseModels/ScheduledMaintenanceOwnerTeam.ts

@@ -64,6 +64,7 @@ import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 @Entity({
   name: "ScheduledMaintenanceOwnerTeam",
 })
+@Index(["scheduledMaintenanceId", "teamId", "projectId"])
 export default class ScheduledMaintenanceOwnerTeam extends BaseModel {
   @ColumnAccessControl({
     create: [

Common/Models/DatabaseModels/ScheduledMaintenanceOwnerUser.ts

@@ -63,6 +63,7 @@ import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 @Entity({
   name: "ScheduledMaintenanceOwnerUser",
 })
+@Index(["scheduledMaintenanceId", "userId", "projectId"])
 export default class ScheduledMaintenanceOwnerUser extends BaseModel {
   @ColumnAccessControl({
     create: [

Common/Models/DatabaseModels/ScheduledMaintenanceState.ts

@@ -76,6 +76,9 @@ import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 @Entity({
   name: "ScheduledMaintenanceState",
 })
+@Index(["projectId", "order"])
+@Index(["projectId", "isOngoingState"])
+@Index(["projectId", "isEndedState"])
 export default class ScheduledMaintenanceState extends BaseModel {
   @ColumnAccessControl({
     create: [

Common/Models/DatabaseModels/ScheduledMaintenanceStateTimeline.ts

@@ -59,6 +59,7 @@ import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 @Entity({
   name: "ScheduledMaintenanceStateTimeline",
 })
+@Index(["scheduledMaintenanceId", "startsAt"])
 @TableMetadata({
   tableName: "ScheduledMaintenanceStateTimeline",
   icon: IconProp.List,

Common/Models/DatabaseModels/StatusPage.ts

@@ -2049,6 +2049,41 @@ export default class StatusPage extends BaseModel {
   })
   public subscriberEmailNotificationFooterText?: string = undefined;
 
+  @ColumnAccessControl({
+    create: [
+      Permission.ProjectOwner,
+      Permission.ProjectAdmin,
+      Permission.ProjectMember,
+      Permission.CreateProjectStatusPage,
+    ],
+    read: [
+      Permission.ProjectOwner,
+      Permission.ProjectAdmin,
+      Permission.ProjectMember,
+      Permission.ReadProjectStatusPage,
+    ],
+    update: [
+      Permission.ProjectOwner,
+      Permission.ProjectAdmin,
+      Permission.ProjectMember,
+      Permission.EditProjectStatusPage,
+    ],
+  })
+  @TableColumn({
+    isDefaultValueColumn: true,
+    type: TableColumnType.Boolean,
+    title: "Enable Custom Subscriber Email Notification Footer Text",
+    description: "Enable custom footer text in subscriber email notifications.",
+    defaultValue: false,
+  })
+  @Column({
+    type: ColumnType.Boolean,
+    default: false,
+    nullable: false,
+  })
+  public enableCustomSubscriberEmailNotificationFooterText?: boolean =
+    undefined;
+
   @ColumnAccessControl({
     create: [
       Permission.ProjectOwner,

Common/Models/DatabaseModels/StatusPageOwnerTeam.ts

@@ -72,6 +72,7 @@ import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 @Entity({
   name: "StatusPageOwnerTeam",
 })
+@Index(["statusPageId", "teamId", "projectId"])
 export default class StatusPageOwnerTeam extends BaseModel {
   @ColumnAccessControl({
     create: [

Common/Models/DatabaseModels/StatusPageOwnerUser.ts

@@ -71,6 +71,7 @@ import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 @Entity({
   name: "StatusPageOwnerUser",
 })
+@Index(["statusPageId", "userId", "projectId"])
 export default class StatusPageOwnerUser extends BaseModel {
   @ColumnAccessControl({
     create: [

Common/Models/DatabaseModels/UserNotificationRule.ts

@@ -4,6 +4,7 @@ import Project from "./Project";
 import User from "./User";
 import UserCall from "./UserCall";
 import UserEmail from "./UserEmail";
+import UserPush from "./UserPush";
 import UserSMS from "./UserSMS";
 import BaseModel from "./DatabaseBaseModel/DatabaseBaseModel";
 import Route from "../../Types/API/Route";
@@ -290,6 +291,52 @@ class UserNotificationRule extends BaseModel {
   })
   public userCallId?: ObjectID = undefined;
 
+  @ColumnAccessControl({
+    create: [Permission.CurrentUser],
+    read: [Permission.CurrentUser],
+    update: [],
+  })
+  @TableColumn({
+    manyToOneRelationColumn: "userPushId",
+    type: TableColumnType.Entity,
+    modelType: UserPush,
+    title: "User Push",
+    description: "Relation to User Push Resource in which this object belongs",
+  })
+  @ManyToOne(
+    () => {
+      return UserPush;
+    },
+    {
+      eager: false,
+      nullable: true,
+      onDelete: "CASCADE",
+      orphanedRowAction: "nullify",
+    },
+  )
+  @JoinColumn({ name: "userPushId" })
+  public userPush?: UserPush = undefined;
+
+  @ColumnAccessControl({
+    create: [Permission.CurrentUser],
+    read: [Permission.CurrentUser],
+    update: [],
+  })
+  @Index()
+  @TableColumn({
+    type: TableColumnType.ObjectID,
+    required: false,
+    canReadOnRelationQuery: true,
+    title: "User Push ID",
+    description: "ID of User Push in which this object belongs",
+  })
+  @Column({
+    type: ColumnType.ObjectID,
+    nullable: true,
+    transformer: ObjectID.getDatabaseTransformer(),
+  })
+  public userPushId?: ObjectID = undefined;
+
   @ColumnAccessControl({
     create: [Permission.CurrentUser],
     read: [Permission.CurrentUser],

Common/Models/DatabaseModels/UserNotificationSetting.ts

@@ -284,6 +284,22 @@ class UserNotificationSetting extends BaseModel {
     default: false,
   })
   public alertByCall?: boolean = undefined;
+
+  @ColumnAccessControl({
+    create: [Permission.CurrentUser],
+    read: [Permission.CurrentUser],
+    update: [Permission.CurrentUser],
+  })
+  @TableColumn({
+    isDefaultValueColumn: true,
+    type: TableColumnType.Boolean,
+    defaultValue: false,
+  })
+  @Column({
+    type: ColumnType.Boolean,
+    default: false,
+  })
+  public alertByPush?: boolean = undefined;
 }
 
 export default UserNotificationSetting;

Common/Models/DatabaseModels/UserOnCallLogTimeline.ts

@@ -10,6 +10,7 @@ import User from "./User";
 import UserCall from "./UserCall";
 import UserEmail from "./UserEmail";
 import UserNotificationRule from "./UserNotificationRule";
+import UserPush from "./UserPush";
 import UserOnCallLog from "./UserOnCallLog";
 import UserSMS from "./UserSMS";
 import BaseModel from "./DatabaseBaseModel/DatabaseBaseModel";
@@ -51,6 +52,9 @@ import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
 @Entity({
   name: "UserOnCallLogTimeline",
 })
+@Index(["userId", "createdAt"])
+@Index(["onCallDutyPolicyExecutionLogId", "status"])
+@Index(["projectId", "status"])
 @TableMetadata({
   tableName: "UserOnCallLogTimeline",
   singularName: "User On-Call Log Timeline",
@@ -876,6 +880,52 @@ export default class UserOnCallLogTimeline extends BaseModel {
   })
   public userEmailId?: ObjectID = undefined;
 
+  @ColumnAccessControl({
+    create: [],
+    read: [Permission.CurrentUser],
+    update: [],
+  })
+  @TableColumn({
+    manyToOneRelationColumn: "userPushId",
+    type: TableColumnType.Entity,
+    modelType: UserPush,
+    title: "User Push",
+    description: "Relation to User Push Resource in which this object belongs",
+  })
+  @ManyToOne(
+    () => {
+      return UserPush;
+    },
+    {
+      eager: false,
+      nullable: true,
+      onDelete: "CASCADE",
+      orphanedRowAction: "nullify",
+    },
+  )
+  @JoinColumn({ name: "userPushId" })
+  public userPush?: UserPush = undefined;
+
+  @ColumnAccessControl({
+    create: [],
+    read: [Permission.CurrentUser],
+    update: [],
+  })
+  @Index()
+  @TableColumn({
+    type: TableColumnType.ObjectID,
+    required: false,
+    canReadOnRelationQuery: true,
+    title: "User Push ID",
+    description: "ID of User Push in which this object belongs",
+  })
+  @Column({
+    type: ColumnType.ObjectID,
+    nullable: true,
+    transformer: ObjectID.getDatabaseTransformer(),
+  })
+  public userPushId?: ObjectID = undefined;
+
   @ColumnAccessControl({
     create: [],
     read: [],

Common/Models/DatabaseModels/UserPush.ts

@@ -0,0 +1,301 @@
+import Project from "./Project";
+import User from "./User";
+import BaseModel from "./DatabaseBaseModel/DatabaseBaseModel";
+import Route from "../../Types/API/Route";
+import AllowAccessIfSubscriptionIsUnpaid from "../../Types/Database/AccessControl/AllowAccessIfSubscriptionIsUnpaid";
+import ColumnAccessControl from "../../Types/Database/AccessControl/ColumnAccessControl";
+import TableAccessControl from "../../Types/Database/AccessControl/TableAccessControl";
+import ColumnLength from "../../Types/Database/ColumnLength";
+import ColumnType from "../../Types/Database/ColumnType";
+import CrudApiEndpoint from "../../Types/Database/CrudApiEndpoint";
+import CurrentUserCanAccessRecordBy from "../../Types/Database/CurrentUserCanAccessRecordBy";
+import TableColumn from "../../Types/Database/TableColumn";
+import TableColumnType from "../../Types/Database/TableColumnType";
+import TableMetadata from "../../Types/Database/TableMetadata";
+import TenantColumn from "../../Types/Database/TenantColumn";
+import IconProp from "../../Types/Icon/IconProp";
+import ObjectID from "../../Types/ObjectID";
+import Permission from "../../Types/Permission";
+import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
+
+@TenantColumn("projectId")
+@AllowAccessIfSubscriptionIsUnpaid()
+@TableAccessControl({
+  create: [Permission.CurrentUser],
+  read: [Permission.CurrentUser],
+  delete: [Permission.CurrentUser],
+  update: [Permission.CurrentUser],
+})
+@CrudApiEndpoint(new Route("/user-push"))
+@Entity({
+  name: "UserPush",
+})
+@TableMetadata({
+  tableName: "UserPush",
+  singularName: "Device for Push Notifications",
+  pluralName: "Devices for Push Notifications",
+  icon: IconProp.Bell,
+  tableDescription: "Devices which will be used for push notifications.",
+})
+@CurrentUserCanAccessRecordBy("userId")
+class UserPush extends BaseModel {
+  @ColumnAccessControl({
+    create: [Permission.CurrentUser],
+    read: [Permission.CurrentUser],
+    update: [],
+  })
+  @TableColumn({
+    manyToOneRelationColumn: "projectId",
+    type: TableColumnType.Entity,
+    modelType: Project,
+    title: "Project",
+    description: "Relation to Project Resource in which this object belongs",
+  })
+  @ManyToOne(
+    () => {
+      return Project;
+    },
+    {
+      eager: false,
+      nullable: true,
+      onDelete: "CASCADE",
+      orphanedRowAction: "nullify",
+    },
+  )
+  @JoinColumn({ name: "projectId" })
+  public project?: Project = undefined;
+
+  @ColumnAccessControl({
+    create: [Permission.CurrentUser],
+    read: [Permission.CurrentUser],
+    update: [],
+  })
+  @Index()
+  @TableColumn({
+    type: TableColumnType.ObjectID,
+    required: true,
+    canReadOnRelationQuery: true,
+    title: "Project ID",
+    description: "ID of your OneUptime Project in which this object belongs",
+  })
+  @Column({
+    type: ColumnType.ObjectID,
+    nullable: false,
+    transformer: ObjectID.getDatabaseTransformer(),
+  })
+  public projectId?: ObjectID = undefined;
+
+  @ColumnAccessControl({
+    create: [Permission.CurrentUser],
+    read: [Permission.CurrentUser],
+    update: [],
+  })
+  @TableColumn({
+    title: "Device Token",
+    required: true,
+    unique: false,
+    type: TableColumnType.VeryLongText,
+    canReadOnRelationQuery: true,
+  })
+  @Column({
+    type: ColumnType.VeryLongText,
+    unique: false,
+    nullable: false,
+  })
+  public deviceToken?: string = undefined;
+
+  @ColumnAccessControl({
+    create: [Permission.CurrentUser],
+    read: [Permission.CurrentUser],
+    update: [],
+  })
+  @TableColumn({
+    title: "Device Type",
+    required: true,
+    unique: false,
+    type: TableColumnType.ShortText,
+    canReadOnRelationQuery: true,
+  })
+  @Column({
+    type: ColumnType.ShortText,
+    length: ColumnLength.ShortText,
+    unique: false,
+    nullable: false,
+  })
+  public deviceType?: "web" = "web" as const; // Only web support for now
+
+  @ColumnAccessControl({
+    create: [Permission.CurrentUser],
+    read: [Permission.CurrentUser],
+    update: [],
+  })
+  @TableColumn({
+    title: "Device Name",
+    required: false,
+    unique: false,
+    type: TableColumnType.ShortText,
+    canReadOnRelationQuery: true,
+  })
+  @Column({
+    type: ColumnType.ShortText,
+    length: ColumnLength.ShortText,
+    unique: false,
+    nullable: true,
+  })
+  public deviceName?: string = undefined;
+
+  @ColumnAccessControl({
+    create: [Permission.CurrentUser],
+    read: [Permission.CurrentUser],
+    update: [],
+  })
+  @TableColumn({
+    manyToOneRelationColumn: "user",
+    type: TableColumnType.Entity,
+    modelType: User,
+    title: "User",
+    description: "Relation to User who this device belongs to",
+  })
+  @ManyToOne(
+    () => {
+      return User;
+    },
+    {
+      eager: false,
+      nullable: true,
+      onDelete: "CASCADE",
+      orphanedRowAction: "nullify",
+    },
+  )
+  @JoinColumn({ name: "userId" })
+  public user?: User = undefined;
+
+  @ColumnAccessControl({
+    create: [Permission.CurrentUser],
+    read: [Permission.CurrentUser],
+    update: [],
+  })
+  @TableColumn({
+    type: TableColumnType.ObjectID,
+    title: "User ID",
+    description: "User ID who this device belongs to",
+  })
+  @Column({
+    type: ColumnType.ObjectID,
+    nullable: true,
+    transformer: ObjectID.getDatabaseTransformer(),
+  })
+  @Index()
+  public userId?: ObjectID = undefined;
+
+  @ColumnAccessControl({
+    create: [Permission.CurrentUser],
+    read: [Permission.CurrentUser],
+    update: [],
+  })
+  @TableColumn({
+    manyToOneRelationColumn: "createdByUserId",
+    type: TableColumnType.Entity,
+    modelType: User,
+    title: "Created by User",
+    description:
+      "Relation to User who created this object (if this object was created by a User)",
+  })
+  @ManyToOne(
+    () => {
+      return User;
+    },
+    {
+      eager: false,
+      nullable: true,
+      onDelete: "SET NULL",
+      orphanedRowAction: "nullify",
+    },
+  )
+  @JoinColumn({ name: "createdByUserId" })
+  public createdByUser?: User = undefined;
+
+  @ColumnAccessControl({
+    create: [Permission.CurrentUser],
+    read: [Permission.CurrentUser],
+    update: [],
+  })
+  @TableColumn({
+    type: TableColumnType.ObjectID,
+    title: "Created by User ID",
+    description:
+      "User ID who created this object (if this object was created by a User)",
+  })
+  @Column({
+    type: ColumnType.ObjectID,
+    nullable: true,
+    transformer: ObjectID.getDatabaseTransformer(),
+  })
+  public createdByUserId?: ObjectID = undefined;
+
+  @ColumnAccessControl({
+    create: [],
+    read: [],
+    update: [],
+  })
+  @TableColumn({
+    manyToOneRelationColumn: "deletedByUserId",
+    type: TableColumnType.Entity,
+    title: "Deleted by User",
+    modelType: User,
+    description:
+      "Relation to User who deleted this object (if this object was deleted by a User)",
+  })
+  @ManyToOne(
+    () => {
+      return User;
+    },
+    {
+      cascade: false,
+      eager: false,
+      nullable: true,
+      onDelete: "SET NULL",
+      orphanedRowAction: "nullify",
+    },
+  )
+  @JoinColumn({ name: "deletedByUserId" })
+  public deletedByUser?: User = undefined;
+
+  @ColumnAccessControl({
+    create: [],
+    read: [],
+    update: [],
+  })
+  @TableColumn({
+    type: TableColumnType.ObjectID,
+    title: "Deleted by User ID",
+    description:
+      "User ID who deleted this object (if this object was deleted by a User)",
+  })
+  @Column({
+    type: ColumnType.ObjectID,
+    nullable: true,
+    transformer: ObjectID.getDatabaseTransformer(),
+  })
+  public deletedByUserId?: ObjectID = undefined;
+
+  @ColumnAccessControl({
+    create: [],
+    read: [Permission.CurrentUser],
+    update: [],
+  })
+  @TableColumn({
+    title: "Is Verified",
+    description: "Is this device verified?",
+    isDefaultValueColumn: true,
+    type: TableColumnType.Boolean,
+    defaultValue: false,
+  })
+  @Column({
+    type: ColumnType.Boolean,
+    default: false,
+  })
+  public isVerified?: boolean = undefined;
+}
+
+export default UserPush;

Common/Scripts/generate-service-worker.js

@@ -0,0 +1,144 @@
+#!/usr/bin/env node
+
+/**
+ * Universal Service Worker Generator for OneUptime Services
+ * 
+ * This script can be used by any OneUptime service to generate
+ * a service worker from a template with dynamic versioning.
+ * 
+ * Usage:
+ *   node generate-service-worker.js [template-path] [output-path]
+ * 
+ * Example:
+ *   node generate-service-worker.js sw.js.template public/sw.js
+ */
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+
+// Default values
+const DEFAULT_APP_VERSION = '1.0.0';
+const DEFAULT_GIT_SHA = 'local';
+
+/**
+ * Get app version from environment or package.json
+ */
+function getAppVersion(packageJsonPath) {
+  // First try environment variable (Docker build)
+  if (process.env.APP_VERSION) {
+    return process.env.APP_VERSION;
+  }
+  
+  // Fallback to package.json version
+  try {
+    const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
+    return packageJson.version || DEFAULT_APP_VERSION;
+  } catch (error) {
+    console.warn('Could not read package.json, using default version');
+    return DEFAULT_APP_VERSION;
+  }
+}
+
+/**
+ * Get git SHA from environment
+ */
+function getGitSha() {
+  // Try environment variable first (Docker build)
+  if (process.env.GIT_SHA) {
+    return process.env.GIT_SHA.substring(0, 8); // Short SHA
+  }
+  
+  // Try to get from git command if available
+  try {
+    const { execSync } = require('child_process');
+    const gitSha = execSync('git rev-parse --short HEAD', { encoding: 'utf8' }).trim();
+    return gitSha;
+  } catch (error) {
+    // Fallback to timestamp-based hash for local development
+    const timestamp = Date.now().toString();
+    const hash = crypto.createHash('md5').update(timestamp).digest('hex');
+    return hash.substring(0, 8);
+  }
+}
+
+/**
+ * Generate service worker from template
+ */
+function generateServiceWorker(templatePath, outputPath, serviceName = 'OneUptime') {
+  // Check if template exists
+  if (!fs.existsSync(templatePath)) {
+    console.error('❌ Service worker template not found:', templatePath);
+    process.exit(1);
+  }
+  
+  // Read template
+  const template = fs.readFileSync(templatePath, 'utf8');
+  
+  // Get version information
+  const packageJsonPath = path.join(path.dirname(templatePath), 'package.json');
+  const appVersion = getAppVersion(packageJsonPath);
+  const gitSha = getGitSha();
+  const buildTimestamp = new Date().toISOString();
+  
+  console.log(`🔧 Generating service worker for ${serviceName}...`);
+  console.log(`   App Version: ${appVersion}`);
+  console.log(`   Git SHA: ${gitSha}`);
+  console.log(`   Build Time: ${buildTimestamp}`);
+  
+  // Replace placeholders
+  const generatedContent = template
+    .replace(/\{\{APP_VERSION\}\}/g, appVersion)
+    .replace(/\{\{GIT_SHA\}\}/g, gitSha)
+    .replace(/\{\{BUILD_TIMESTAMP\}\}/g, buildTimestamp)
+    .replace(/\{\{SERVICE_NAME\}\}/g, serviceName);
+  
+  // Add generation comment at the top
+  const header = `/*
+ * Generated Service Worker for ${serviceName}
+ * 
+ * Generated at: ${buildTimestamp}
+ * App Version: ${appVersion}
+ * Git SHA: ${gitSha}
+ * 
+ * DO NOT EDIT THIS FILE DIRECTLY
+ * Edit the template file instead and run the generator script
+ */
+
+`;
+  
+  const finalContent = header + generatedContent;
+  
+  // Ensure output directory exists
+  const outputDir = path.dirname(outputPath);
+  if (!fs.existsSync(outputDir)) {
+    fs.mkdirSync(outputDir, { recursive: true });
+  }
+  
+  // Write generated service worker
+  fs.writeFileSync(outputPath, finalContent, 'utf8');
+  
+  console.log('✅ Service worker generated successfully:', outputPath);
+  console.log(`   Cache version: oneuptime-v${appVersion}-${gitSha}`);
+}
+
+// Command line interface
+if (require.main === module) {
+  const args = process.argv.slice(2);
+  const templatePath = args[0] || 'sw.js.template';
+  const outputPath = args[1] || 'public/sw.js';
+  const serviceName = args[2] || path.basename(process.cwd());
+  
+  try {
+    // Resolve paths relative to current working directory
+    const resolvedTemplatePath = path.resolve(templatePath);
+    const resolvedOutputPath = path.resolve(outputPath);
+    
+    generateServiceWorker(resolvedTemplatePath, resolvedOutputPath, serviceName);
+  } catch (error) {
+    console.error('❌ Failed to generate service worker:', error.message);
+    process.exit(1);
+  }
+}
+
+module.exports = { generateServiceWorker, getAppVersion, getGitSha };

Common/Server/API/StatusPageAPI.ts

@@ -1152,6 +1152,7 @@ export default class StatusPageAPI extends BaseAPI<
               select: {
                 _id: true,
                 createdAt: true,
+                startsAt: true,
                 incidentId: true,
                 incidentState: {
                   _id: true,
@@ -1164,7 +1165,7 @@ export default class StatusPageAPI extends BaseAPI<
               },
 
               sort: {
-                createdAt: SortOrder.Descending, // new note first
+                startsAt: SortOrder.Descending, // newer state changes first
               },
               skip: 0,
               limit: LIMIT_PER_PROJECT,
@@ -1340,6 +1341,7 @@ export default class StatusPageAPI extends BaseAPI<
                 select: {
                   _id: true,
                   createdAt: true,
+                  startsAt: true,
                   scheduledMaintenanceId: true,
                   scheduledMaintenanceState: {
                     _id: true,
@@ -1352,7 +1354,7 @@ export default class StatusPageAPI extends BaseAPI<
                 },
 
                 sort: {
-                  createdAt: SortOrder.Descending, // new note first
+                  startsAt: SortOrder.Descending, // newer state changes first
                 },
                 skip: 0,
                 limit: LIMIT_PER_PROJECT,
@@ -1878,6 +1880,7 @@ export default class StatusPageAPI extends BaseAPI<
           select: {
             _id: true,
             createdAt: true,
+            startsAt: true,
             scheduledMaintenanceId: true,
             scheduledMaintenanceState: {
               name: true,
@@ -1889,7 +1892,7 @@ export default class StatusPageAPI extends BaseAPI<
           },
 
           sort: {
-            createdAt: SortOrder.Descending, // new note first
+            startsAt: SortOrder.Descending, // newer state changes first
           },
           skip: 0,
           limit: LIMIT_PER_PROJECT,
@@ -2341,7 +2344,7 @@ export default class StatusPageAPI extends BaseAPI<
                 ? "true"
                 : "false",
               subscriberEmailNotificationFooterText:
-                statusPage.subscriberEmailNotificationFooterText || "",
+                StatusPageServiceType.getSubscriberEmailFooterText(statusPage),
 
               manageSubscriptionUrl: manageUrlink,
             },
@@ -2924,6 +2927,7 @@ export default class StatusPageAPI extends BaseAPI<
         select: {
           _id: true,
           createdAt: true,
+          startsAt: true,
           incidentId: true,
           incidentState: {
             name: true,
@@ -2931,7 +2935,7 @@ export default class StatusPageAPI extends BaseAPI<
           },
         },
         sort: {
-          createdAt: SortOrder.Descending, // new note first
+          startsAt: SortOrder.Descending, // newer state changes first
         },
 
         skip: 0,

Common/Server/API/UserPushAPI.ts

@@ -0,0 +1,302 @@
+import UserMiddleware from "../Middleware/UserAuthorization";
+import UserPushService, {
+  Service as UserPushServiceType,
+} from "../Services/UserPushService";
+import PushNotificationService from "../Services/PushNotificationService";
+import PushNotificationUtil from "../Utils/PushNotificationUtil";
+import {
+  ExpressRequest,
+  ExpressResponse,
+  NextFunction,
+  OneUptimeRequest,
+} from "../Utils/Express";
+import Response from "../Utils/Response";
+import BaseAPI from "./BaseAPI";
+import BadDataException from "../../Types/Exception/BadDataException";
+import ObjectID from "../../Types/ObjectID";
+import UserPush from "../../Models/DatabaseModels/UserPush";
+import PushNotificationMessage from "../../Types/PushNotification/PushNotificationMessage";
+
+export default class UserPushAPI extends BaseAPI<
+  UserPush,
+  UserPushServiceType
+> {
+  public constructor() {
+    super(UserPush, UserPushService);
+
+    this.router.post(
+      `/user-push/register`,
+      UserMiddleware.getUserMiddleware,
+      async (req: ExpressRequest, res: ExpressResponse, next: NextFunction) => {
+        try {
+          req = req as OneUptimeRequest;
+
+          if (!req.body.deviceToken) {
+            return Response.sendErrorResponse(
+              req,
+              res,
+              new BadDataException("Device token is required"),
+            );
+          }
+
+          if (!req.body.deviceType || req.body.deviceType !== "web") {
+            return Response.sendErrorResponse(
+              req,
+              res,
+              new BadDataException("Only web device type is supported"),
+            );
+          }
+
+          if (!req.body.projectId) {
+            return Response.sendErrorResponse(
+              req,
+              res,
+              new BadDataException("Project ID is required"),
+            );
+          }
+
+          // Check if device is already registered
+          const existingDevice: UserPush | null = await this.service.findOneBy({
+            query: {
+              userId: (req as OneUptimeRequest).userAuthorization!.userId!,
+              projectId: new ObjectID(req.body.projectId),
+              deviceToken: req.body.deviceToken,
+            },
+            props: {
+              isRoot: true,
+            },
+            select: {
+              _id: true,
+            },
+          });
+
+          if (existingDevice) {
+            // Mark as used and return a specific response indicating device was already registered
+            throw new BadDataException(
+              "This device is already registered for push notifications",
+            );
+          }
+
+          // Create new device registration
+          const userPush: UserPush = new UserPush();
+          userPush.userId = (
+            req as OneUptimeRequest
+          ).userAuthorization!.userId!;
+          userPush.projectId = new ObjectID(req.body.projectId);
+          userPush.deviceToken = req.body.deviceToken;
+          userPush.deviceType = req.body.deviceType;
+          userPush.deviceName = req.body.deviceName || "Unknown Device";
+          userPush.isVerified = true; // For web push, we consider it verified immediately
+
+          const savedDevice: UserPush = await this.service.create({
+            data: userPush,
+            props: {
+              isRoot: true,
+            },
+          });
+
+          return Response.sendJsonObjectResponse(req, res, {
+            success: true,
+            deviceId: savedDevice._id!.toString(),
+          });
+        } catch (error: any) {
+          next(error);
+        }
+      },
+    );
+
+    this.router.post(
+      `/user-push/:deviceId/test-notification`,
+      UserMiddleware.getUserMiddleware,
+      async (req: ExpressRequest, res: ExpressResponse) => {
+        req = req as OneUptimeRequest;
+
+        if (!req.params["deviceId"]) {
+          return Response.sendErrorResponse(
+            req,
+            res,
+            new BadDataException("Device ID is required"),
+          );
+        }
+
+        // Get the device
+        const device: UserPush | null = await this.service.findOneById({
+          id: new ObjectID(req.params["deviceId"]),
+          props: {
+            isRoot: true,
+          },
+          select: {
+            userId: true,
+            deviceToken: true,
+            deviceType: true,
+            isVerified: true,
+          },
+        });
+
+        if (!device) {
+          return Response.sendErrorResponse(
+            req,
+            res,
+            new BadDataException("Device not found"),
+          );
+        }
+
+        // Check if the device belongs to the current user
+        if (
+          device.userId?.toString() !==
+          (req as OneUptimeRequest).userAuthorization!.userId!.toString()
+        ) {
+          return Response.sendErrorResponse(
+            req,
+            res,
+            new BadDataException("Unauthorized access to device"),
+          );
+        }
+
+        if (!device.isVerified) {
+          return Response.sendErrorResponse(
+            req,
+            res,
+            new BadDataException("Device is not verified"),
+          );
+        }
+
+        try {
+          // Send test notification
+          const testMessage: PushNotificationMessage =
+            PushNotificationUtil.createGenericNotification({
+              title: "Test Notification from OneUptime",
+              body: "This is a test notification to verify your device is working correctly.",
+              clickAction: "/dashboard",
+              tag: "test-notification",
+              requireInteraction: false,
+            });
+
+          await PushNotificationService.sendPushNotification(
+            {
+              deviceTokens: [device.deviceToken!],
+              message: testMessage,
+              deviceType: device.deviceType!,
+            },
+            {
+              isSensitive: false,
+            },
+          );
+
+          return Response.sendJsonObjectResponse(req, res, {
+            success: true,
+            message: "Test notification sent successfully",
+          });
+        } catch (error: any) {
+          return Response.sendErrorResponse(
+            req,
+            res,
+            new BadDataException(
+              `Failed to send test notification: ${error.message}`,
+            ),
+          );
+        }
+      },
+    );
+
+    this.router.post(
+      `/user-push/:deviceId/verify`,
+      UserMiddleware.getUserMiddleware,
+      async (req: ExpressRequest, res: ExpressResponse) => {
+        req = req as OneUptimeRequest;
+
+        if (!req.params["deviceId"]) {
+          return Response.sendErrorResponse(
+            req,
+            res,
+            new BadDataException("Device ID is required"),
+          );
+        }
+
+        const device: UserPush | null = await this.service.findOneById({
+          id: new ObjectID(req.params["deviceId"]),
+          props: {
+            isRoot: true,
+          },
+          select: {
+            userId: true,
+          },
+        });
+
+        if (!device) {
+          return Response.sendErrorResponse(
+            req,
+            res,
+            new BadDataException("Device not found"),
+          );
+        }
+
+        // Check if the device belongs to the current user
+        if (
+          device.userId?.toString() !==
+          (req as OneUptimeRequest).userAuthorization!.userId!.toString()
+        ) {
+          return Response.sendErrorResponse(
+            req,
+            res,
+            new BadDataException("Unauthorized access to device"),
+          );
+        }
+
+        await this.service.verifyDevice(device._id!.toString());
+
+        return Response.sendEmptySuccessResponse(req, res);
+      },
+    );
+
+    this.router.post(
+      `/user-push/:deviceId/unverify`,
+      UserMiddleware.getUserMiddleware,
+      async (req: ExpressRequest, res: ExpressResponse) => {
+        req = req as OneUptimeRequest;
+
+        if (!req.params["deviceId"]) {
+          return Response.sendErrorResponse(
+            req,
+            res,
+            new BadDataException("Device ID is required"),
+          );
+        }
+
+        const device: UserPush | null = await this.service.findOneById({
+          id: new ObjectID(req.params["deviceId"]),
+          props: {
+            isRoot: true,
+          },
+          select: {
+            userId: true,
+          },
+        });
+
+        if (!device) {
+          return Response.sendErrorResponse(
+            req,
+            res,
+            new BadDataException("Device not found"),
+          );
+        }
+
+        // Check if the device belongs to the current user
+        if (
+          device.userId?.toString() !==
+          (req as OneUptimeRequest).userAuthorization!.userId!.toString()
+        ) {
+          return Response.sendErrorResponse(
+            req,
+            res,
+            new BadDataException("Unauthorized access to device"),
+          );
+        }
+
+        await this.service.unverifyDevice(device._id!.toString());
+
+        return Response.sendEmptySuccessResponse(req, res);
+      },
+    );
+  }
+}

Common/Server/EnvironmentConfig.ts

@@ -327,3 +327,13 @@ export const SlackAppClientSecret: string | null =
   process.env["SLACK_APP_CLIENT_SECRET"] || null;
 export const SlackAppSigningSecret: string | null =
   process.env["SLACK_APP_SIGNING_SECRET"] || null;
+
+// VAPID Configuration for Web Push Notifications
+export const VapidPublicKey: string | undefined =
+  process.env["VAPID_PUBLIC_KEY"] || undefined;
+
+export const VapidPrivateKey: string | undefined =
+  process.env["VAPID_PRIVATE_KEY"] || undefined;
+
+export const VapidSubject: string =
+  process.env["VAPID_SUBJECT"] || "mailto:support@oneuptime.com";

Common/Server/Infrastructure/Postgres/SchemaMigrations/1752659054949-MigrationName.ts

@@ -0,0 +1,87 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class MigrationName1752659054949 implements MigrationInterface {
+  public name = "MigrationName1752659054949";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `CREATE TABLE "UserPush" ("_id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "deletedAt" TIMESTAMP WITH TIME ZONE, "version" integer NOT NULL, "projectId" uuid NOT NULL, "deviceToken" character varying(500) NOT NULL, "deviceType" character varying(100) NOT NULL, "deviceName" character varying(100), "userId" uuid, "createdByUserId" uuid, "deletedByUserId" uuid, "isVerified" boolean NOT NULL DEFAULT false, "lastUsedAt" TIMESTAMP WITH TIME ZONE, CONSTRAINT "PK_bc3271178002ba8d92824d36db6" PRIMARY KEY ("_id"))`,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_24d281c51868189d985c4a81cb" ON "UserPush" ("projectId") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_507f0b3fea4f091410f99d2170" ON "UserPush" ("userId") `,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserNotificationRule" ADD "userPushId" uuid`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserNotificationSetting" ADD "alertByPush" boolean NOT NULL DEFAULT false`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "rotation" SET DEFAULT '{"_type":"Recurring","value":{"intervalType":"Day","intervalCount":{"_type":"PositiveNumber","value":1}}}'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "restrictionTimes" SET DEFAULT '{"_type":"RestrictionTimes","value":{"restictionType":"None","dayRestrictionTimes":null,"weeklyRestrictionTimes":[]}}'`,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_e6d756cbda1e68aae728531269" ON "UserNotificationRule" ("userPushId") `,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserPush" ADD CONSTRAINT "FK_24d281c51868189d985c4a81cb8" FOREIGN KEY ("projectId") REFERENCES "Project"("_id") ON DELETE CASCADE ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserPush" ADD CONSTRAINT "FK_507f0b3fea4f091410f99d2170a" FOREIGN KEY ("userId") REFERENCES "User"("_id") ON DELETE CASCADE ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserPush" ADD CONSTRAINT "FK_2d2819503cd8a8517e9ce502bd8" FOREIGN KEY ("createdByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserPush" ADD CONSTRAINT "FK_964b240ccbb12a9a8c947272540" FOREIGN KEY ("deletedByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserNotificationRule" ADD CONSTRAINT "FK_e6d756cbda1e68aae7285312694" FOREIGN KEY ("userPushId") REFERENCES "UserPush"("_id") ON DELETE CASCADE ON UPDATE NO ACTION`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "UserNotificationRule" DROP CONSTRAINT "FK_e6d756cbda1e68aae7285312694"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserPush" DROP CONSTRAINT "FK_964b240ccbb12a9a8c947272540"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserPush" DROP CONSTRAINT "FK_2d2819503cd8a8517e9ce502bd8"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserPush" DROP CONSTRAINT "FK_507f0b3fea4f091410f99d2170a"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserPush" DROP CONSTRAINT "FK_24d281c51868189d985c4a81cb8"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_e6d756cbda1e68aae728531269"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "restrictionTimes" SET DEFAULT '{"_type": "RestrictionTimes", "value": {"restictionType": "None", "dayRestrictionTimes": null, "weeklyRestrictionTimes": []}}'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "rotation" SET DEFAULT '{"_type": "Recurring", "value": {"intervalType": "Day", "intervalCount": {"_type": "PositiveNumber", "value": 1}}}'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserNotificationSetting" DROP COLUMN "alertByPush"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserNotificationRule" DROP COLUMN "userPushId"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_507f0b3fea4f091410f99d2170"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_24d281c51868189d985c4a81cb"`,
+    );
+    await queryRunner.query(`DROP TABLE "UserPush"`);
+  }
+}

Common/Server/Infrastructure/Postgres/SchemaMigrations/1752774923063-MigrationName.ts

@@ -0,0 +1,27 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class MigrationName1752774923063 implements MigrationInterface {
+  public name = "MigrationName1752774923063";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE "UserPush" DROP COLUMN "lastUsedAt"`);
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "rotation" SET DEFAULT '{"_type":"Recurring","value":{"intervalType":"Day","intervalCount":{"_type":"PositiveNumber","value":1}}}'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "restrictionTimes" SET DEFAULT '{"_type":"RestrictionTimes","value":{"restictionType":"None","dayRestrictionTimes":null,"weeklyRestrictionTimes":[]}}'`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "restrictionTimes" SET DEFAULT '{"_type": "RestrictionTimes", "value": {"restictionType": "None", "dayRestrictionTimes": null, "weeklyRestrictionTimes": []}}'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "rotation" SET DEFAULT '{"_type": "Recurring", "value": {"intervalType": "Day", "intervalCount": {"_type": "PositiveNumber", "value": 1}}}'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserPush" ADD "lastUsedAt" TIMESTAMP WITH TIME ZONE`,
+    );
+  }
+}

Common/Server/Infrastructure/Postgres/SchemaMigrations/1753109689244-MigrationName.ts

@@ -0,0 +1,41 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class MigrationName1753109689244 implements MigrationInterface {
+  public name = "MigrationName1753109689244";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "UserOnCallLogTimeline" ADD "userPushId" uuid`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "rotation" SET DEFAULT '{"_type":"Recurring","value":{"intervalType":"Day","intervalCount":{"_type":"PositiveNumber","value":1}}}'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "restrictionTimes" SET DEFAULT '{"_type":"RestrictionTimes","value":{"restictionType":"None","dayRestrictionTimes":null,"weeklyRestrictionTimes":[]}}'`,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_d3e187a7828a990cdf6429a692" ON "UserOnCallLogTimeline" ("userPushId") `,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserOnCallLogTimeline" ADD CONSTRAINT "FK_d3e187a7828a990cdf6429a692f" FOREIGN KEY ("userPushId") REFERENCES "UserPush"("_id") ON DELETE CASCADE ON UPDATE NO ACTION`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "UserOnCallLogTimeline" DROP CONSTRAINT "FK_d3e187a7828a990cdf6429a692f"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_d3e187a7828a990cdf6429a692"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "restrictionTimes" SET DEFAULT '{"_type": "RestrictionTimes", "value": {"restictionType": "None", "dayRestrictionTimes": null, "weeklyRestrictionTimes": []}}'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "rotation" SET DEFAULT '{"_type": "Recurring", "value": {"intervalType": "Day", "intervalCount": {"_type": "PositiveNumber", "value": 1}}}'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserOnCallLogTimeline" DROP COLUMN "userPushId"`,
+    );
+  }
+}

Common/Server/Infrastructure/Postgres/SchemaMigrations/1753131488925-AddEnableCustomSubscriberEmailNotificationFooterText.ts

@@ -0,0 +1,26 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddEnableCustomSubscriberEmailNotificationFooterText1753131488925
+  implements MigrationInterface
+{
+  public name =
+    "AddEnableCustomSubscriberEmailNotificationFooterText1753131488925";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "StatusPage" ADD "enableCustomSubscriberEmailNotificationFooterText" boolean NOT NULL DEFAULT false`,
+    );
+
+    // Data migration: Set existing status pages to have enableCustomSubscriberEmailNotificationFooterText = true
+    // This ensures backward compatibility for existing status pages that already have custom footer text
+    await queryRunner.query(
+      `UPDATE "StatusPage" SET "enableCustomSubscriberEmailNotificationFooterText" = true WHERE "subscriberEmailNotificationFooterText" IS NOT NULL AND "subscriberEmailNotificationFooterText" != ''`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "StatusPage" DROP COLUMN "enableCustomSubscriberEmailNotificationFooterText"`,
+    );
+  }
+}

Common/Server/Infrastructure/Postgres/SchemaMigrations/1753343522987-MigrationName.ts

@@ -0,0 +1,17 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class MigrationName1753343522987 implements MigrationInterface {
+  public name = "MigrationName1753343522987";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "UserPush" ALTER COLUMN "deviceToken" TYPE text`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "UserPush" ALTER COLUMN "deviceToken" TYPE character varying(500)`,
+    );
+  }
+}

Common/Server/Infrastructure/Postgres/SchemaMigrations/1753377161288-MigrationName.ts

@@ -0,0 +1,47 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class MigrationName1753377161288 implements MigrationInterface {
+  public name = "MigrationName1753377161288";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `CREATE INDEX "IDX_16db786b562f1db40c93d463c7" ON "IncidentStateTimeline" ("incidentId", "projectId", "startsAt") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_410cf30b966f88c287d368aa48" ON "IncidentStateTimeline" ("incidentId", "startsAt") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_ac648c5f1961bc1d5ec1ba21bd" ON "MonitorProbe" ("monitorId", "projectId") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_bde10e600047b06718db90a636" ON "MonitorProbe" ("monitorId", "probeId") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_570f164ca5b3559eb8555eb1b1" ON "MonitorStatusTimeline" ("monitorId", "startsAt") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_466d392af405ccf2e8b552eb0e" ON "MonitorStatusTimeline" ("monitorId", "projectId", "startsAt") `,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_466d392af405ccf2e8b552eb0e"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_570f164ca5b3559eb8555eb1b1"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_bde10e600047b06718db90a636"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_ac648c5f1961bc1d5ec1ba21bd"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_410cf30b966f88c287d368aa48"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_16db786b562f1db40c93d463c7"`,
+    );
+  }
+}

Common/Server/Infrastructure/Postgres/SchemaMigrations/1753378524062-AddPerformanceIndexes.ts

@@ -0,0 +1,131 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddPerformanceIndexes1753378524062 implements MigrationInterface {
+  public name = "AddPerformanceIndexes1753378524062";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "rotation" SET DEFAULT '{"_type":"Recurring","value":{"intervalType":"Day","intervalCount":{"_type":"PositiveNumber","value":1}}}'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "restrictionTimes" SET DEFAULT '{"_type":"RestrictionTimes","value":{"restictionType":"None","dayRestrictionTimes":null,"weeklyRestrictionTimes":[]}}'`,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_3c2f8998deba67cedb958fc08f" ON "IncidentSeverity" ("projectId", "order") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_2283c2d1aab23419b784db0d84" ON "IncidentState" ("projectId", "order") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_4ed23cf5e6614ee930972ab6b5" ON "IncidentState" ("projectId", "isResolvedState") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_b231eb3cdc945e53947495cf76" ON "IncidentState" ("projectId", "isCreatedState") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_5c9760b0f7df9fe68efd52151d" ON "MonitorStatus" ("projectId", "isOfflineState") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_9c64d2b5df8c5cac0ece90d899" ON "MonitorStatus" ("projectId", "isOperationalState") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_4490b10d3394a9be5f27f8fc3b" ON "IncidentOwnerTeam" ("incidentId", "teamId", "projectId") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_1d8d2229e31e4ec13ec99c79ae" ON "IncidentOwnerUser" ("incidentId", "userId", "projectId") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_7b7272644aab237d503ed3429a" ON "MonitorOwnerTeam" ("monitorId", "teamId", "projectId") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_6f6246149ab744fd62ada06ee5" ON "MonitorOwnerUser" ("monitorId", "userId", "projectId") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_c98e7e9e31d674cf5c47b15f36" ON "AlertSeverity" ("projectId", "order") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_3bb6dc217814170a3b37e21bf5" ON "AlertState" ("projectId", "order") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_b20be7b2ca1a6dc602da305f8a" ON "AlertState" ("projectId", "isAcknowledgedState") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_ae2854ea86740fdd56eaf2fea9" ON "AlertState" ("projectId", "isResolvedState") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_91ad158d170a9b51a2046fcc87" ON "AlertState" ("projectId", "isCreatedState") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_d640454e87b3dd4f24f9c527d2" ON "AlertStateTimeline" ("alertId", "startsAt") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_dfbcaebaa02d06a556fd2e155c" ON "AlertOwnerTeam" ("alertId", "teamId", "projectId") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_042a7841d65141fb940de9d881" ON "AlertOwnerUser" ("alertId", "userId", "projectId") `,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_042a7841d65141fb940de9d881"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_dfbcaebaa02d06a556fd2e155c"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_d640454e87b3dd4f24f9c527d2"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_91ad158d170a9b51a2046fcc87"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_ae2854ea86740fdd56eaf2fea9"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_b20be7b2ca1a6dc602da305f8a"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_3bb6dc217814170a3b37e21bf5"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_c98e7e9e31d674cf5c47b15f36"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_6f6246149ab744fd62ada06ee5"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_7b7272644aab237d503ed3429a"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_1d8d2229e31e4ec13ec99c79ae"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_4490b10d3394a9be5f27f8fc3b"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_9c64d2b5df8c5cac0ece90d899"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_5c9760b0f7df9fe68efd52151d"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_b231eb3cdc945e53947495cf76"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_4ed23cf5e6614ee930972ab6b5"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_2283c2d1aab23419b784db0d84"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_3c2f8998deba67cedb958fc08f"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "restrictionTimes" SET DEFAULT '{"_type": "RestrictionTimes", "value": {"restictionType": "None", "dayRestrictionTimes": null, "weeklyRestrictionTimes": []}}'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "rotation" SET DEFAULT '{"_type": "Recurring", "value": {"intervalType": "Day", "intervalCount": {"_type": "PositiveNumber", "value": 1}}}'`,
+    );
+  }
+}

Common/Server/Infrastructure/Postgres/SchemaMigrations/1753383711511-MigrationName.ts

@@ -0,0 +1,119 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class MigrationName1753383711511 implements MigrationInterface {
+  public name = "MigrationName1753383711511";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "rotation" SET DEFAULT '{"_type":"Recurring","value":{"intervalType":"Day","intervalCount":{"_type":"PositiveNumber","value":1}}}'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "restrictionTimes" SET DEFAULT '{"_type":"RestrictionTimes","value":{"restictionType":"None","dayRestrictionTimes":null,"weeklyRestrictionTimes":[]}}'`,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_b03e14b5a5fc9f5b8603283c88" ON "OnCallDutyPolicyExecutionLogTimeline" ("alertSentToUserId", "projectId") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_114e3f761691867aa919ab6b6e" ON "OnCallDutyPolicyExecutionLogTimeline" ("projectId", "createdAt") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_f34e1244e487f705e7c6b25831" ON "OnCallDutyPolicyExecutionLogTimeline" ("onCallDutyPolicyExecutionLogId", "createdAt") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_34f21c8ae164fb90be806818a8" ON "OnCallDutyPolicyOwnerTeam" ("onCallDutyPolicyId", "teamId", "projectId") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_1539db4bbd6ada58abb940b058" ON "OnCallDutyPolicyOwnerUser" ("onCallDutyPolicyId", "userId", "projectId") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_00439dd14338c3ee4e81d0714a" ON "ScheduledMaintenanceState" ("projectId", "isEndedState") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_7addde4d27f13be56651000df9" ON "ScheduledMaintenanceState" ("projectId", "isOngoingState") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_e84431ba010571147933477cff" ON "ScheduledMaintenanceState" ("projectId", "order") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_b737666365dbea2e4c914fc6d3" ON "ScheduledMaintenanceOwnerTeam" ("scheduledMaintenanceId", "teamId", "projectId") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_a4621b7155a01292b92569549f" ON "ScheduledMaintenanceOwnerUser" ("scheduledMaintenanceId", "userId", "projectId") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_c4ac940ddb05242a166567edbb" ON "ScheduledMaintenanceStateTimeline" ("scheduledMaintenanceId", "startsAt") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_4873976169085f14bdc39e168d" ON "StatusPageOwnerTeam" ("statusPageId", "teamId", "projectId") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_a9f80dc4f648f0957ce695dc61" ON "StatusPageOwnerUser" ("statusPageId", "userId", "projectId") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_33ba145fe2826bb953e2ce9d3d" ON "UserOnCallLogTimeline" ("projectId", "status") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_90363cc35c22e377df8fdc5dfb" ON "UserOnCallLogTimeline" ("onCallDutyPolicyExecutionLogId", "status") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_89cccd6782b1ee84d20e9690d0" ON "UserOnCallLogTimeline" ("userId", "createdAt") `,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_89cccd6782b1ee84d20e9690d0"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_90363cc35c22e377df8fdc5dfb"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_33ba145fe2826bb953e2ce9d3d"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_a9f80dc4f648f0957ce695dc61"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_4873976169085f14bdc39e168d"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_c4ac940ddb05242a166567edbb"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_a4621b7155a01292b92569549f"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_b737666365dbea2e4c914fc6d3"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_e84431ba010571147933477cff"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_7addde4d27f13be56651000df9"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_00439dd14338c3ee4e81d0714a"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_1539db4bbd6ada58abb940b058"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_34f21c8ae164fb90be806818a8"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_f34e1244e487f705e7c6b25831"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_114e3f761691867aa919ab6b6e"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_b03e14b5a5fc9f5b8603283c88"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "restrictionTimes" SET DEFAULT '{"_type": "RestrictionTimes", "value": {"restictionType": "None", "dayRestrictionTimes": null, "weeklyRestrictionTimes": []}}'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "rotation" SET DEFAULT '{"_type": "Recurring", "value": {"intervalType": "Day", "intervalCount": {"_type": "PositiveNumber", "value": 1}}}'`,
+    );
+  }
+}

Common/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts

@@ -138,6 +138,14 @@ import { MigrationName1749065784320 } from "./1749065784320-MigrationName";
 import { MigrationName1749133333893 } from "./1749133333893-MigrationName";
 import { MigrationName1749813704371 } from "./1749813704371-MigrationName";
 import { MigrationName1750250435756 } from "./1750250435756-MigrationName";
+import { MigrationName1752659054949 } from "./1752659054949-MigrationName";
+import { MigrationName1752774923063 } from "./1752774923063-MigrationName";
+import { MigrationName1753109689244 } from "./1753109689244-MigrationName";
+import { AddEnableCustomSubscriberEmailNotificationFooterText1753131488925 } from "./1753131488925-AddEnableCustomSubscriberEmailNotificationFooterText";
+import { MigrationName1753343522987 } from "./1753343522987-MigrationName";
+import { MigrationName1753377161288 } from "./1753377161288-MigrationName";
+import { AddPerformanceIndexes1753378524062 } from "./1753378524062-AddPerformanceIndexes";
+import { MigrationName1753383711511 } from "./1753383711511-MigrationName";
 
 export default [
   InitialMigration,
@@ -280,4 +288,12 @@ export default [
   MigrationName1749133333893,
   MigrationName1749813704371,
   MigrationName1750250435756,
+  MigrationName1752659054949,
+  MigrationName1752774923063,
+  MigrationName1753109689244,
+  AddEnableCustomSubscriberEmailNotificationFooterText1753131488925,
+  MigrationName1753343522987,
+  MigrationName1753377161288,
+  AddPerformanceIndexes1753378524062,
+  MigrationName1753383711511,
 ];

Common/Server/Infrastructure/Queue.ts

@@ -16,6 +16,11 @@ import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 export enum QueueName {
   Workflow = "Workflow",
   Worker = "Worker",
+  Telemetry = "Telemetry",
+  FluentIngest = "FluentIngest",
+  IncomingRequestIngest = "IncomingRequestIngest",
+  ServerMonitorIngest = "ServerMonitorIngest",
+  ProbeIngest = "ProbeIngest",
 }
 
 export type QueueJob = Job;
@@ -133,4 +138,81 @@ export default class Queue {
 
     return jobAdded;
   }
+
+  @CaptureSpan()
+  public static async getQueueSize(queueName: QueueName): Promise<number> {
+    const queue: BullQueue = this.getQueue(queueName);
+    const waitingCount: number = await queue.getWaitingCount();
+    const activeCount: number = await queue.getActiveCount();
+    const delayedCount: number = await queue.getDelayedCount();
+
+    return waitingCount + activeCount + delayedCount;
+  }
+
+  @CaptureSpan()
+  public static async getQueueStats(queueName: QueueName): Promise<{
+    waiting: number;
+    active: number;
+    completed: number;
+    failed: number;
+    delayed: number;
+    total: number;
+  }> {
+    const queue: BullQueue = this.getQueue(queueName);
+    const waitingCount: number = await queue.getWaitingCount();
+    const activeCount: number = await queue.getActiveCount();
+    const completedCount: number = await queue.getCompletedCount();
+    const failedCount: number = await queue.getFailedCount();
+    const delayedCount: number = await queue.getDelayedCount();
+
+    return {
+      waiting: waitingCount,
+      active: activeCount,
+      completed: completedCount,
+      failed: failedCount,
+      delayed: delayedCount,
+      total:
+        waitingCount +
+        activeCount +
+        completedCount +
+        failedCount +
+        delayedCount,
+    };
+  }
+
+  @CaptureSpan()
+  public static async getFailedJobs(
+    queueName: QueueName,
+    options?: {
+      start?: number;
+      end?: number;
+    },
+  ): Promise<
+    Array<{
+      id: string;
+      name: string;
+      data: JSONObject;
+      failedReason: string;
+      processedOn: Date | null;
+      finishedOn: Date | null;
+      attemptsMade: number;
+    }>
+  > {
+    const queue: BullQueue = this.getQueue(queueName);
+    const start: number = options?.start || 0;
+    const end: number = options?.end || 100;
+    const failed: Job[] = await queue.getFailed(start, end);
+
+    return failed.map((job: Job) => {
+      return {
+        id: job.id || "unknown",
+        name: job.name || "unknown",
+        data: job.data as JSONObject,
+        failedReason: job.failedReason || "No reason provided",
+        processedOn: job.processedOn ? new Date(job.processedOn) : null,
+        finishedOn: job.finishedOn ? new Date(job.finishedOn) : null,
+        attemptsMade: job.attemptsMade || 0,
+      };
+    });
+  }
 }

Common/Server/Middleware/ClusterKeyAuthorization.ts

@@ -38,6 +38,9 @@ export default class ClusterKeyAuthorization {
     } else if (req.headers && req.headers["clusterkey"]) {
       // Header keys are automatically transformed to lowercase
       clusterKey = req.headers["clusterkey"] as string;
+    } else if (req.headers && req.headers["x-clusterkey"]) {
+      // KEDA TriggerAuthentication sends headers with X- prefix
+      clusterKey = req.headers["x-clusterkey"] as string;
     } else if (req.body && req.body.clusterKey) {
       clusterKey = req.body.clusterKey;
     } else {

Common/Server/Middleware/ProjectAuthorization.ts

@@ -96,38 +96,40 @@ export default class ProjectMiddleware {
           props: { isRoot: true },
         });
 
-        tenantId = apiKeyModel?.projectId || null;
-
-        if (!tenantId) {
-          throw new BadDataException("Invalid API Key");
-        }
-
-        (req as OneUptimeRequest).tenantId = tenantId;
-
         if (apiKeyModel) {
-          (req as OneUptimeRequest).userType = UserType.API;
-          // TODO: Add API key permissions.
-          // (req as OneUptimeRequest).permissions =
-          //     apiKeyModel.permissions || [];
-          (req as OneUptimeRequest).userGlobalAccessPermission =
-            await APIKeyAccessPermission.getDefaultApiGlobalPermission(
-              tenantId,
-            );
+          tenantId = apiKeyModel?.projectId || null;
 
-          const userTenantAccessPermission: UserTenantAccessPermission | null =
-            await APIKeyAccessPermission.getApiTenantAccessPermission(
-              tenantId,
-              apiKeyModel.id!,
-            );
+          if (!tenantId) {
+            throw new BadDataException("Invalid API Key");
+          }
 
-          if (userTenantAccessPermission) {
-            (req as OneUptimeRequest).userTenantAccessPermission = {};
-            (
-              (req as OneUptimeRequest)
-                .userTenantAccessPermission as Dictionary<UserTenantAccessPermission>
-            )[tenantId.toString()] = userTenantAccessPermission;
+          (req as OneUptimeRequest).tenantId = tenantId;
 
-            return next();
+          if (apiKeyModel) {
+            (req as OneUptimeRequest).userType = UserType.API;
+            // TODO: Add API key permissions.
+            // (req as OneUptimeRequest).permissions =
+            //     apiKeyModel.permissions || [];
+            (req as OneUptimeRequest).userGlobalAccessPermission =
+              await APIKeyAccessPermission.getDefaultApiGlobalPermission(
+                tenantId,
+              );
+
+            const userTenantAccessPermission: UserTenantAccessPermission | null =
+              await APIKeyAccessPermission.getApiTenantAccessPermission(
+                tenantId,
+                apiKeyModel.id!,
+              );
+
+            if (userTenantAccessPermission) {
+              (req as OneUptimeRequest).userTenantAccessPermission = {};
+              (
+                (req as OneUptimeRequest)
+                  .userTenantAccessPermission as Dictionary<UserTenantAccessPermission>
+              )[tenantId.toString()] = userTenantAccessPermission;
+
+              return next();
+            }
           }
         }
       }

Common/Server/Services/AlertService.ts

@@ -54,6 +54,7 @@ import { MessageBlocksByWorkspaceType } from "./WorkspaceNotificationRuleService
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import MetricType from "../../Models/DatabaseModels/MetricType";
 import Dictionary from "../../Types/Dictionary";
+import OnCallDutyPolicy from "../../Models/DatabaseModels/OnCallDutyPolicy";
 
 export class Service extends DatabaseService<Model> {
   public constructor() {
@@ -272,6 +273,7 @@ export class Service extends DatabaseService<Model> {
       throw new BadDataException("currentAlertStateId is required");
     }
 
+    // Get alert data for feed creation
     const alert: Model | null = await this.findOneById({
       id: createdItem.id,
       select: {
@@ -304,147 +306,258 @@ export class Service extends DatabaseService<Model> {
       throw new BadDataException("Alert not found");
     }
 
-    const createdByUserId: ObjectID | undefined | null =
-      createdItem.createdByUserId || createdItem.createdByUser?.id;
+    // Execute core operations in parallel first
+    const coreOperations: Array<Promise<any>> = [];
 
-    // send message to workspaces - slack, teams,   etc.
-    const workspaceResult: {
-      channelsCreated: Array<NotificationRuleWorkspaceChannel>;
-    } | null =
-      await AlertWorkspaceMessages.createChannelsAndInviteUsersToChannels({
-        projectId: createdItem.projectId,
-        alertId: createdItem.id!,
-        alertNumber: createdItem.alertNumber!,
-      });
+    // Create feed item asynchronously
+    coreOperations.push(this.createAlertFeedAsync(alert, createdItem));
 
-    logger.debug("Alert created. Workspace result:");
-    logger.debug(workspaceResult);
-
-    if (workspaceResult && workspaceResult.channelsCreated?.length > 0) {
-      // update alert with these channels.
-      await this.updateOneById({
-        id: createdItem.id!,
-        data: {
-          postUpdatesToWorkspaceChannels: workspaceResult.channelsCreated || [],
-        },
-        props: {
-          isRoot: true,
-        },
-      });
-    }
-
-    let feedInfoInMarkdown: string = `#### 🚨 Alert ${createdItem.alertNumber?.toString()} Created: 
-         
-**${createdItem.title || "No title provided."}**:
-   
-${createdItem.description || "No description provided."}
-   
-   `;
-
-    if (alert.currentAlertState?.name) {
-      feedInfoInMarkdown += `🔴 **Alert State**: ${alert.currentAlertState.name} \n\n`;
-    }
-
-    if (alert.alertSeverity?.name) {
-      feedInfoInMarkdown += `⚠️ **Severity**: ${alert.alertSeverity.name} \n\n`;
-    }
-
-    if (alert.monitor) {
-      feedInfoInMarkdown += `🌎 **Resources Affected**:\n`;
-
-      const monitor: Monitor = alert.monitor;
-      feedInfoInMarkdown += `- [${monitor.name}](${(await MonitorService.getMonitorLinkInDashboard(createdItem.projectId!, monitor.id!)).toString()})\n`;
-
-      feedInfoInMarkdown += `\n\n`;
-    }
-
-    if (createdItem.rootCause) {
-      feedInfoInMarkdown += `\n
-📄 **Root Cause**:
-   
-${createdItem.rootCause || "No root cause provided."}
-   
-`;
-    }
-
-    if (createdItem.remediationNotes) {
-      feedInfoInMarkdown += `\n 
-🎯 **Remediation Notes**:
-   
-${createdItem.remediationNotes || "No remediation notes provided."}
-   
-   
-   `;
-    }
-
-    const alertCreateMessageBlocks: Array<MessageBlocksByWorkspaceType> =
-      await AlertWorkspaceMessages.getAlertCreateMessageBlocks({
-        alertId: createdItem.id!,
-        projectId: createdItem.projectId!,
-      });
-
-    await AlertFeedService.createAlertFeedItem({
-      alertId: createdItem.id!,
-      projectId: createdItem.projectId!,
-      alertFeedEventType: AlertFeedEventType.AlertCreated,
-      displayColor: Red500,
-      feedInfoInMarkdown: feedInfoInMarkdown,
-      userId: createdByUserId || undefined,
-      workspaceNotification: {
-        appendMessageBlocks: alertCreateMessageBlocks,
-        sendWorkspaceNotification: true,
-      },
-    });
-
-    await this.changeAlertState({
-      projectId: createdItem.projectId,
-      alertId: createdItem.id,
-      alertStateId: createdItem.currentAlertStateId,
-      notifyOwners: false,
-      rootCause: createdItem.rootCause,
-      stateChangeLog: createdItem.createdStateLog,
-      props: {
-        isRoot: true,
-      },
-    });
-
-    // add owners.
+    // Handle state change asynchronously
+    coreOperations.push(this.handleAlertStateChangeAsync(createdItem));
 
+    // Handle owner assignment asynchronously
     if (
       onCreate.createBy.miscDataProps &&
       (onCreate.createBy.miscDataProps["ownerTeams"] ||
         onCreate.createBy.miscDataProps["ownerUsers"])
     ) {
-      await this.addOwners(
-        createdItem.projectId,
-        createdItem.id,
-        (onCreate.createBy.miscDataProps["ownerUsers"] as Array<ObjectID>) ||
-          [],
-        (onCreate.createBy.miscDataProps["ownerTeams"] as Array<ObjectID>) ||
-          [],
-        false,
-        onCreate.createBy.props,
+      coreOperations.push(
+        this.addOwners(
+          createdItem.projectId,
+          createdItem.id,
+          (onCreate.createBy.miscDataProps["ownerUsers"] as Array<ObjectID>) ||
+            [],
+          (onCreate.createBy.miscDataProps["ownerTeams"] as Array<ObjectID>) ||
+            [],
+          false,
+          onCreate.createBy.props,
+        ),
       );
     }
 
-    if (
-      createdItem.onCallDutyPolicies?.length &&
-      createdItem.onCallDutyPolicies?.length > 0
-    ) {
-      for (const policy of createdItem.onCallDutyPolicies) {
-        await OnCallDutyPolicyService.executePolicy(
-          new ObjectID(policy._id as string),
-          {
-            triggeredByAlertId: createdItem.id!,
-            userNotificationEventType: UserNotificationEventType.AlertCreated,
-          },
+    // Execute core operations in parallel with error handling
+    Promise.allSettled(coreOperations)
+      .then((coreResults: any[]) => {
+        // Log any errors from core operations
+        coreResults.forEach((result: any, index: number) => {
+          if (result.status === "rejected") {
+            logger.error(
+              `Core operation ${index} failed in AlertService.onCreateSuccess: ${result.reason}`,
+            );
+          }
+        });
+
+        // Handle on-call duty policies asynchronously
+        if (
+          createdItem.onCallDutyPolicies?.length &&
+          createdItem.onCallDutyPolicies?.length > 0
+        ) {
+          this.executeAlertOnCallDutyPoliciesAsync(createdItem).catch(
+            (error: Error) => {
+              logger.error(
+                `On-call duty policy execution failed in AlertService.onCreateSuccess: ${error}`,
+              );
+            },
+          );
+        }
+
+        // Handle workspace operations after core operations complete
+        if (createdItem.projectId && createdItem.id) {
+          // Run workspace operations in background without blocking response
+          this.handleAlertWorkspaceOperationsAsync(createdItem).catch(
+            (error: Error) => {
+              logger.error(
+                `Workspace operations failed in AlertService.onCreateSuccess: ${error}`,
+              );
+            },
+          );
+        }
+      })
+      .catch((error: Error) => {
+        logger.error(
+          `Critical error in AlertService core operations: ${error}`,
         );
-      }
-    }
+      });
 
     return createdItem;
   }
 
+  @CaptureSpan()
+  private async handleAlertWorkspaceOperationsAsync(
+    createdItem: Model,
+  ): Promise<void> {
+    try {
+      if (!createdItem.projectId || !createdItem.id) {
+        throw new BadDataException(
+          "projectId and id are required for workspace operations",
+        );
+      }
+
+      // send message to workspaces - slack, teams, etc.
+      const workspaceResult: {
+        channelsCreated: Array<NotificationRuleWorkspaceChannel>;
+      } | null =
+        await AlertWorkspaceMessages.createChannelsAndInviteUsersToChannels({
+          projectId: createdItem.projectId,
+          alertId: createdItem.id,
+          alertNumber: createdItem.alertNumber!,
+        });
+
+      logger.debug("Alert created. Workspace result:");
+      logger.debug(workspaceResult);
+
+      if (workspaceResult && workspaceResult.channelsCreated?.length > 0) {
+        // update alert with these channels.
+        await this.updateOneById({
+          id: createdItem.id,
+          data: {
+            postUpdatesToWorkspaceChannels:
+              workspaceResult.channelsCreated || [],
+          },
+          props: {
+            isRoot: true,
+          },
+        });
+      }
+    } catch (error) {
+      logger.error(`Error in handleAlertWorkspaceOperationsAsync: ${error}`);
+      throw error;
+    }
+  }
+
+  @CaptureSpan()
+  private async createAlertFeedAsync(
+    alert: Model,
+    createdItem: Model,
+  ): Promise<void> {
+    try {
+      const createdByUserId: ObjectID | undefined | null =
+        createdItem.createdByUserId || createdItem.createdByUser?.id;
+
+      let feedInfoInMarkdown: string = `#### 🚨 Alert ${createdItem.alertNumber?.toString()} Created: 
+           
+**${createdItem.title || "No title provided."}**:
+     
+${createdItem.description || "No description provided."}
+     
+     `;
+
+      if (alert.currentAlertState?.name) {
+        feedInfoInMarkdown += `🔴 **Alert State**: ${alert.currentAlertState.name} \n\n`;
+      }
+
+      if (alert.alertSeverity?.name) {
+        feedInfoInMarkdown += `⚠️ **Severity**: ${alert.alertSeverity.name} \n\n`;
+      }
+
+      if (alert.monitor) {
+        feedInfoInMarkdown += `🌎 **Resources Affected**:\n`;
+
+        const monitor: Monitor = alert.monitor;
+        feedInfoInMarkdown += `- [${monitor.name}](${(await MonitorService.getMonitorLinkInDashboard(createdItem.projectId!, monitor.id!)).toString()})\n`;
+
+        feedInfoInMarkdown += `\n\n`;
+      }
+
+      if (createdItem.rootCause) {
+        feedInfoInMarkdown += `\n
+📄 **Root Cause**:
+     
+${createdItem.rootCause || "No root cause provided."}
+     
+`;
+      }
+
+      if (createdItem.remediationNotes) {
+        feedInfoInMarkdown += `\n 
+🎯 **Remediation Notes**:
+     
+${createdItem.remediationNotes || "No remediation notes provided."}
+     
+     
+     `;
+      }
+
+      const alertCreateMessageBlocks: Array<MessageBlocksByWorkspaceType> =
+        await AlertWorkspaceMessages.getAlertCreateMessageBlocks({
+          alertId: createdItem.id!,
+          projectId: createdItem.projectId!,
+        });
+
+      await AlertFeedService.createAlertFeedItem({
+        alertId: createdItem.id!,
+        projectId: createdItem.projectId!,
+        alertFeedEventType: AlertFeedEventType.AlertCreated,
+        displayColor: Red500,
+        feedInfoInMarkdown: feedInfoInMarkdown,
+        userId: createdByUserId || undefined,
+        workspaceNotification: {
+          appendMessageBlocks: alertCreateMessageBlocks,
+          sendWorkspaceNotification: true,
+        },
+      });
+    } catch (error) {
+      logger.error(`Error in createAlertFeedAsync: ${error}`);
+      throw error;
+    }
+  }
+
+  @CaptureSpan()
+  private async handleAlertStateChangeAsync(createdItem: Model): Promise<void> {
+    try {
+      if (!createdItem.projectId || !createdItem.id) {
+        throw new BadDataException(
+          "projectId and id are required for state change",
+        );
+      }
+
+      await this.changeAlertState({
+        projectId: createdItem.projectId,
+        alertId: createdItem.id,
+        alertStateId: createdItem.currentAlertStateId!,
+        notifyOwners: false,
+        rootCause: createdItem.rootCause,
+        stateChangeLog: createdItem.createdStateLog,
+        props: {
+          isRoot: true,
+        },
+      });
+    } catch (error) {
+      logger.error(`Error in handleAlertStateChangeAsync: ${error}`);
+      throw error;
+    }
+  }
+
+  @CaptureSpan()
+  private async executeAlertOnCallDutyPoliciesAsync(
+    createdItem: Model,
+  ): Promise<void> {
+    try {
+      if (
+        createdItem.onCallDutyPolicies?.length &&
+        createdItem.onCallDutyPolicies?.length > 0
+      ) {
+        // Execute all on-call policies in parallel
+        const policyPromises: Promise<void>[] =
+          createdItem.onCallDutyPolicies.map((policy: OnCallDutyPolicy) => {
+            return OnCallDutyPolicyService.executePolicy(
+              new ObjectID(policy["_id"] as string),
+              {
+                triggeredByAlertId: createdItem.id!,
+                userNotificationEventType:
+                  UserNotificationEventType.AlertCreated,
+              },
+            );
+          });
+
+        await Promise.allSettled(policyPromises);
+      }
+    } catch (error) {
+      logger.error(`Error in executeAlertOnCallDutyPoliciesAsync: ${error}`);
+      throw error;
+    }
+  }
+
   @CaptureSpan()
   public async getWorkspaceChannelForAlert(data: {
     alertId: ObjectID;

Common/Server/Services/IncidentService.ts

@@ -59,9 +59,10 @@ import WorkspaceType from "../../Types/Workspace/WorkspaceType";
 import { MessageBlocksByWorkspaceType } from "./WorkspaceNotificationRuleService";
 import NotificationRuleWorkspaceChannel from "../../Types/Workspace/NotificationRules/NotificationRuleWorkspaceChannel";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
-import { Dictionary } from "lodash";
 import MetricType from "../../Models/DatabaseModels/MetricType";
 import UpdateBy from "../Types/Database/UpdateBy";
+import OnCallDutyPolicy from "../../Models/DatabaseModels/OnCallDutyPolicy";
+import Dictionary from "../../Types/Dictionary";
 
 // key is incidentId for this dictionary.
 type UpdateCarryForward = Dictionary<{
@@ -544,6 +545,7 @@ export class Service extends DatabaseService<Model> {
       throw new BadDataException("id is required");
     }
 
+    // Get incident data for feed creation
     const incident: Model | null = await this.findOneById({
       id: createdItem.id,
       select: {
@@ -576,202 +578,343 @@ export class Service extends DatabaseService<Model> {
       throw new BadDataException("Incident not found");
     }
 
-    // release the mutex.
-    if (onCreate.carryForward && onCreate.carryForward.mutex) {
-      const mutex: SemaphoreMutex = onCreate.carryForward.mutex;
-      const projectId: ObjectID = createdItem.projectId!;
+    // Execute core operations in parallel first
+    const coreOperations: Array<Promise<any>> = [];
 
-      try {
-        await Semaphore.release(mutex);
-        logger.debug(
-          "Mutex released - IncidentService.incident-create " +
-            projectId.toString() +
-            " at " +
-            OneUptimeDate.getCurrentDateAsFormattedString(),
+    // Create feed item asynchronously
+    coreOperations.push(this.createIncidentFeedAsync(incident, createdItem));
+
+    // Handle state change asynchronously
+    coreOperations.push(this.handleIncidentStateChangeAsync(createdItem));
+
+    // Handle owner assignment asynchronously
+    if (
+      onCreate.createBy.miscDataProps &&
+      (onCreate.createBy.miscDataProps["ownerTeams"] ||
+        onCreate.createBy.miscDataProps["ownerUsers"])
+    ) {
+      coreOperations.push(
+        this.addOwners(
+          createdItem.projectId,
+          createdItem.id,
+          (onCreate.createBy.miscDataProps["ownerUsers"] as Array<ObjectID>) ||
+            [],
+          (onCreate.createBy.miscDataProps["ownerTeams"] as Array<ObjectID>) ||
+            [],
+          false,
+          onCreate.createBy.props,
+        ),
+      );
+    }
+
+    // Handle monitor status change and active monitoring asynchronously
+    if (createdItem.changeMonitorStatusToId && createdItem.projectId) {
+      coreOperations.push(
+        this.handleMonitorStatusChangeAsync(createdItem, onCreate),
+      );
+    }
+
+    coreOperations.push(
+      this.disableActiveMonitoringIfManualIncident(createdItem.id!),
+    );
+
+    // Release mutex immediately
+    this.releaseMutexAsync(onCreate, createdItem.projectId!);
+
+    // Execute core operations in parallel with error handling
+    Promise.allSettled(coreOperations)
+      .then((coreResults: any[]) => {
+        // Log any errors from core operations
+        coreResults.forEach((result: any, index: number) => {
+          if (result.status === "rejected") {
+            logger.error(
+              `Core operation ${index} failed in IncidentService.onCreateSuccess: ${result.reason}`,
+            );
+          }
+        });
+
+        // Handle on-call duty policies asynchronously
+        if (
+          createdItem.onCallDutyPolicies?.length &&
+          createdItem.onCallDutyPolicies?.length > 0
+        ) {
+          this.executeOnCallDutyPoliciesAsync(createdItem).catch(
+            (error: Error) => {
+              logger.error(
+                `On-call duty policy execution failed in IncidentService.onCreateSuccess: ${error}`,
+              );
+            },
+          );
+        }
+
+        // Handle workspace operations after core operations complete
+        if (createdItem.projectId && createdItem.id) {
+          // Run workspace operations in background without blocking response
+          this.handleIncidentWorkspaceOperationsAsync(createdItem).catch(
+            (error: Error) => {
+              logger.error(
+                `Workspace operations failed in IncidentService.onCreateSuccess: ${error}`,
+              );
+            },
+          );
+        }
+      })
+      .catch((error: Error) => {
+        logger.error(
+          `Critical error in IncidentService core operations: ${error}`,
         );
-      } catch (err) {
-        logger.debug(
-          "Mutex release failed -  IncidentService.incident-create " +
-            projectId.toString() +
-            " at " +
-            OneUptimeDate.getCurrentDateAsFormattedString(),
+      });
+
+    return createdItem;
+  }
+
+  @CaptureSpan()
+  private async handleIncidentWorkspaceOperationsAsync(
+    createdItem: Model,
+  ): Promise<void> {
+    try {
+      if (!createdItem.projectId || !createdItem.id) {
+        throw new BadDataException(
+          "projectId and id are required for workspace operations",
         );
-        logger.error(err);
       }
+
+      // send message to workspaces - slack, teams, etc.
+      const workspaceResult: {
+        channelsCreated: Array<NotificationRuleWorkspaceChannel>;
+      } | null =
+        await IncidentWorkspaceMessages.createChannelsAndInviteUsersToChannels({
+          projectId: createdItem.projectId,
+          incidentId: createdItem.id,
+          incidentNumber: createdItem.incidentNumber!,
+        });
+
+      if (workspaceResult && workspaceResult.channelsCreated?.length > 0) {
+        // update incident with these channels.
+        await this.updateOneById({
+          id: createdItem.id,
+          data: {
+            postUpdatesToWorkspaceChannels:
+              workspaceResult.channelsCreated || [],
+          },
+          props: {
+            isRoot: true,
+          },
+        });
+      }
+    } catch (error) {
+      logger.error(`Error in handleIncidentWorkspaceOperationsAsync: ${error}`);
+      throw error;
     }
+  }
 
-    const createdByUserId: ObjectID | undefined | null =
-      createdItem.createdByUserId || createdItem.createdByUser?.id;
+  @CaptureSpan()
+  private async createIncidentFeedAsync(
+    incident: Model,
+    createdItem: Model,
+  ): Promise<void> {
+    try {
+      const createdByUserId: ObjectID | undefined | null =
+        createdItem.createdByUserId || createdItem.createdByUser?.id;
 
-    // send message to workspaces - slack, teams,   etc.
-    const workspaceResult: {
-      channelsCreated: Array<NotificationRuleWorkspaceChannel>;
-    } | null =
-      await IncidentWorkspaceMessages.createChannelsAndInviteUsersToChannels({
-        projectId: createdItem.projectId,
-        incidentId: createdItem.id!,
-        incidentNumber: createdItem.incidentNumber!,
-      });
-
-    if (workspaceResult && workspaceResult.channelsCreated?.length > 0) {
-      // update incident with these channels.
-      await this.updateOneById({
-        id: createdItem.id!,
-        data: {
-          postUpdatesToWorkspaceChannels: workspaceResult.channelsCreated || [],
-        },
-        props: {
-          isRoot: true,
-        },
-      });
-    }
-
-    let feedInfoInMarkdown: string = `#### 🚨 Incident ${createdItem.incidentNumber?.toString()} Created: 
-      
+      let feedInfoInMarkdown: string = `#### 🚨 Incident ${createdItem.incidentNumber?.toString()} Created: 
+        
 **${createdItem.title || "No title provided."}**:
 
 ${createdItem.description || "No description provided."}
 
 `;
 
-    if (incident.currentIncidentState?.name) {
-      feedInfoInMarkdown += `🔴 **Incident State**: ${incident.currentIncidentState.name} \n\n`;
-    }
-
-    if (incident.incidentSeverity?.name) {
-      feedInfoInMarkdown += `⚠️ **Severity**: ${incident.incidentSeverity.name} \n\n`;
-    }
-
-    if (incident.monitors && incident.monitors.length > 0) {
-      feedInfoInMarkdown += `🌎 **Resources Affected**:\n`;
-
-      for (const monitor of incident.monitors) {
-        feedInfoInMarkdown += `- [${monitor.name}](${(await MonitorService.getMonitorLinkInDashboard(createdItem.projectId!, monitor.id!)).toString()})\n`;
+      if (incident.currentIncidentState?.name) {
+        feedInfoInMarkdown += `🔴 **Incident State**: ${incident.currentIncidentState.name} \n\n`;
       }
 
-      feedInfoInMarkdown += `\n\n`;
-    }
+      if (incident.incidentSeverity?.name) {
+        feedInfoInMarkdown += `⚠️ **Severity**: ${incident.incidentSeverity.name} \n\n`;
+      }
 
-    if (createdItem.rootCause) {
-      feedInfoInMarkdown += `\n
+      if (incident.monitors && incident.monitors.length > 0) {
+        feedInfoInMarkdown += `🌎 **Resources Affected**:\n`;
+
+        for (const monitor of incident.monitors) {
+          feedInfoInMarkdown += `- [${monitor.name}](${(await MonitorService.getMonitorLinkInDashboard(createdItem.projectId!, monitor.id!)).toString()})\n`;
+        }
+
+        feedInfoInMarkdown += `\n\n`;
+      }
+
+      if (createdItem.rootCause) {
+        feedInfoInMarkdown += `\n
 📄 **Root Cause**:
 
 ${createdItem.rootCause || "No root cause provided."}
 
 `;
-    }
+      }
 
-    if (createdItem.remediationNotes) {
-      feedInfoInMarkdown += `\n 
+      if (createdItem.remediationNotes) {
+        feedInfoInMarkdown += `\n 
 🎯 **Remediation Notes**:
 
 ${createdItem.remediationNotes || "No remediation notes provided."}
 
 
 `;
-    }
+      }
 
-    const incidentCreateMessageBlocks: Array<MessageBlocksByWorkspaceType> =
-      await IncidentWorkspaceMessages.getIncidentCreateMessageBlocks({
+      const incidentCreateMessageBlocks: Array<MessageBlocksByWorkspaceType> =
+        await IncidentWorkspaceMessages.getIncidentCreateMessageBlocks({
+          incidentId: createdItem.id!,
+          projectId: createdItem.projectId!,
+        });
+
+      await IncidentFeedService.createIncidentFeedItem({
         incidentId: createdItem.id!,
         projectId: createdItem.projectId!,
+        incidentFeedEventType: IncidentFeedEventType.IncidentCreated,
+        displayColor: Red500,
+        feedInfoInMarkdown: feedInfoInMarkdown,
+        userId: createdByUserId || undefined,
+        workspaceNotification: {
+          appendMessageBlocks: incidentCreateMessageBlocks,
+          sendWorkspaceNotification: true,
+        },
       });
-
-    await IncidentFeedService.createIncidentFeedItem({
-      incidentId: createdItem.id!,
-      projectId: createdItem.projectId!,
-      incidentFeedEventType: IncidentFeedEventType.IncidentCreated,
-      displayColor: Red500,
-      feedInfoInMarkdown: feedInfoInMarkdown,
-      userId: createdByUserId || undefined,
-      workspaceNotification: {
-        appendMessageBlocks: incidentCreateMessageBlocks,
-        sendWorkspaceNotification: true,
-      },
-    });
-
-    if (!createdItem.currentIncidentStateId) {
-      throw new BadDataException("currentIncidentStateId is required");
+    } catch (error) {
+      logger.error(`Error in createIncidentFeedAsync: ${error}`);
+      throw error;
     }
-
-    if (createdItem.changeMonitorStatusToId && createdItem.projectId) {
-      // change status of all the monitors.
-      await MonitorService.changeMonitorStatus(
-        createdItem.projectId,
-        createdItem.monitors?.map((monitor: Monitor) => {
-          return new ObjectID(monitor._id || "");
-        }) || [],
-        createdItem.changeMonitorStatusToId,
-        true, // notifyMonitorOwners
-        createdItem.rootCause ||
-          "Status was changed because Incident #" +
-            createdItem.incidentNumber?.toString() +
-            " was created.",
-        createdItem.createdStateLog,
-        onCreate.createBy.props,
-      );
-    }
-
-    await this.changeIncidentState({
-      projectId: createdItem.projectId,
-      incidentId: createdItem.id,
-      incidentStateId: createdItem.currentIncidentStateId,
-      shouldNotifyStatusPageSubscribers: Boolean(
-        createdItem.shouldStatusPageSubscribersBeNotifiedOnIncidentCreated,
-      ),
-      isSubscribersNotified: Boolean(
-        createdItem.shouldStatusPageSubscribersBeNotifiedOnIncidentCreated,
-      ), // we dont want to notify subscribers when incident state changes because they are already notified when the incident is created.
-      notifyOwners: false,
-      rootCause: createdItem.rootCause,
-      stateChangeLog: createdItem.createdStateLog,
-      props: {
-        isRoot: true,
-      },
-    });
-
-    // add owners.
-
-    if (
-      onCreate.createBy.miscDataProps &&
-      (onCreate.createBy.miscDataProps["ownerTeams"] ||
-        onCreate.createBy.miscDataProps["ownerUsers"])
-    ) {
-      await this.addOwners(
-        createdItem.projectId,
-        createdItem.id,
-        (onCreate.createBy.miscDataProps["ownerUsers"] as Array<ObjectID>) ||
-          [],
-        (onCreate.createBy.miscDataProps["ownerTeams"] as Array<ObjectID>) ||
-          [],
-        false,
-        onCreate.createBy.props,
-      );
-    }
-
-    if (
-      createdItem.onCallDutyPolicies?.length &&
-      createdItem.onCallDutyPolicies?.length > 0
-    ) {
-      for (const policy of createdItem.onCallDutyPolicies) {
-        await OnCallDutyPolicyService.executePolicy(
-          new ObjectID(policy._id as string),
-          {
-            triggeredByIncidentId: createdItem.id!,
-            userNotificationEventType:
-              UserNotificationEventType.IncidentCreated,
-          },
-        );
-      }
-    }
-
-    // check if the incident is created manaull by a user and if thats the case, then disable active monitoting on that monitor.
-
-    await this.disableActiveMonitoringIfManualIncident(createdItem.id!);
-
-    return createdItem;
   }
 
+  @CaptureSpan()
+  private async handleIncidentStateChangeAsync(
+    createdItem: Model,
+  ): Promise<void> {
+    try {
+      if (!createdItem.currentIncidentStateId) {
+        throw new BadDataException("currentIncidentStateId is required");
+      }
+
+      if (!createdItem.projectId || !createdItem.id) {
+        throw new BadDataException(
+          "projectId and id are required for state change",
+        );
+      }
+
+      await this.changeIncidentState({
+        projectId: createdItem.projectId,
+        incidentId: createdItem.id,
+        incidentStateId: createdItem.currentIncidentStateId,
+        shouldNotifyStatusPageSubscribers: Boolean(
+          createdItem.shouldStatusPageSubscribersBeNotifiedOnIncidentCreated,
+        ),
+        isSubscribersNotified: Boolean(
+          createdItem.shouldStatusPageSubscribersBeNotifiedOnIncidentCreated,
+        ), // we dont want to notify subscribers when incident state changes because they are already notified when the incident is created.
+        notifyOwners: false,
+        rootCause: createdItem.rootCause,
+        stateChangeLog: createdItem.createdStateLog,
+        props: {
+          isRoot: true,
+        },
+      });
+    } catch (error) {
+      logger.error(`Error in handleIncidentStateChangeAsync: ${error}`);
+      throw error;
+    }
+  }
+
+  @CaptureSpan()
+  private async executeOnCallDutyPoliciesAsync(
+    createdItem: Model,
+  ): Promise<void> {
+    try {
+      if (
+        createdItem.onCallDutyPolicies?.length &&
+        createdItem.onCallDutyPolicies?.length > 0
+      ) {
+        // Execute all on-call policies in parallel
+        const policyPromises: Promise<void>[] =
+          createdItem.onCallDutyPolicies.map((policy: OnCallDutyPolicy) => {
+            return OnCallDutyPolicyService.executePolicy(
+              new ObjectID(policy["_id"] as string),
+              {
+                triggeredByIncidentId: createdItem.id!,
+                userNotificationEventType:
+                  UserNotificationEventType.IncidentCreated,
+              },
+            );
+          });
+
+        await Promise.allSettled(policyPromises);
+      }
+    } catch (error) {
+      logger.error(`Error in executeOnCallDutyPoliciesAsync: ${error}`);
+      throw error;
+    }
+  }
+
+  @CaptureSpan()
+  private async handleMonitorStatusChangeAsync(
+    createdItem: Model,
+    onCreate: OnCreate<Model>,
+  ): Promise<void> {
+    try {
+      if (createdItem.changeMonitorStatusToId && createdItem.projectId) {
+        // change status of all the monitors.
+        await MonitorService.changeMonitorStatus(
+          createdItem.projectId,
+          createdItem.monitors?.map((monitor: Monitor) => {
+            return new ObjectID(monitor._id || "");
+          }) || [],
+          createdItem.changeMonitorStatusToId,
+          true, // notifyMonitorOwners
+          createdItem.rootCause ||
+            "Status was changed because Incident #" +
+              createdItem.incidentNumber?.toString() +
+              " was created.",
+          createdItem.createdStateLog,
+          onCreate.createBy.props,
+        );
+      }
+    } catch (error) {
+      logger.error(`Error in handleMonitorStatusChangeAsync: ${error}`);
+      throw error;
+    }
+  }
+
+  @CaptureSpan()
+  private releaseMutexAsync(
+    onCreate: OnCreate<Model>,
+    projectId: ObjectID,
+  ): void {
+    // Release mutex in background without blocking
+    if (onCreate.carryForward && onCreate.carryForward.mutex) {
+      const mutex: SemaphoreMutex = onCreate.carryForward.mutex;
+
+      setImmediate(async () => {
+        try {
+          await Semaphore.release(mutex);
+          logger.debug(
+            "Mutex released - IncidentService.incident-create " +
+              projectId.toString() +
+              " at " +
+              OneUptimeDate.getCurrentDateAsFormattedString(),
+          );
+        } catch (err) {
+          logger.debug(
+            "Mutex release failed - IncidentService.incident-create " +
+              projectId.toString() +
+              " at " +
+              OneUptimeDate.getCurrentDateAsFormattedString(),
+          );
+          logger.error(err);
+        }
+      });
+    }
+  }
+
+  @CaptureSpan()
   public async disableActiveMonitoringIfManualIncident(
     incidentId: ObjectID,
   ): Promise<void> {

Common/Server/Services/IncidentStateTimelineService.ts

@@ -116,8 +116,9 @@ export class Service extends DatabaseService<IncidentStateTimeline> {
         throw new BadDataException("incidentStateId is null");
       }
 
-      const stateBeforeThis: IncidentStateTimeline | null =
-        await this.findOneBy({
+      // Execute queries for before and after states in parallel for better performance
+      const [stateBeforeThis, stateAfterThis] = await Promise.all([
+        this.findOneBy({
           query: {
             incidentId: createBy.data.incidentId,
             startsAt: QueryHelper.lessThanEqualTo(createBy.data.startsAt),
@@ -138,7 +139,25 @@ export class Service extends DatabaseService<IncidentStateTimeline> {
             startsAt: true,
             endsAt: true,
           },
-        });
+        }),
+        this.findOneBy({
+          query: {
+            incidentId: createBy.data.incidentId,
+            startsAt: QueryHelper.greaterThan(createBy.data.startsAt),
+          },
+          sort: {
+            startsAt: SortOrder.Ascending,
+          },
+          props: {
+            isRoot: true,
+          },
+          select: {
+            incidentStateId: true,
+            startsAt: true,
+            endsAt: true,
+          },
+        }),
+      ]);
 
       logger.debug("State Before this");
       logger.debug(stateBeforeThis);
@@ -197,26 +216,6 @@ export class Service extends DatabaseService<IncidentStateTimeline> {
         }
       }
 
-      const stateAfterThis: IncidentStateTimeline | null = await this.findOneBy(
-        {
-          query: {
-            incidentId: createBy.data.incidentId,
-            startsAt: QueryHelper.greaterThan(createBy.data.startsAt),
-          },
-          sort: {
-            startsAt: SortOrder.Ascending,
-          },
-          props: {
-            isRoot: true,
-          },
-          select: {
-            incidentStateId: true,
-            startsAt: true,
-            endsAt: true,
-          },
-        },
-      );
-
       // compute ends at. It's the start of the next status.
       if (stateAfterThis && stateAfterThis.startsAt) {
         createBy.data.endsAt = stateAfterThis.startsAt;

Common/Server/Services/MonitorService.ts

@@ -66,6 +66,7 @@ import LabelService from "./LabelService";
 import QueryOperator from "../../Types/BaseDatabase/QueryOperator";
 import { FindWhere } from "../../Types/BaseDatabase/Query";
 import logger from "../Utils/Logger";
+import PushNotificationUtil from "../Utils/PushNotificationUtil";
 
 export class Service extends DatabaseService<Model> {
   public constructor() {
@@ -501,20 +502,132 @@ ${createdItem.description?.trim() || "No description provided."}
       feedInfoInMarkdown += `\n\n`;
     }
 
-    // send message to workspaces - slack, teams,   etc.
+    // Parallelize operations that don't depend on each other
+    const parallelOperations: Array<Promise<any>> = [];
+
+    // 1. Essential monitor status operation (must complete first)
+    await this.changeMonitorStatus(
+      createdItem.projectId,
+      [createdItem.id],
+      createdItem.currentMonitorStatusId,
+      false, // notifyOwners = false
+      "This status was created when the monitor was created.",
+      undefined,
+      onCreate.createBy.props,
+    );
+
+    // 2. Start core operations in parallel that can run asynchronously (excluding workspace operations)
+
+    // Add default probes if needed (can be slow with many probes)
+    if (
+      createdItem.monitorType &&
+      MonitorTypeHelper.isProbableMonitor(createdItem.monitorType)
+    ) {
+      parallelOperations.push(
+        this.addDefaultProbesToMonitor(
+          createdItem.projectId,
+          createdItem.id,
+        ).catch((error: Error) => {
+          logger.error("Error in adding default probes");
+          logger.error(error);
+          // Don't fail monitor creation due to probe creation issues
+        }),
+      );
+    }
+
+    // Billing operations
+    if (IsBillingEnabled) {
+      parallelOperations.push(
+        ActiveMonitoringMeteredPlan.reportQuantityToBillingProvider(
+          createdItem.projectId,
+        ).catch((error: Error) => {
+          logger.error("Error in billing operations");
+          logger.error(error);
+          // Don't fail monitor creation due to billing issues
+        }),
+      );
+    }
+
+    // Owner operations
+    if (
+      onCreate.createBy.miscDataProps &&
+      (onCreate.createBy.miscDataProps["ownerTeams"] ||
+        onCreate.createBy.miscDataProps["ownerUsers"])
+    ) {
+      parallelOperations.push(
+        this.addOwners(
+          createdItem.projectId,
+          createdItem.id,
+          (onCreate.createBy.miscDataProps["ownerUsers"] as Array<ObjectID>) ||
+            [],
+          (onCreate.createBy.miscDataProps["ownerTeams"] as Array<ObjectID>) ||
+            [],
+          false,
+          onCreate.createBy.props,
+        ).catch((error: Error) => {
+          logger.error("Error in adding owners");
+          logger.error(error);
+          // Don't fail monitor creation due to owner issues
+        }),
+      );
+    }
+
+    // Probe status refresh (can be expensive with many probes)
+    parallelOperations.push(
+      this.refreshMonitorProbeStatus(createdItem.id).catch((error: Error) => {
+        logger.error("Error in refreshing probe status");
+        logger.error(error);
+        // Don't fail monitor creation due to probe status issues
+      }),
+    );
+
+    // Wait for core operations to complete, then handle workspace operations
+    Promise.allSettled(parallelOperations)
+      .then(() => {
+        // Handle workspace operations after core operations complete
+        // Run workspace operations in background without blocking response
+        this.handleWorkspaceOperationsAsync({
+          projectId: createdItem.projectId!,
+          monitorId: createdItem.id!,
+          monitorName: createdItem.name!,
+          feedInfoInMarkdown,
+          createdByUserId,
+        }).catch((error: Error) => {
+          logger.error("Error in workspace operations");
+          logger.error(error);
+          // Don't fail monitor creation due to workspace issues
+        });
+      })
+      .catch((error: Error) => {
+        logger.error("Error in parallel monitor creation operations");
+        logger.error(error);
+      });
+
+    return createdItem;
+  }
+
+  @CaptureSpan()
+  private async handleWorkspaceOperationsAsync(data: {
+    projectId: ObjectID;
+    monitorId: ObjectID;
+    monitorName: string;
+    feedInfoInMarkdown: string;
+    createdByUserId: ObjectID | undefined | null;
+  }): Promise<void> {
+    // send message to workspaces - slack, teams, etc.
     const workspaceResult: {
       channelsCreated: Array<NotificationRuleWorkspaceChannel>;
     } | null =
       await MonitorWorkspaceMessages.createChannelsAndInviteUsersToChannels({
-        projectId: createdItem.projectId,
-        monitorId: createdItem.id!,
-        monitorName: createdItem.name!,
+        projectId: data.projectId,
+        monitorId: data.monitorId,
+        monitorName: data.monitorName,
       });
 
     if (workspaceResult && workspaceResult.channelsCreated?.length > 0) {
-      // update incident with these channels.
+      // update monitor with these channels.
       await this.updateOneById({
-        id: createdItem.id!,
+        id: data.monitorId,
         data: {
           postUpdatesToWorkspaceChannels: workspaceResult.channelsCreated || [],
         },
@@ -526,72 +639,22 @@ ${createdItem.description?.trim() || "No description provided."}
 
     const monitorCreateMessageBlocks: Array<MessageBlocksByWorkspaceType> =
       await MonitorWorkspaceMessages.getMonitorCreateMessageBlocks({
-        monitorId: createdItem.id!,
-        projectId: createdItem.projectId!,
+        monitorId: data.monitorId,
+        projectId: data.projectId,
       });
 
     await MonitorFeedService.createMonitorFeedItem({
-      monitorId: createdItem.id!,
-      projectId: createdItem.projectId!,
+      monitorId: data.monitorId,
+      projectId: data.projectId,
       monitorFeedEventType: MonitorFeedEventType.MonitorCreated,
       displayColor: Green500,
-      feedInfoInMarkdown: feedInfoInMarkdown,
-      userId: createdByUserId || undefined,
+      feedInfoInMarkdown: data.feedInfoInMarkdown,
+      userId: data.createdByUserId || undefined,
       workspaceNotification: {
         appendMessageBlocks: monitorCreateMessageBlocks,
         sendWorkspaceNotification: true,
       },
     });
-
-    await this.changeMonitorStatus(
-      createdItem.projectId,
-      [createdItem.id],
-      createdItem.currentMonitorStatusId,
-      false, // notifyOwners = false
-      "This status was created when the monitor was created.",
-      undefined,
-      onCreate.createBy.props,
-    );
-
-    if (
-      createdItem.monitorType &&
-      MonitorTypeHelper.isProbableMonitor(createdItem.monitorType)
-    ) {
-      await this.addDefaultProbesToMonitor(
-        createdItem.projectId,
-        createdItem.id,
-      );
-    }
-
-    if (IsBillingEnabled) {
-      await ActiveMonitoringMeteredPlan.reportQuantityToBillingProvider(
-        createdItem.projectId,
-      );
-    }
-
-    // add owners.
-
-    if (
-      onCreate.createBy.miscDataProps &&
-      (onCreate.createBy.miscDataProps["ownerTeams"] ||
-        onCreate.createBy.miscDataProps["ownerUsers"])
-    ) {
-      await this.addOwners(
-        createdItem.projectId,
-        createdItem.id,
-        (onCreate.createBy.miscDataProps["ownerUsers"] as Array<ObjectID>) ||
-          [],
-        (onCreate.createBy.miscDataProps["ownerTeams"] as Array<ObjectID>) ||
-          [],
-        false,
-        onCreate.createBy.props,
-      );
-    }
-
-    // refresh probe status.
-    await this.refreshMonitorProbeStatus(createdItem.id);
-
-    return createdItem;
   }
 
   @CaptureSpan()
@@ -760,21 +823,32 @@ ${createdItem.description?.trim() || "No description provided."}
 
     const totalProbes: Array<Probe> = [...globalProbes, ...projectProbes];
 
+    if (totalProbes.length === 0) {
+      return;
+    }
+
+    // Create all monitor probes in parallel for better performance
+    const createPromises: Array<Promise<MonitorProbe>> = [];
+
     for (const probe of totalProbes) {
       const monitorProbe: MonitorProbe = new MonitorProbe();
-
       monitorProbe.monitorId = monitorId;
       monitorProbe.probeId = probe.id!;
       monitorProbe.projectId = projectId;
       monitorProbe.isEnabled = true;
 
-      await MonitorProbeService.create({
-        data: monitorProbe,
-        props: {
-          isRoot: true,
-        },
-      });
+      createPromises.push(
+        MonitorProbeService.create({
+          data: monitorProbe,
+          props: {
+            isRoot: true,
+          },
+        }),
+      );
     }
+
+    // Execute all creates in parallel
+    await Promise.all(createPromises);
   }
 
   @CaptureSpan()
@@ -1085,6 +1159,14 @@ ${createdItem.description?.trim() || "No description provided."}
         emailEnvelope: emailMessage,
         smsMessage: sms,
         callRequestMessage: callMessage,
+        pushNotificationMessage:
+          PushNotificationUtil.createMonitorProbeStatusNotification({
+            title: "OneUptime: Monitor Probe Status",
+            body: `Probes for monitor ${monitor.name} is ${enabledStatus}`,
+            tag: "monitor-probe-status",
+            monitorId: monitor.id!.toString(),
+            monitorName: monitor.name!,
+          }),
         eventType:
           NotificationSettingEventType.SEND_MONITOR_NOTIFICATION_WHEN_NO_PROBES_ARE_MONITORING_THE_MONITOR,
       });
@@ -1184,6 +1266,11 @@ ${createdItem.description?.trim() || "No description provided."}
         emailEnvelope: emailMessage,
         smsMessage: sms,
         callRequestMessage: callMessage,
+        pushNotificationMessage:
+          PushNotificationUtil.createMonitorCreatedNotification({
+            monitorName: monitor.name!,
+            monitorId: monitor.id!.toString(),
+          }),
         eventType:
           NotificationSettingEventType.SEND_MONITOR_NOTIFICATION_WHEN_PORBE_STATUS_CHANGES,
       });

Common/Server/Services/NotificationService.ts

@@ -1,4 +1,7 @@
-import { IsBillingEnabled } from "../EnvironmentConfig";
+import {
+  IsBillingEnabled,
+  NotificationSlackWebhookOnSubscriptionUpdate,
+} from "../EnvironmentConfig";
 import logger from "../Utils/Logger";
 import BaseService from "./BaseService";
 import BillingService from "./BillingService";
@@ -7,6 +10,9 @@ import BadDataException from "../../Types/Exception/BadDataException";
 import ObjectID from "../../Types/ObjectID";
 import Project from "../../Models/DatabaseModels/Project";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import SlackUtil from "../Utils/Workspace/Slack/Slack";
+import URL from "../../Types/API/URL";
+import Exception from "../../Types/Exception/Exception";
 
 export class NotificationService extends BaseService {
   public constructor() {
@@ -105,6 +111,17 @@ export class NotificationService extends BaseService {
         } USD.`,
       );
 
+      // Send Slack notification for balance refill
+      this.sendBalanceRefillSlackNotification({
+        project: project,
+        amountInUSD: amountInUSD,
+        currentBalanceInUSD: updatedAmount / 100,
+      }).catch((error: Exception) => {
+        logger.error(
+          "Error sending slack message for balance refill: " + error,
+        );
+      });
+
       project.smsOrCallCurrentBalanceInUSDCents = updatedAmount;
 
       return updatedAmount;
@@ -194,6 +211,34 @@ export class NotificationService extends BaseService {
 
     return project?.smsOrCallCurrentBalanceInUSDCents || 0;
   }
+
+  @CaptureSpan()
+  private async sendBalanceRefillSlackNotification(data: {
+    project: Project;
+    amountInUSD: number;
+    currentBalanceInUSD: number;
+  }): Promise<void> {
+    const { project, amountInUSD, currentBalanceInUSD } = data;
+
+    if (NotificationSlackWebhookOnSubscriptionUpdate) {
+      const slackMessage: string = `*SMS and Call Balance Refilled:*
+*Project Name:* ${project.name?.toString() || "N/A"}
+*Project ID:* ${project.id?.toString() || "N/A"}
+*Refill Amount:* $${amountInUSD} USD
+*Current Balance:* $${currentBalanceInUSD} USD
+
+${project.createdOwnerName && project.createdOwnerEmail ? `*Project Created By:* ${project.createdOwnerName.toString()} (${project.createdOwnerEmail.toString()})` : ""}`;
+
+      SlackUtil.sendMessageToChannelViaIncomingWebhook({
+        url: URL.fromString(NotificationSlackWebhookOnSubscriptionUpdate),
+        text: slackMessage,
+      }).catch((error: Exception) => {
+        logger.error(
+          "Error sending slack message for balance refill: " + error,
+        );
+      });
+    }
+  }
 }
 
 export default new NotificationService();

Common/Server/Services/OnCallDutyPolicyEscalationRuleScheduleService.ts

@@ -18,6 +18,8 @@ import OnCallDutyPolicySchedule from "../../Models/DatabaseModels/OnCallDutyPoli
 import OnCallDutyPolicyFeedService from "./OnCallDutyPolicyFeedService";
 import { OnCallDutyPolicyFeedEventType } from "../../Models/DatabaseModels/OnCallDutyPolicyFeed";
 import { Gray500, Red500 } from "../../Types/BrandColors";
+import PushNotificationMessage from "../../Types/PushNotification/PushNotificationMessage";
+import PushNotificationUtil from "../Utils/PushNotificationUtil";
 
 export class Service extends DatabaseService<Model> {
   public constructor() {
@@ -128,12 +130,18 @@ export class Service extends DatabaseService<Model> {
       ],
     };
 
+    const pushMessage: PushNotificationMessage =
+      PushNotificationUtil.createOnCallPolicyAddedNotification({
+        policyName: createdModel.onCallDutyPolicy?.name || "No name provided",
+      });
+
     await UserNotificationSettingService.sendUserNotification({
       userId: sendEmailToUserId,
       projectId: createdModel!.projectId!,
       emailEnvelope: emailMessage,
       smsMessage: sms,
       callRequestMessage: callMessage,
+      pushNotificationMessage: pushMessage,
       eventType:
         NotificationSettingEventType.SEND_WHEN_USER_IS_ADDED_TO_ON_CALL_POLICY,
     });
@@ -304,12 +312,18 @@ export class Service extends DatabaseService<Model> {
         ],
       };
 
+      const pushMessage: PushNotificationMessage =
+        PushNotificationUtil.createOnCallPolicyRemovedNotification({
+          policyName: deletedItem.onCallDutyPolicy?.name || "No name provided",
+        });
+
       await UserNotificationSettingService.sendUserNotification({
         userId: sendEmailToUserId,
         projectId: deletedItem!.projectId!,
         emailEnvelope: emailMessage,
         smsMessage: sms,
         callRequestMessage: callMessage,
+        pushNotificationMessage: pushMessage,
         eventType:
           NotificationSettingEventType.SEND_WHEN_USER_IS_REMOVED_FROM_ON_CALL_POLICY,
       });

Common/Server/Services/OnCallDutyPolicyEscalationRuleTeamService.ts

@@ -18,6 +18,8 @@ import User from "../../Models/DatabaseModels/User";
 import OnCallDutyPolicyFeedService from "./OnCallDutyPolicyFeedService";
 import { OnCallDutyPolicyFeedEventType } from "../../Models/DatabaseModels/OnCallDutyPolicyFeed";
 import { Gray500, Red500 } from "../../Types/BrandColors";
+import PushNotificationMessage from "../../Types/PushNotification/PushNotificationMessage";
+import PushNotificationUtil from "../Utils/PushNotificationUtil";
 import Team from "../../Models/DatabaseModels/Team";
 import OnCallDutyPolicyTimeLogService from "./OnCallDutyPolicyTimeLogService";
 import OneUptimeDate from "../../Types/Date";
@@ -127,12 +129,18 @@ export class Service extends DatabaseService<Model> {
         ],
       };
 
+      const pushMessage: PushNotificationMessage =
+        PushNotificationUtil.createOnCallPolicyAddedNotification({
+          policyName: createdModel.onCallDutyPolicy?.name || "No name provided",
+        });
+
       await UserNotificationSettingService.sendUserNotification({
         userId: sendEmailToUserId,
         projectId: createdModel!.projectId!,
         emailEnvelope: emailMessage,
         smsMessage: sms,
         callRequestMessage: callMessage,
+        pushNotificationMessage: pushMessage,
         eventType:
           NotificationSettingEventType.SEND_WHEN_USER_IS_ADDED_TO_ON_CALL_POLICY,
       });
@@ -308,12 +316,19 @@ export class Service extends DatabaseService<Model> {
           ],
         };
 
+        const pushMessage: PushNotificationMessage =
+          PushNotificationUtil.createOnCallPolicyRemovedNotification({
+            policyName:
+              deletedItem.onCallDutyPolicy?.name || "No name provided",
+          });
+
         UserNotificationSettingService.sendUserNotification({
           userId: sendEmailToUserId,
           projectId: deletedItem!.projectId!,
           emailEnvelope: emailMessage,
           smsMessage: sms,
           callRequestMessage: callMessage,
+          pushNotificationMessage: pushMessage,
           eventType:
             NotificationSettingEventType.SEND_WHEN_USER_IS_REMOVED_FROM_ON_CALL_POLICY,
         });

Common/Server/Services/OnCallDutyPolicyEscalationRuleUserService.ts

@@ -18,6 +18,8 @@ import { OnCallDutyPolicyFeedEventType } from "../../Models/DatabaseModels/OnCal
 import { Gray500, Red500 } from "../../Types/BrandColors";
 import UserService from "./UserService";
 import User from "../../Models/DatabaseModels/User";
+import PushNotificationMessage from "../../Types/PushNotification/PushNotificationMessage";
+import PushNotificationUtil from "../Utils/PushNotificationUtil";
 import OnCallDutyPolicyTimeLogService from "./OnCallDutyPolicyTimeLogService";
 import OneUptimeDate from "../../Types/Date";
 import logger from "../Utils/Logger";
@@ -110,12 +112,18 @@ export class Service extends DatabaseService<Model> {
       ],
     };
 
+    const pushMessage: PushNotificationMessage =
+      PushNotificationUtil.createOnCallPolicyAddedNotification({
+        policyName: createdModel.onCallDutyPolicy?.name || "",
+      });
+
     await UserNotificationSettingService.sendUserNotification({
       userId: sendEmailToUserId,
       projectId: createdModel!.projectId!,
       emailEnvelope: emailMessage,
       smsMessage: sms,
       callRequestMessage: callMessage,
+      pushNotificationMessage: pushMessage,
       eventType:
         NotificationSettingEventType.SEND_WHEN_USER_IS_ADDED_TO_ON_CALL_POLICY,
     });
@@ -306,12 +314,18 @@ export class Service extends DatabaseService<Model> {
         ],
       };
 
+      const pushMessage: PushNotificationMessage =
+        PushNotificationUtil.createOnCallPolicyRemovedNotification({
+          policyName: deletedItem.onCallDutyPolicy?.name || "",
+        });
+
       UserNotificationSettingService.sendUserNotification({
         userId: sendEmailToUserId,
         projectId: deletedItem!.projectId!,
         emailEnvelope: emailMessage,
         smsMessage: sms,
         callRequestMessage: callMessage,
+        pushNotificationMessage: pushMessage,
         eventType:
           NotificationSettingEventType.SEND_WHEN_USER_IS_REMOVED_FROM_ON_CALL_POLICY,
       });

Common/Server/Services/OnCallDutyPolicyScheduleService.ts

@@ -34,6 +34,8 @@ import { Green500 } from "../../Types/BrandColors";
 import OnCallDutyPolicyTimeLogService from "./OnCallDutyPolicyTimeLogService";
 import DeleteBy from "../Types/Database/DeleteBy";
 import { OnDelete } from "../Types/Database/Hooks";
+import PushNotificationMessage from "../../Types/PushNotification/PushNotificationMessage";
+import PushNotificationUtil from "../Utils/PushNotificationUtil";
 
 export class Service extends DatabaseService<OnCallDutyPolicySchedule> {
   private layerUtil = new LayerUtil();
@@ -253,12 +255,21 @@ export class Service extends DatabaseService<OnCallDutyPolicySchedule> {
             ],
           };
 
+          const pushMessage: PushNotificationMessage =
+            PushNotificationUtil.createGenericNotification({
+              title: "On-Call Duty Ended",
+              body: `You are no longer on-call for ${onCallPolicy.name!} as your roster on schedule ${onCallSchedule.name} has ended.`,
+              tag: "on-call-duty-ended",
+              requireInteraction: false,
+            });
+
           await UserNotificationSettingService.sendUserNotification({
             userId: sendEmailToUserId,
             projectId: projectId,
             emailEnvelope: emailMessage,
             smsMessage: sms,
             callRequestMessage: callMessage,
+            pushNotificationMessage: pushMessage,
             eventType:
               NotificationSettingEventType.SEND_WHEN_USER_IS_NO_LONGER_ACTIVE_ON_ON_CALL_ROSTER,
           });
@@ -360,12 +371,21 @@ export class Service extends DatabaseService<OnCallDutyPolicySchedule> {
             ],
           };
 
+          const pushMessage: PushNotificationMessage =
+            PushNotificationUtil.createGenericNotification({
+              title: "On-Call Duty Started",
+              body: `You are now on-call for ${onCallPolicy.name!} on schedule ${onCallSchedule.name}.`,
+              tag: "on-call-duty-started",
+              requireInteraction: true,
+            });
+
           await UserNotificationSettingService.sendUserNotification({
             userId: sendEmailToUserId,
             projectId: projectId,
             emailEnvelope: emailMessage,
             smsMessage: sms,
             callRequestMessage: callMessage,
+            pushNotificationMessage: pushMessage,
             eventType:
               NotificationSettingEventType.SEND_WHEN_USER_IS_ON_CALL_ROSTER,
           });
@@ -487,12 +507,21 @@ export class Service extends DatabaseService<OnCallDutyPolicySchedule> {
             ],
           };
 
+          const pushMessage: PushNotificationMessage =
+            PushNotificationUtil.createGenericNotification({
+              title: "Next On-Call Duty",
+              body: `You are next on-call for ${onCallPolicy.name!} on schedule ${onCallSchedule.name}.`,
+              tag: "next-on-call-duty",
+              requireInteraction: false,
+            });
+
           await UserNotificationSettingService.sendUserNotification({
             userId: sendEmailToUserId,
             projectId: projectId,
             emailEnvelope: emailMessage,
             smsMessage: sms,
             callRequestMessage: callMessage,
+            pushNotificationMessage: pushMessage,
             eventType:
               NotificationSettingEventType.SEND_WHEN_USER_IS_NEXT_ON_CALL_ROSTER,
           });

Common/Server/Services/ProbeService.ts

@@ -28,6 +28,8 @@ import DatabaseConfig from "../DatabaseConfig";
 import URL from "../../Types/API/URL";
 import UpdateBy from "../Types/Database/UpdateBy";
 import MonitorService from "./MonitorService";
+import PushNotificationMessage from "../../Types/PushNotification/PushNotificationMessage";
+import PushNotificationUtil from "../Utils/PushNotificationUtil";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import { IsBillingEnabled } from "../EnvironmentConfig";
 import GlobalCache from "../Infrastructure/GlobalCache";
@@ -365,12 +367,33 @@ export class Service extends DatabaseService<Model> {
           ],
         };
 
+        const pushMessageParams: {
+          probeName: string;
+          projectName: string;
+          connectionStatus: string;
+          clickAction?: string;
+        } = {
+          probeName: probe.name!,
+          projectName: probe.project?.name || "Project",
+          connectionStatus: connectionStatus,
+        };
+
+        if (vars["viewProbesLink"]) {
+          pushMessageParams.clickAction = vars["viewProbesLink"];
+        }
+
+        const pushMessage: PushNotificationMessage =
+          PushNotificationUtil.createProbeStatusChangedNotification(
+            pushMessageParams,
+          );
+
         await UserNotificationSettingService.sendUserNotification({
           userId: user.id!,
           projectId: probe.projectId!,
           emailEnvelope: emailMessage,
           smsMessage: sms,
           callRequestMessage: callMessage,
+          pushNotificationMessage: pushMessage,
           eventType:
             NotificationSettingEventType.SEND_PROBE_STATUS_CHANGED_OWNER_NOTIFICATION,
         });

Common/Server/Services/PushNotificationService.ts

@@ -0,0 +1,253 @@
+import PushNotificationRequest from "../../Types/PushNotification/PushNotificationRequest";
+import PushNotificationMessage from "../../Types/PushNotification/PushNotificationMessage";
+import ObjectID from "../../Types/ObjectID";
+import logger from "../Utils/Logger";
+import UserPushService from "./UserPushService";
+import UserOnCallLogTimelineService from "./UserOnCallLogTimelineService";
+import UserNotificationStatus from "../../Types/UserNotification/UserNotificationStatus";
+import {
+  VapidPublicKey,
+  VapidPrivateKey,
+  VapidSubject,
+} from "../EnvironmentConfig";
+import webpush from "web-push";
+import PushNotificationUtil from "../Utils/PushNotificationUtil";
+import { LIMIT_PER_PROJECT } from "../../Types/Database/LimitMax";
+import UserPush from "../../Models/DatabaseModels/UserPush";
+
+export interface PushNotificationOptions {
+  projectId?: ObjectID | undefined;
+  isSensitive?: boolean;
+  userOnCallLogTimelineId?: ObjectID | undefined;
+}
+
+export default class PushNotificationService {
+  public static isWebPushInitialized = false;
+
+  public static initializeWebPush(): void {
+    if (this.isWebPushInitialized) {
+      return;
+    }
+
+    if (!VapidPublicKey || !VapidPrivateKey) {
+      logger.warn(
+        "VAPID keys not configured. Web push notifications will not work.",
+      );
+      logger.warn(`VapidPublicKey present: ${Boolean(VapidPublicKey)}`);
+      logger.warn(`VapidPrivateKey present: ${Boolean(VapidPrivateKey)}`);
+      logger.warn(`VapidSubject: ${VapidSubject}`);
+      return;
+    }
+
+    logger.info(`Initializing web push with VAPID subject: ${VapidSubject}`);
+    webpush.setVapidDetails(VapidSubject, VapidPublicKey, VapidPrivateKey);
+    this.isWebPushInitialized = true;
+    logger.info("Web push notifications initialized successfully");
+  }
+
+  public static async sendPushNotification(
+    request: PushNotificationRequest,
+    options: PushNotificationOptions = {},
+  ): Promise<void> {
+    logger.info(
+      `Sending push notification to ${request.deviceTokens?.length} devices`,
+    );
+
+    if (!request.deviceTokens || request.deviceTokens.length === 0) {
+      logger.error("No device tokens provided for push notification");
+      throw new Error("No device tokens provided");
+    }
+
+    if (request.deviceType !== "web") {
+      logger.error(`Unsupported device type: ${request.deviceType}`);
+      throw new Error("Only web push notifications are supported");
+    }
+
+    logger.info(
+      `Sending web push notifications to ${request.deviceTokens.length} devices`,
+    );
+    logger.info(`Notification message: ${JSON.stringify(request.message)}`);
+
+    const promises: Promise<void>[] = [];
+
+    for (const deviceToken of request.deviceTokens) {
+      promises.push(
+        this.sendWebPushNotification(deviceToken, request.message, options),
+      );
+    }
+
+    const results: Array<any> = await Promise.allSettled(promises);
+
+    let successCount: number = 0;
+    let errorCount: number = 0;
+
+    results.forEach((result: any, index: number) => {
+      if (result.status === "fulfilled") {
+        successCount++;
+        logger.info(`Device ${index + 1}: Notification sent successfully`);
+      } else {
+        errorCount++;
+        logger.error(
+          `Failed to send notification to device ${index + 1}: ${result.reason}`,
+        );
+      }
+    });
+
+    logger.info(
+      `Push notification results: ${successCount} successful, ${errorCount} failed`,
+    );
+
+    // Update user on call log timeline status if provided
+    if (options.userOnCallLogTimelineId) {
+      const status: UserNotificationStatus =
+        successCount > 0
+          ? UserNotificationStatus.Sent
+          : UserNotificationStatus.Error;
+      const statusMessage: string =
+        successCount > 0
+          ? "Push notification sent successfully"
+          : `Failed to send push notification: ${errorCount} errors`;
+
+      await UserOnCallLogTimelineService.updateOneById({
+        id: options.userOnCallLogTimelineId,
+        data: {
+          status,
+          statusMessage,
+        },
+        props: {
+          isRoot: true,
+        },
+      });
+    }
+
+    if (errorCount > 0 && successCount === 0) {
+      throw new Error(
+        `Failed to send push notification to all ${errorCount} devices`,
+      );
+    }
+  }
+
+  private static async sendWebPushNotification(
+    deviceToken: string,
+    message: PushNotificationMessage,
+    _options: PushNotificationOptions,
+  ): Promise<void> {
+    if (!this.isWebPushInitialized) {
+      this.initializeWebPush();
+    }
+
+    if (!this.isWebPushInitialized) {
+      throw new Error("Web push notifications not configured");
+    }
+
+    try {
+      const payload: string = JSON.stringify({
+        title: message.title,
+        body: message.body,
+        icon: message.icon || PushNotificationUtil.DEFAULT_ICON,
+        badge: message.badge || PushNotificationUtil.DEFAULT_BADGE,
+        data: message.data || {},
+        tag: message.tag || "oneuptime-notification",
+        requireInteraction: message.requireInteraction || false,
+        actions: message.actions || [],
+        url: message.url || message.clickAction,
+      });
+
+      logger.debug(`Sending push notification with payload: ${payload}`);
+      logger.debug(`Device token: ${deviceToken}`);
+
+      let subscriptionObject: any;
+      try {
+        subscriptionObject = JSON.parse(deviceToken);
+        logger.debug(
+          `Parsed subscription object: ${JSON.stringify(subscriptionObject)}`,
+        );
+      } catch (parseError) {
+        logger.error(`Failed to parse device token: ${parseError}`);
+        throw new Error(`Invalid device token format: ${parseError}`);
+      }
+
+      const result: webpush.SendResult = await webpush.sendNotification(
+        subscriptionObject,
+        payload,
+        {
+          TTL: 24 * 60 * 60, // 24 hours
+        },
+      );
+
+      logger.debug(`Web push notification sent successfully:`);
+      logger.debug(`Result: ${JSON.stringify(result, null, 2)}`);
+      logger.debug(`Payload: ${JSON.stringify(payload, null, 2)}`);
+      logger.debug(
+        `Subscription object: ${JSON.stringify(subscriptionObject, null, 2)}`,
+      );
+
+      logger.info(`Web push notification sent successfully`);
+    } catch (error: any) {
+      logger.error(`Failed to send web push notification: ${error.message}`);
+      logger.error(error);
+
+      // If the subscription is no longer valid, remove it
+      if (error.statusCode === 410 || error.statusCode === 404) {
+        logger.info("Removing invalid web push subscription");
+        // You would implement removal logic here
+      }
+
+      throw error;
+    }
+  }
+
+  public static async sendPushNotificationToUser(
+    userId: ObjectID,
+    projectId: ObjectID,
+    message: PushNotificationMessage,
+    options: PushNotificationOptions = {},
+  ): Promise<void> {
+    // Get all verified push devices for the user
+    const userPushDevices: UserPush[] = await UserPushService.findBy({
+      query: {
+        userId: userId,
+        projectId: projectId,
+        isVerified: true,
+      },
+      select: {
+        deviceToken: true,
+        deviceType: true,
+        _id: true,
+      },
+      limit: LIMIT_PER_PROJECT,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    });
+
+    if (userPushDevices.length === 0) {
+      logger.info(
+        `No verified web push devices found for user ${userId.toString()}`,
+      );
+      return;
+    }
+
+    // Get web device tokens
+    const webDevices: string[] = [];
+
+    for (const device of userPushDevices) {
+      if (device.deviceType === "web") {
+        webDevices.push(device.deviceToken!);
+      }
+    }
+
+    // Send notifications to web devices
+    if (webDevices.length > 0) {
+      await this.sendPushNotification(
+        {
+          deviceTokens: webDevices,
+          message: message,
+          deviceType: "web",
+        },
+        options,
+      );
+    }
+  }
+}

Common/Server/Services/ScheduledMaintenanceService.ts

@@ -547,36 +547,7 @@ ${resourcesAffected ? `**Resources Affected:** ${resourcesAffected}` : ""}
     onCreate: OnCreate<Model>,
     createdItem: Model,
   ): Promise<Model> {
-    // create new scheduled maintenance state timeline.
-
-    const createdByUserId: ObjectID | undefined | null =
-      createdItem.createdByUserId || createdItem.createdByUser?.id;
-
-    // send message to workspaces - slack, teams,   etc.
-    const workspaceResult: {
-      channelsCreated: Array<NotificationRuleWorkspaceChannel>;
-    } | null =
-      await ScheduledMaintenanceWorkspaceMessages.createChannelsAndInviteUsersToChannels(
-        {
-          projectId: createdItem.projectId!,
-          scheduledMaintenanceId: createdItem.id!,
-          scheduledMaintenanceNumber: createdItem.scheduledMaintenanceNumber!,
-        },
-      );
-
-    if (workspaceResult && workspaceResult.channelsCreated?.length > 0) {
-      // update scheduledMaintenance with these channels.
-      await this.updateOneById({
-        id: createdItem.id!,
-        data: {
-          postUpdatesToWorkspaceChannels: workspaceResult.channelsCreated || [],
-        },
-        props: {
-          isRoot: true,
-        },
-      });
-    }
-
+    // Get scheduled maintenance data for feed creation
     const scheduledMaintenance: Model | null = await this.findOneById({
       id: createdItem.id!,
       select: {
@@ -606,83 +577,23 @@ ${resourcesAffected ? `**Resources Affected:** ${resourcesAffected}` : ""}
       throw new BadDataException("Scheduled Maintenance not found");
     }
 
-    let feedInfoInMarkdown: string = `#### 🕒 Scheduled Maintenance ${createdItem.scheduledMaintenanceNumber?.toString()} Created: 
-          
-**${createdItem.title || "No title provided."}**:
-    
-${createdItem.description || "No description provided."}
-    
-`;
+    // Execute core operations in parallel first
+    const coreOperations: Array<Promise<any>> = [];
 
-    // add starts at and ends at.
-    if (scheduledMaintenance.startsAt) {
-      feedInfoInMarkdown += `**Starts At**: ${OneUptimeDate.getDateAsLocalFormattedString(scheduledMaintenance.startsAt)} \n\n`;
-    }
-
-    if (scheduledMaintenance.endsAt) {
-      feedInfoInMarkdown += `**Ends At**: ${OneUptimeDate.getDateAsLocalFormattedString(scheduledMaintenance.endsAt)} \n\n`;
-    }
-
-    if (scheduledMaintenance.currentScheduledMaintenanceState?.name) {
-      feedInfoInMarkdown += `⏳ **Scheduled Maintenance State**: ${scheduledMaintenance.currentScheduledMaintenanceState.name} \n\n`;
-    }
-
-    if (
-      scheduledMaintenance.monitors &&
-      scheduledMaintenance.monitors.length > 0
-    ) {
-      feedInfoInMarkdown += `🌎 **Resources Affected**:\n`;
-
-      for (const monitor of scheduledMaintenance.monitors) {
-        feedInfoInMarkdown += `- [${monitor.name}](${(await MonitorService.getMonitorLinkInDashboard(createdItem.projectId!, monitor.id!)).toString()})\n`;
-      }
-
-      feedInfoInMarkdown += `\n\n`;
-    }
-
-    const scheduledMaintenanceCreateMessageBlocks: Array<MessageBlocksByWorkspaceType> =
-      await ScheduledMaintenanceWorkspaceMessages.getScheduledMaintenanceCreateMessageBlocks(
-        {
-          scheduledMaintenanceId: createdItem.id!,
-          projectId: createdItem.projectId!,
-        },
-      );
-
-    await ScheduledMaintenanceFeedService.createScheduledMaintenanceFeedItem({
-      scheduledMaintenanceId: createdItem.id!,
-      projectId: createdItem.projectId!,
-      scheduledMaintenanceFeedEventType:
-        ScheduledMaintenanceFeedEventType.ScheduledMaintenanceCreated,
-      displayColor: Red500,
-      feedInfoInMarkdown: feedInfoInMarkdown,
-      userId: createdByUserId || undefined,
-      workspaceNotification: {
-        appendMessageBlocks: scheduledMaintenanceCreateMessageBlocks,
-        sendWorkspaceNotification: true,
-      },
-    });
-
-    const timeline: ScheduledMaintenanceStateTimeline =
-      new ScheduledMaintenanceStateTimeline();
-    timeline.projectId = createdItem.projectId!;
-    timeline.scheduledMaintenanceId = createdItem.id!;
-    timeline.isOwnerNotified = true; // ignore notifying owners because you already notify for Scheduled Event, no need to notify them for timeline event.
-    timeline.shouldStatusPageSubscribersBeNotified = Boolean(
-      createdItem.shouldStatusPageSubscribersBeNotifiedOnEventCreated,
+    // Create feed item asynchronously
+    coreOperations.push(
+      this.createScheduledMaintenanceFeedAsync(
+        scheduledMaintenance,
+        createdItem,
+      ),
     );
-    timeline.isStatusPageSubscribersNotified = Boolean(
-      createdItem.shouldStatusPageSubscribersBeNotifiedOnEventCreated,
-    ); // ignore notifying subscribers because you already notify for Scheduled Event, no need to notify them for timeline event.
-    timeline.scheduledMaintenanceStateId =
-      createdItem.currentScheduledMaintenanceStateId!;
 
-    await ScheduledMaintenanceStateTimelineService.create({
-      data: timeline,
-      props: {
-        isRoot: true,
-      },
-    });
+    // Create state timeline asynchronously
+    coreOperations.push(
+      this.createScheduledMaintenanceStateTimelineAsync(createdItem),
+    );
 
+    // Handle owner assignment asynchronously
     if (
       createdItem.projectId &&
       createdItem.id &&
@@ -690,21 +601,200 @@ ${createdItem.description || "No description provided."}
       (onCreate.createBy.miscDataProps["ownerTeams"] ||
         onCreate.createBy.miscDataProps["ownerUsers"])
     ) {
-      await this.addOwners(
-        createdItem.projectId!,
-        createdItem.id!,
-        (onCreate.createBy.miscDataProps["ownerUsers"] as Array<ObjectID>) ||
-          [],
-        (onCreate.createBy.miscDataProps["ownerTeams"] as Array<ObjectID>) ||
-          [],
-        false,
-        onCreate.createBy.props,
+      coreOperations.push(
+        this.addOwners(
+          createdItem.projectId!,
+          createdItem.id!,
+          (onCreate.createBy.miscDataProps["ownerUsers"] as Array<ObjectID>) ||
+            [],
+          (onCreate.createBy.miscDataProps["ownerTeams"] as Array<ObjectID>) ||
+            [],
+          false,
+          onCreate.createBy.props,
+        ),
       );
     }
 
+    // Execute core operations in parallel with error handling
+    Promise.allSettled(coreOperations)
+      .then((coreResults: any[]) => {
+        // Log any errors from core operations
+        coreResults.forEach((result: any, index: number) => {
+          if (result.status === "rejected") {
+            logger.error(
+              `Core operation ${index} failed in ScheduledMaintenanceService.onCreateSuccess: ${result.reason}`,
+            );
+          }
+        });
+
+        // Handle workspace operations after core operations complete
+        if (createdItem.projectId && createdItem.id) {
+          // Run workspace operations in background without blocking response
+          this.handleScheduledMaintenanceWorkspaceOperationsAsync(
+            createdItem,
+          ).catch((error: Error) => {
+            logger.error(
+              `Workspace operations failed in ScheduledMaintenanceService.onCreateSuccess: ${error}`,
+            );
+          });
+        }
+      })
+      .catch((error: Error) => {
+        logger.error(
+          `Critical error in ScheduledMaintenanceService core operations: ${error}`,
+        );
+      });
+
     return createdItem;
   }
 
+  @CaptureSpan()
+  private async handleScheduledMaintenanceWorkspaceOperationsAsync(
+    createdItem: Model,
+  ): Promise<void> {
+    try {
+      if (!createdItem.projectId || !createdItem.id) {
+        throw new BadDataException(
+          "projectId and id are required for workspace operations",
+        );
+      }
+
+      // send message to workspaces - slack, teams, etc.
+      const workspaceResult: {
+        channelsCreated: Array<NotificationRuleWorkspaceChannel>;
+      } | null =
+        await ScheduledMaintenanceWorkspaceMessages.createChannelsAndInviteUsersToChannels(
+          {
+            projectId: createdItem.projectId,
+            scheduledMaintenanceId: createdItem.id,
+            scheduledMaintenanceNumber: createdItem.scheduledMaintenanceNumber!,
+          },
+        );
+
+      if (workspaceResult && workspaceResult.channelsCreated?.length > 0) {
+        // update scheduledMaintenance with these channels.
+        await this.updateOneById({
+          id: createdItem.id,
+          data: {
+            postUpdatesToWorkspaceChannels:
+              workspaceResult.channelsCreated || [],
+          },
+          props: {
+            isRoot: true,
+          },
+        });
+      }
+    } catch (error) {
+      logger.error(
+        `Error in handleScheduledMaintenanceWorkspaceOperationsAsync: ${error}`,
+      );
+      throw error;
+    }
+  }
+
+  @CaptureSpan()
+  private async createScheduledMaintenanceFeedAsync(
+    scheduledMaintenance: Model,
+    createdItem: Model,
+  ): Promise<void> {
+    try {
+      const createdByUserId: ObjectID | undefined | null =
+        createdItem.createdByUserId || createdItem.createdByUser?.id;
+
+      let feedInfoInMarkdown: string = `#### 🕒 Scheduled Maintenance ${createdItem.scheduledMaintenanceNumber?.toString()} Created: 
+            
+**${createdItem.title || "No title provided."}**:
+      
+${createdItem.description || "No description provided."}
+      
+`;
+
+      // add starts at and ends at.
+      if (scheduledMaintenance.startsAt) {
+        feedInfoInMarkdown += `**Starts At**: ${OneUptimeDate.getDateAsLocalFormattedString(scheduledMaintenance.startsAt)} \n\n`;
+      }
+
+      if (scheduledMaintenance.endsAt) {
+        feedInfoInMarkdown += `**Ends At**: ${OneUptimeDate.getDateAsLocalFormattedString(scheduledMaintenance.endsAt)} \n\n`;
+      }
+
+      if (scheduledMaintenance.currentScheduledMaintenanceState?.name) {
+        feedInfoInMarkdown += `⏳ **Scheduled Maintenance State**: ${scheduledMaintenance.currentScheduledMaintenanceState.name} \n\n`;
+      }
+
+      if (
+        scheduledMaintenance.monitors &&
+        scheduledMaintenance.monitors.length > 0
+      ) {
+        feedInfoInMarkdown += `🌎 **Resources Affected**:\n`;
+
+        for (const monitor of scheduledMaintenance.monitors) {
+          feedInfoInMarkdown += `- [${monitor.name}](${(await MonitorService.getMonitorLinkInDashboard(createdItem.projectId!, monitor.id!)).toString()})\n`;
+        }
+
+        feedInfoInMarkdown += `\n\n`;
+      }
+
+      const scheduledMaintenanceCreateMessageBlocks: Array<MessageBlocksByWorkspaceType> =
+        await ScheduledMaintenanceWorkspaceMessages.getScheduledMaintenanceCreateMessageBlocks(
+          {
+            scheduledMaintenanceId: createdItem.id!,
+            projectId: createdItem.projectId!,
+          },
+        );
+
+      await ScheduledMaintenanceFeedService.createScheduledMaintenanceFeedItem({
+        scheduledMaintenanceId: createdItem.id!,
+        projectId: createdItem.projectId!,
+        scheduledMaintenanceFeedEventType:
+          ScheduledMaintenanceFeedEventType.ScheduledMaintenanceCreated,
+        displayColor: Red500,
+        feedInfoInMarkdown: feedInfoInMarkdown,
+        userId: createdByUserId || undefined,
+        workspaceNotification: {
+          appendMessageBlocks: scheduledMaintenanceCreateMessageBlocks,
+          sendWorkspaceNotification: true,
+        },
+      });
+    } catch (error) {
+      logger.error(`Error in createScheduledMaintenanceFeedAsync: ${error}`);
+      throw error;
+    }
+  }
+
+  @CaptureSpan()
+  private async createScheduledMaintenanceStateTimelineAsync(
+    createdItem: Model,
+  ): Promise<void> {
+    try {
+      const timeline: ScheduledMaintenanceStateTimeline =
+        new ScheduledMaintenanceStateTimeline();
+      timeline.projectId = createdItem.projectId!;
+      timeline.scheduledMaintenanceId = createdItem.id!;
+      timeline.isOwnerNotified = true; // ignore notifying owners because you already notify for Scheduled Event, no need to notify them for timeline event.
+      timeline.shouldStatusPageSubscribersBeNotified = Boolean(
+        createdItem.shouldStatusPageSubscribersBeNotifiedOnEventCreated,
+      );
+      timeline.isStatusPageSubscribersNotified = Boolean(
+        createdItem.shouldStatusPageSubscribersBeNotifiedOnEventCreated,
+      ); // ignore notifying subscribers because you already notify for Scheduled Event, no need to notify them for timeline event.
+      timeline.scheduledMaintenanceStateId =
+        createdItem.currentScheduledMaintenanceStateId!;
+
+      await ScheduledMaintenanceStateTimelineService.create({
+        data: timeline,
+        props: {
+          isRoot: true,
+        },
+      });
+    } catch (error) {
+      logger.error(
+        `Error in createScheduledMaintenanceStateTimelineAsync: ${error}`,
+      );
+      throw error;
+    }
+  }
+
   @CaptureSpan()
   public async addOwners(
     projectId: ObjectID,

Common/Server/Services/StatusPageService.ts

@@ -84,6 +84,20 @@ export class Service extends DatabaseService<StatusPage> {
     super(StatusPage);
   }
 
+  public static getDefaultEmailFooterText(): string {
+    return "This is an automated email sent to you because you are subscribed to this Status Page.";
+  }
+
+  public static getSubscriberEmailFooterText(statusPage: StatusPage): string {
+    if (
+      statusPage.enableCustomSubscriberEmailNotificationFooterText &&
+      statusPage.subscriberEmailNotificationFooterText
+    ) {
+      return statusPage.subscriberEmailNotificationFooterText;
+    }
+    return this.getDefaultEmailFooterText();
+  }
+
   @CaptureSpan()
   protected override async onBeforeCreate(
     createBy: CreateBy<StatusPage>,
@@ -154,10 +168,19 @@ export class Service extends DatabaseService<StatusPage> {
       createBy.data.defaultBarColor = Green;
     }
 
+    // For new status pages, set enableCustomSubscriberEmailNotificationFooterText to false by default
+    // and provide a default custom footer text only if not provided
+    if (
+      createBy.data.enableCustomSubscriberEmailNotificationFooterText ===
+      undefined
+    ) {
+      createBy.data.enableCustomSubscriberEmailNotificationFooterText = false;
+    }
+
     if (!createBy.data.subscriberEmailNotificationFooterText) {
       createBy.data.subscriberEmailNotificationFooterText =
         "This is an automated email sent to you because you are subscribed to " +
-        createBy.data.name;
+        (createBy.data?.pageTitle || createBy.data?.name || "Status Page");
     }
 
     return {
@@ -171,8 +194,7 @@ export class Service extends DatabaseService<StatusPage> {
     onCreate: OnCreate<StatusPage>,
     createdItem: StatusPage,
   ): Promise<StatusPage> {
-    // add owners.
-
+    // Execute owner assignment asynchronously
     if (
       createdItem.projectId &&
       createdItem.id &&
@@ -180,16 +202,19 @@ export class Service extends DatabaseService<StatusPage> {
       (onCreate.createBy.miscDataProps["ownerTeams"] ||
         onCreate.createBy.miscDataProps["ownerUsers"])
     ) {
-      await this.addOwners(
+      // Run owner assignment in background without blocking
+      this.addOwners(
         createdItem.projectId!,
         createdItem.id!,
-        (onCreate.createBy.miscDataProps["ownerUsers"] as Array<ObjectID>) ||
+        (onCreate.createBy.miscDataProps!["ownerUsers"] as Array<ObjectID>) ||
           [],
-        (onCreate.createBy.miscDataProps["ownerTeams"] as Array<ObjectID>) ||
+        (onCreate.createBy.miscDataProps!["ownerTeams"] as Array<ObjectID>) ||
           [],
         false,
         onCreate.createBy.props,
-      );
+      ).catch((error: Error) => {
+        logger.error(`Error in StatusPageService owner assignment: ${error}`);
+      });
     }
 
     return createdItem;
@@ -730,7 +755,7 @@ export class Service extends DatabaseService<StatusPage> {
           vars: {
             statusPageName: statusPageName,
             subscriberEmailNotificationFooterText:
-              statuspage.subscriberEmailNotificationFooterText || "",
+              Service.getSubscriberEmailFooterText(statuspage),
             statusPageUrl: statusPageURL,
             hasResources: report.totalResources > 0 ? "true" : "false",
             report: report as any,

Common/Server/Services/StatusPageSubscriberService.ts

@@ -839,6 +839,7 @@ Stay informed about service availability! 🚀`;
         logoFileId: true,
         allowSubscribersToChooseResources: true,
         subscriberEmailNotificationFooterText: true,
+        enableCustomSubscriberEmailNotificationFooterText: true,
         allowSubscribersToChooseEventTypes: true,
         smtpConfig: {
           _id: true,

Common/Server/Services/UserNotificationRuleService.ts

@@ -45,8 +45,11 @@ import AlertSeverity from "../../Models/DatabaseModels/AlertSeverity";
 import AlertSeverityService from "./AlertSeverityService";
 import WorkspaceNotificationRule from "../../Models/DatabaseModels/WorkspaceNotificationRule";
 import WorkspaceNotificationRuleService from "./WorkspaceNotificationRuleService";
+import PushNotificationService from "./PushNotificationService";
 import NotificationRuleEventType from "../../Types/Workspace/NotificationRules/EventType";
 import NotificationRuleWorkspaceChannel from "../../Types/Workspace/NotificationRules/NotificationRuleWorkspaceChannel";
+import PushNotificationUtil from "../Utils/PushNotificationUtil";
+import PushNotificationMessage from "../../Types/PushNotification/PushNotificationMessage";
 import logger from "../Utils/Logger";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 
@@ -135,6 +138,11 @@ export class Service extends DatabaseService<Model> {
           email: true,
           isVerified: true,
         },
+        userPush: {
+          deviceToken: true,
+          deviceType: true,
+          isVerified: true,
+        },
       },
       props: {
         isRoot: true,
@@ -585,6 +593,140 @@ export class Service extends DatabaseService<Model> {
         },
       });
     }
+
+    // send push notification.
+    if (
+      notificationRuleItem.userPush?.deviceToken &&
+      notificationRuleItem.userPush?.isVerified
+    ) {
+      // send push notification for alert
+      if (
+        options.userNotificationEventType ===
+          UserNotificationEventType.AlertCreated &&
+        alert
+      ) {
+        // create a log.
+        logTimelineItem.status = UserNotificationStatus.Sending;
+        logTimelineItem.statusMessage = `Sending push notification to device.`;
+        logTimelineItem.userPushId = notificationRuleItem.userPush.id!;
+
+        const updatedLog: UserOnCallLogTimeline =
+          await UserOnCallLogTimelineService.create({
+            data: logTimelineItem,
+            props: {
+              isRoot: true,
+            },
+          });
+
+        const pushMessage: PushNotificationMessage =
+          PushNotificationUtil.createAlertCreatedNotification({
+            alertTitle: alert.title!,
+            projectName: alert.project?.name || "OneUptime",
+            alertViewLink: (
+              await AlertService.getAlertLinkInDashboard(
+                alert.projectId!,
+                alert.id!,
+              )
+            ).toString(),
+          });
+
+        // send push notification.
+        PushNotificationService.sendPushNotification(
+          {
+            deviceTokens: [notificationRuleItem.userPush.deviceToken!],
+            message: pushMessage,
+            deviceType: notificationRuleItem.userPush.deviceType!,
+          },
+          {
+            projectId: options.projectId,
+            userOnCallLogTimelineId: updatedLog.id!,
+          },
+        ).catch(async (err: Error) => {
+          await UserOnCallLogTimelineService.updateOneById({
+            id: updatedLog.id!,
+            data: {
+              status: UserNotificationStatus.Error,
+              statusMessage: err.message || "Error sending push notification.",
+            },
+            props: {
+              isRoot: true,
+            },
+          });
+        });
+      }
+
+      // send push notification for incident
+      if (
+        options.userNotificationEventType ===
+          UserNotificationEventType.IncidentCreated &&
+        incident
+      ) {
+        // create a log.
+        logTimelineItem.status = UserNotificationStatus.Sending;
+        logTimelineItem.statusMessage = `Sending push notification to device.`;
+        logTimelineItem.userPushId = notificationRuleItem.userPush.id!;
+
+        const updatedLog: UserOnCallLogTimeline =
+          await UserOnCallLogTimelineService.create({
+            data: logTimelineItem,
+            props: {
+              isRoot: true,
+            },
+          });
+
+        const pushMessage: PushNotificationMessage =
+          PushNotificationUtil.createIncidentCreatedNotification({
+            incidentTitle: incident.title!,
+            projectName: incident.project?.name || "OneUptime",
+            incidentViewLink: (
+              await IncidentService.getIncidentLinkInDashboard(
+                incident.projectId!,
+                incident.id!,
+              )
+            ).toString(),
+          });
+
+        // send push notification.
+        PushNotificationService.sendPushNotification(
+          {
+            deviceTokens: [notificationRuleItem.userPush.deviceToken!],
+            message: pushMessage,
+            deviceType: notificationRuleItem.userPush.deviceType!,
+          },
+          {
+            projectId: options.projectId,
+            userOnCallLogTimelineId: updatedLog.id!,
+          },
+        ).catch(async (err: Error) => {
+          await UserOnCallLogTimelineService.updateOneById({
+            id: updatedLog.id!,
+            data: {
+              status: UserNotificationStatus.Error,
+              statusMessage: err.message || "Error sending push notification.",
+            },
+            props: {
+              isRoot: true,
+            },
+          });
+        });
+      }
+    }
+
+    if (
+      notificationRuleItem.userPush?.deviceToken &&
+      !notificationRuleItem.userPush?.isVerified
+    ) {
+      // create a log.
+      logTimelineItem.status = UserNotificationStatus.Error;
+      logTimelineItem.statusMessage = `Push notification not sent because device is not verified.`;
+
+      await UserOnCallLogTimelineService.create({
+        data: logTimelineItem,
+        props: {
+          isRoot: true,
+        },
+      });
+    }
   }
 
   @CaptureSpan()
@@ -989,9 +1131,13 @@ export class Service extends DatabaseService<Model> {
       !createBy.data.userEmail &&
       !createBy.data.userSms &&
       !createBy.data.userSmsId &&
-      !createBy.data.userEmailId
+      !createBy.data.userEmailId &&
+      !createBy.data.userPushId &&
+      !createBy.data.userPush
     ) {
-      throw new BadDataException("Call, SMS, or Email is required");
+      throw new BadDataException(
+        "Call, SMS, Email, or Push notification is required",
+      );
     }
 
     return {

Common/Server/Services/UserNotificationSettingService.ts

@@ -9,6 +9,7 @@ import TeamMemberService from "./TeamMemberService";
 import UserCallService from "./UserCallService";
 import UserEmailService from "./UserEmailService";
 import UserSmsService from "./UserSmsService";
+import PushNotificationService from "./PushNotificationService";
 import { CallRequestMessage } from "../../Types/Call/CallRequest";
 import { LIMIT_PER_PROJECT } from "../../Types/Database/LimitMax";
 import { EmailEnvelope } from "../../Types/Email/EmailMessage";
@@ -17,6 +18,7 @@ import NotificationSettingEventType from "../../Types/NotificationSetting/Notifi
 import ObjectID from "../../Types/ObjectID";
 import PositiveNumber from "../../Types/PositiveNumber";
 import { SMSMessage } from "../../Types/SMS/SMS";
+import PushNotificationMessage from "../../Types/PushNotification/PushNotificationMessage";
 import UserCall from "../../Models/DatabaseModels/UserCall";
 import UserEmail from "../../Models/DatabaseModels/UserEmail";
 import UserNotificationSetting from "../../Models/DatabaseModels/UserNotificationSetting";
@@ -36,6 +38,7 @@ export class Service extends DatabaseService<UserNotificationSetting> {
     emailEnvelope: EmailEnvelope;
     smsMessage: SMSMessage;
     callRequestMessage: CallRequestMessage;
+    pushNotificationMessage: PushNotificationMessage;
   }): Promise<void> {
     if (!data.projectId) {
       throw new BadDataException(
@@ -54,6 +57,7 @@ export class Service extends DatabaseService<UserNotificationSetting> {
           alertByEmail: true,
           alertBySMS: true,
           alertByCall: true,
+          alertByPush: true,
         },
         props: {
           isRoot: true,
@@ -157,6 +161,22 @@ export class Service extends DatabaseService<UserNotificationSetting> {
           });
         }
       }
+
+      if (notificationSettings.alertByPush) {
+        logger.debug(
+          `Sending push notification to user ${data.userId.toString()} for event ${data.eventType}`,
+        );
+        PushNotificationService.sendPushNotificationToUser(
+          data.userId,
+          data.projectId,
+          data.pushNotificationMessage,
+          {
+            projectId: data.projectId,
+          },
+        ).catch((err: Error) => {
+          logger.error(err);
+        });
+      }
     }
   }
 

Common/Server/Services/UserPushService.ts

@@ -0,0 +1,95 @@
+import CreateBy from "../Types/Database/CreateBy";
+import DeleteBy from "../Types/Database/DeleteBy";
+import { OnCreate, OnDelete } from "../Types/Database/Hooks";
+import DatabaseService from "./DatabaseService";
+import BadDataException from "../../Types/Exception/BadDataException";
+import PositiveNumber from "../../Types/PositiveNumber";
+import UserPush from "../../Models/DatabaseModels/UserPush";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+
+export class Service extends DatabaseService<UserPush> {
+  public constructor() {
+    super(UserPush);
+  }
+
+  @CaptureSpan()
+  protected override async onBeforeCreate(
+    createBy: CreateBy<UserPush>,
+  ): Promise<OnCreate<UserPush>> {
+    if (!createBy.data.deviceToken) {
+      throw new BadDataException("Device token is required");
+    }
+
+    if (!createBy.data.deviceType) {
+      throw new BadDataException("Device type is required");
+    }
+
+    // Validate device type
+    const validDeviceTypes: string[] = ["web", "android", "ios"];
+    if (!validDeviceTypes.includes(createBy.data.deviceType)) {
+      throw new BadDataException(
+        "Device type must be one of: " + validDeviceTypes.join(", "),
+      );
+    }
+
+    // Check if this device token already exists for this user and project
+    const existingCount: PositiveNumber = await this.countBy({
+      query: {
+        deviceToken: createBy.data.deviceToken,
+        userId: createBy.data.userId!,
+        projectId: createBy.data.projectId!,
+      },
+      props: {
+        isRoot: true,
+      },
+    });
+
+    if (existingCount.toNumber() > 0) {
+      throw new BadDataException(
+        "This device is already registered for push notifications",
+      );
+    }
+
+    return { carryForward: null, createBy };
+  }
+
+  @CaptureSpan()
+  protected override async onBeforeDelete(
+    deleteBy: DeleteBy<UserPush>,
+  ): Promise<OnDelete<UserPush>> {
+    // Add any cleanup logic here if needed
+    return { carryForward: null, deleteBy };
+  }
+
+  @CaptureSpan()
+  public async verifyDevice(deviceId: string): Promise<void> {
+    await this.updateOneBy({
+      query: {
+        _id: deviceId,
+      },
+      data: {
+        isVerified: true,
+      },
+      props: {
+        isRoot: true,
+      },
+    });
+  }
+
+  @CaptureSpan()
+  public async unverifyDevice(deviceId: string): Promise<void> {
+    await this.updateOneBy({
+      query: {
+        _id: deviceId,
+      },
+      data: {
+        isVerified: false,
+      },
+      props: {
+        isRoot: true,
+      },
+    });
+  }
+}
+
+export default new Service();

Common/Server/Utils/Monitor/Criteria/IncomingRequestCriteria.ts

@@ -137,7 +137,7 @@ export default class IncomingRequestCriteria {
               input.dataToProcess.monitorId.toString() +
               " is true",
           );
-          return `Incoming request / heartbeat received in ${value} minutes.`;
+          return `Incoming request / heartbeat received in ${value} minutes. It was received ${differenceInMinutes} minutes ago.`;
         }
         return null;
       }
@@ -153,7 +153,7 @@ export default class IncomingRequestCriteria {
               input.dataToProcess.monitorId.toString() +
               " is true",
           );
-          return `Incoming request / heartbeat not received in ${value} minutes.`;
+          return `Incoming request / heartbeat not received in ${value} minutes. It was received ${differenceInMinutes} minutes ago.`;
         }
         return null;
       }

Common/Server/Utils/Monitor/Criteria/ServerMonitorCriteria.ts

@@ -56,9 +56,13 @@ export default class ServerMonitorCriteria {
     const lastCheckTime: Date = (input.dataToProcess as ServerMonitorResponse)
       .requestReceivedAt;
 
+    const timeNow: Date =
+      (input.dataToProcess as ServerMonitorResponse).timeNow ||
+      OneUptimeDate.getCurrentDate();
+
     const differenceInMinutes: number = OneUptimeDate.getDifferenceInMinutes(
       lastCheckTime,
-      OneUptimeDate.getCurrentDate(),
+      timeNow,
     );
 
     let offlineIfNotCheckedInMinutes: number = 3;

Common/Server/Utils/Monitor/MonitorResource.ts

@@ -228,6 +228,8 @@ export default class MonitorResourceUtil {
       await MonitorService.updateOneById({
         id: monitor.id!,
         data: {
+          incomingRequestMonitorHeartbeatCheckedAt:
+            OneUptimeDate.getCurrentDate(),
           incomingMonitorRequest: {
             ...dataToProcess,
           } as any,
@@ -1372,7 +1374,7 @@ export default class MonitorResourceUtil {
     }
 
     if (input.monitor.monitorType === MonitorType.SSLCertificate) {
-      // check server monitor
+      // check SSL monitor
       const sslMonitorResult: string | null =
         await SSLMonitorCriteria.isMonitorInstanceCriteriaFilterMet({
           dataToProcess: input.dataToProcess,

Common/Server/Utils/PushNotificationUtil.ts

@@ -0,0 +1,308 @@
+import PushNotificationMessage from "../../Types/PushNotification/PushNotificationMessage";
+
+export default class PushNotificationUtil {
+  public static readonly DEFAULT_ICON =
+    "/dashboard/assets/img/OneUptimePNG/1.png";
+  public static readonly DEFAULT_BADGE =
+    "/dashboard/assets/img/OneUptimePNG/6.png";
+
+  private static applyDefaults(
+    notification: Partial<PushNotificationMessage>,
+  ): PushNotificationMessage {
+    return {
+      icon: PushNotificationUtil.DEFAULT_ICON,
+      badge: PushNotificationUtil.DEFAULT_BADGE,
+      ...notification,
+    } as PushNotificationMessage;
+  }
+  public static createIncidentCreatedNotification(params: {
+    incidentTitle: string;
+    projectName: string;
+    incidentViewLink: string;
+  }): PushNotificationMessage {
+    const { incidentTitle, projectName, incidentViewLink } = params;
+    return PushNotificationUtil.applyDefaults({
+      title: `New Incident: ${incidentTitle}`,
+      body: `A new incident has been created in ${projectName}. Click to view details.`,
+      clickAction: incidentViewLink,
+      url: incidentViewLink,
+      tag: "incident-created",
+      requireInteraction: true,
+      data: {
+        type: "incident-created",
+        incidentTitle: incidentTitle,
+        projectName: projectName,
+        url: incidentViewLink,
+      },
+    });
+  }
+
+  public static createIncidentStateChangedNotification(params: {
+    incidentTitle: string;
+    projectName: string;
+    newState: string;
+    incidentViewLink: string;
+  }): PushNotificationMessage {
+    const { incidentTitle, projectName, newState, incidentViewLink } = params;
+    return PushNotificationUtil.applyDefaults({
+      title: `Incident Updated: ${incidentTitle}`,
+      body: `Incident state changed to ${newState} in ${projectName}. Click to view details.`,
+      clickAction: incidentViewLink,
+      url: incidentViewLink,
+      tag: "incident-state-changed",
+      requireInteraction: true,
+      data: {
+        type: "incident-state-changed",
+        incidentTitle: incidentTitle,
+        projectName: projectName,
+        newState: newState,
+        url: incidentViewLink,
+      },
+    });
+  }
+
+  public static createIncidentNotePostedNotification(params: {
+    incidentTitle: string;
+    projectName: string;
+    isPrivateNote: boolean;
+    incidentViewLink: string;
+  }): PushNotificationMessage {
+    const { incidentTitle, projectName, isPrivateNote, incidentViewLink } =
+      params;
+    const noteType: string = isPrivateNote ? "Private" : "Public";
+    return PushNotificationUtil.applyDefaults({
+      title: `${noteType} Note Added: ${incidentTitle}`,
+      body: `A ${noteType.toLowerCase()} note has been posted on incident in ${projectName}. Click to view details.`,
+      clickAction: incidentViewLink,
+      url: incidentViewLink,
+      tag: "incident-note-posted",
+      requireInteraction: true,
+      data: {
+        type: "incident-note-posted",
+        incidentTitle: incidentTitle,
+        projectName: projectName,
+        isPrivateNote: isPrivateNote,
+        url: incidentViewLink,
+      },
+    });
+  }
+
+  public static createAlertCreatedNotification(params: {
+    alertTitle: string;
+    projectName: string;
+    alertViewLink: string;
+  }): PushNotificationMessage {
+    const { alertTitle, projectName, alertViewLink } = params;
+    return PushNotificationUtil.applyDefaults({
+      title: `New Alert: ${alertTitle}`,
+      body: `A new alert has been created in ${projectName}. Click to view details.`,
+      clickAction: alertViewLink,
+      url: alertViewLink,
+      tag: "alert-created",
+      requireInteraction: true,
+      data: {
+        type: "alert-created",
+        alertTitle: alertTitle,
+        projectName: projectName,
+        url: alertViewLink,
+      },
+    });
+  }
+
+  public static createMonitorStatusChangedNotification(params: {
+    monitorName: string;
+    projectName: string;
+    newStatus: string;
+    monitorViewLink: string;
+  }): PushNotificationMessage {
+    const { monitorName, projectName, newStatus, monitorViewLink } = params;
+    return PushNotificationUtil.applyDefaults({
+      title: `Monitor ${newStatus}: ${monitorName}`,
+      body: `Monitor status changed to ${newStatus} in ${projectName}. Click to view details.`,
+      clickAction: monitorViewLink,
+      url: monitorViewLink,
+      tag: "monitor-status-changed",
+      requireInteraction: true,
+      data: {
+        type: "monitor-status-changed",
+        monitorName: monitorName,
+        projectName: projectName,
+        newStatus: newStatus,
+        url: monitorViewLink,
+      },
+    });
+  }
+
+  public static createScheduledMaintenanceNotification(params: {
+    title: string;
+    projectName: string;
+    state: string;
+    viewLink: string;
+  }): PushNotificationMessage {
+    const { title, projectName, state, viewLink } = params;
+    return PushNotificationUtil.applyDefaults({
+      title: `Scheduled Maintenance ${state}: ${title}`,
+      body: `Scheduled maintenance ${state.toLowerCase()} in ${projectName}. Click to view details.`,
+      clickAction: viewLink,
+      url: viewLink,
+      tag: "scheduled-maintenance",
+      requireInteraction: false,
+      data: {
+        type: "scheduled-maintenance",
+        title: title,
+        projectName: projectName,
+        state: state,
+        url: viewLink,
+      },
+    });
+  }
+
+  public static createGenericNotification(params: {
+    title: string;
+    body: string;
+    clickAction?: string;
+    tag?: string;
+    requireInteraction?: boolean;
+  }): PushNotificationMessage {
+    const {
+      title,
+      body,
+      clickAction,
+      tag,
+      requireInteraction = false,
+    } = params;
+    const notification: Partial<PushNotificationMessage> = {
+      title: title,
+      body: body,
+      tag: tag || "OneUptime",
+      requireInteraction: requireInteraction,
+      data: {
+        type: "generic",
+      },
+    };
+
+    if (clickAction) {
+      notification.clickAction = clickAction;
+      notification.url = clickAction;
+      notification.data!["url"] = clickAction;
+    }
+
+    return PushNotificationUtil.applyDefaults(notification);
+  }
+
+  public static createMonitorProbeStatusNotification(params: {
+    title: string;
+    body: string;
+    tag: string;
+    monitorId: string;
+    monitorName: string;
+  }): PushNotificationMessage {
+    const { title, body, tag, monitorId, monitorName } = params;
+    return PushNotificationUtil.applyDefaults({
+      title: title,
+      body: body,
+      tag: tag,
+      requireInteraction: false,
+      data: {
+        type: "monitor-probe-status",
+        monitorId: monitorId,
+        monitorName: monitorName,
+      },
+    });
+  }
+
+  public static createMonitorCreatedNotification(params: {
+    monitorName: string;
+    monitorId: string;
+  }): PushNotificationMessage {
+    const { monitorName, monitorId } = params;
+    return PushNotificationUtil.applyDefaults({
+      title: "OneUptime: New Monitor Created",
+      body: `New monitor was created: ${monitorName}`,
+      tag: "monitor-created",
+      requireInteraction: false,
+      data: {
+        type: "monitor-created",
+        monitorId: monitorId,
+        monitorName: monitorName,
+      },
+    });
+  }
+
+  public static createOnCallPolicyAddedNotification(params: {
+    policyName: string;
+  }): PushNotificationMessage {
+    const { policyName } = params;
+    return PushNotificationUtil.applyDefaults({
+      title: "Added to On-Call Policy",
+      body: `You have been added to the on-call duty policy ${policyName}.`,
+      tag: "on-call-policy-added",
+      requireInteraction: false,
+      data: {
+        type: "on-call-policy-added",
+        policyName: policyName,
+      },
+    });
+  }
+
+  public static createOnCallPolicyRemovedNotification(params: {
+    policyName: string;
+  }): PushNotificationMessage {
+    const { policyName } = params;
+    return PushNotificationUtil.applyDefaults({
+      title: "Removed from On-Call Policy",
+      body: `You have been removed from the on-call duty policy ${policyName}.`,
+      tag: "on-call-policy-removed",
+      requireInteraction: false,
+      data: {
+        type: "on-call-policy-removed",
+        policyName: policyName,
+      },
+    });
+  }
+
+  public static createProbeDisconnectedNotification(params: {
+    probeName: string;
+  }): PushNotificationMessage {
+    const { probeName } = params;
+    return PushNotificationUtil.applyDefaults({
+      title: "OneUptime: Probe Disconnected",
+      body: `Your probe ${probeName} is disconnected. It was last seen 5 minutes ago.`,
+      tag: "probe-disconnected",
+      requireInteraction: false,
+      data: {
+        type: "probe-disconnected",
+        probeName: probeName,
+      },
+    });
+  }
+
+  public static createProbeStatusChangedNotification(params: {
+    probeName: string;
+    projectName: string;
+    connectionStatus: string;
+    clickAction?: string;
+  }): PushNotificationMessage {
+    const { probeName, projectName, connectionStatus, clickAction } = params;
+    const notification: Partial<PushNotificationMessage> = {
+      title: `Probe ${connectionStatus}: ${probeName}`,
+      body: `Probe ${probeName} is ${connectionStatus} in ${projectName}. Click to view details.`,
+      tag: "probe-status-changed",
+      requireInteraction: true,
+      data: {
+        type: "probe-status-changed",
+        probeName: probeName,
+        projectName: projectName,
+        connectionStatus: connectionStatus,
+      },
+    };
+
+    if (clickAction) {
+      notification.clickAction = clickAction;
+      notification.url = clickAction;
+      notification.data!["url"] = clickAction;
+    }
+
+    return PushNotificationUtil.applyDefaults(notification);
+  }
+}

Common/Server/Utils/Telemetry/Telemetry.ts

@@ -1,4 +1,3 @@
-import { Dictionary } from "lodash";
 import { JSONArray, JSONObject, JSONValue } from "../../../Types/JSON";
 import ObjectID from "../../../Types/ObjectID";
 import TelemetryType from "../../../Types/Telemetry/TelemetryType";
@@ -9,6 +8,7 @@ import logger from "../Logger";
 import MetricType from "../../../Models/DatabaseModels/MetricType";
 import MetricTypeService from "../../Services/MetricTypeService";
 import TelemetryService from "../../../Models/DatabaseModels/TelemetryService";
+import Dictionary from "../../../Types/Dictionary";
 
 export type AttributeType = string | number | boolean | null;
 

Common/Server/Utils/Workspace/Slack/Actions/Auth.ts

@@ -11,8 +11,8 @@ import SlackActionType from "./ActionTypes";
 import WorkspaceProjectAuthTokenService from "../../../../Services/WorkspaceProjectAuthTokenService";
 import logger from "../../../Logger";
 import { JSONArray, JSONObject } from "../../../../../Types/JSON";
-import { Dictionary } from "lodash";
 import CaptureSpan from "../../../Telemetry/CaptureSpan";
+import Dictionary from "../../../../../Types/Dictionary";
 
 export interface SlackAction {
   actionValue?: string | undefined;

Common/Tests/Server/Utils/AnalyticsDatabase/StatementGenerator.test.ts

@@ -295,7 +295,7 @@ describe("StatementGenerator", () => {
             (\n<columns-create-statement>
             )
             ENGINE = MergeTree
-        PARTITION BY (column_ObjectID)
+                PARTITION BY (column_ObjectID)
         
             PRIMARY KEY (${'column_ObjectID'})
             ORDER BY (${'column_ObjectID'})

Common/Tests/UI/Components/NavBar.test.tsx

@@ -1,9 +1,13 @@
-import Navbar, { ComponentProps } from "../../../UI/Components/Navbar/NavBar";
+import Navbar, {
+  ComponentProps,
+  NavItem,
+} from "../../../UI/Components/Navbar/NavBar";
 import { describe, expect, it } from "@jest/globals";
 import "@testing-library/jest-dom/extend-expect";
 import { render, screen } from "@testing-library/react";
 import React from "react";
-import { ReactElement } from "react-markdown/lib/react-markdown";
+import Route from "../../../Types/API/Route";
+import IconProp from "../../../Types/Icon/IconProp";
 
 describe("Navbar", () => {
   const defaultProps: ComponentProps = {
@@ -26,7 +30,12 @@ describe("Navbar", () => {
   });
 
   it("renders with a rightElement", () => {
-    const rightElement: ReactElement = <div>Right Element</div>;
+    const rightElement: NavItem = {
+      id: "test-right-element",
+      title: "Right Element",
+      icon: IconProp.User,
+      route: new Route("/test"),
+    };
     const customProps: ComponentProps = { ...defaultProps, rightElement };
     render(<Navbar {...customProps} />);
     expect(screen.getByText("Right Element")).toBeInTheDocument();

Common/Types/API/Route.ts

@@ -1,5 +1,5 @@
-import { Dictionary } from "lodash";
 import DatabaseProperty from "../Database/DatabaseProperty";
+import Dictionary from "../Dictionary";
 import BadDataException from "../Exception/BadDataException";
 import { JSONObject, ObjectType } from "../JSON";
 import { FindOperator } from "typeorm";

Common/Types/Monitor/ServerMonitor/ServerMonitorResponse.ts

@@ -16,4 +16,5 @@ export default interface ServerMonitorResponse {
   onlyCheckRequestReceivedAt: boolean;
   processes?: ServerProcess[] | undefined;
   failureCause?: string | undefined;
+  timeNow?: Date | undefined; // Time when the response was generated
 }

Common/Types/Probe/ProbeMonitorResponse.ts

@@ -27,4 +27,5 @@ export default interface ProbeMonitorResponse {
   customCodeMonitorResponse?: CustomCodeMonitorResponse | undefined;
   monitoredAt: Date;
   isTimeout?: boolean | undefined;
+  ingestedAt?: Date | undefined;
 }

Common/Types/PushNotification/PushNotificationMessage.ts

@@ -0,0 +1,18 @@
+interface PushNotificationMessage {
+  title: string;
+  body: string;
+  icon?: string;
+  badge?: string;
+  data?: { [key: string]: any };
+  tag?: string;
+  requireInteraction?: boolean;
+  actions?: Array<{
+    action: string;
+    title: string;
+    icon?: string;
+  }>;
+  clickAction?: string;
+  url?: string;
+}
+
+export default PushNotificationMessage;

Common/Types/PushNotification/PushNotificationRequest.ts

@@ -0,0 +1,22 @@
+interface PushNotificationRequest {
+  deviceTokens: string[];
+  message: {
+    title: string;
+    body: string;
+    icon?: string;
+    badge?: string;
+    data?: { [key: string]: any };
+    tag?: string;
+    requireInteraction?: boolean;
+    actions?: Array<{
+      action: string;
+      title: string;
+      icon?: string;
+    }>;
+    clickAction?: string;
+    url?: string;
+  };
+  deviceType: "web";
+}
+
+export default PushNotificationRequest;

Common/UI/Components/ActionButton/ActionButtonSchema.ts

@@ -9,6 +9,7 @@ interface ActionButtonSchema<T extends GenericObject> {
   buttonStyleType: ButtonStyleType;
   isLoading?: boolean | undefined;
   isVisible?: (item: T) => boolean | undefined;
+  hideOnMobile?: boolean | undefined;
   onClick: (
     item: T,
     onCompleteAction: VoidFunction,

Common/UI/Components/Banner/Banner.tsx

@@ -9,6 +9,7 @@ export interface ComponentProps {
   description: string;
   link?: URL | Route | undefined;
   openInNewTab?: boolean | undefined;
+  hideOnMobile?: boolean | undefined;
 }
 
 const Banner: FunctionComponent<ComponentProps> = (
@@ -32,7 +33,9 @@ const Banner: FunctionComponent<ComponentProps> = (
   };
 
   return (
-    <div className="flex border-gray-200 rounded-xl border-2 py-2.5 px-6 sm:px-3.5">
+    <div
+      className={`flex border-gray-200 rounded-xl border-2 py-2.5 px-6 sm:px-3.5${props.hideOnMobile ? " hidden md:flex" : ""}`}
+    >
       <p className="text-sm text-gray-400 hover:text-gray-500">
         {props.link && (
           <Link to={props.link} openInNewTab={props.openInNewTab}>