Add sitemap check [skip ci]

2026-04-06 00:32:12 +02:00 · 2020-06-03 18:13:06 +01:00
parent 10788149e4
commit d7d32ff128
10 changed files with 320 additions and 279 deletions
--- a/backend/backend/api/probe.js
+++ b/backend/backend/api/probe.js
@@ -15,7 +15,6 @@ const isAuthorizedProbe = require('../middlewares/probeAuthorization')
 const sendErrorResponse = require('../middlewares/response').sendErrorResponse;
 const sendItemResponse = require('../middlewares/response').sendItemResponse;
 const sendListResponse = require('../middlewares/response').sendListResponse;
-const sendEmptyResponse = require('../middlewares/response').sendEmptyResponse;
 const getUser = require('../middlewares/user').getUser;
 const { isAuthorized } = require('../middlewares/authorization');

@@ -80,7 +79,7 @@ router.post('/ping/:monitorId', isAuthorizedProbe, async function(
 ) {
    try {
        const { monitor, res, resp, type } = req.body;
-        let status;
+        let status, log;

        if (type === 'api' || type === 'url') {
            const validUp = await (monitor &&
@@ -130,17 +129,42 @@ router.post('/ping/:monitorId', isAuthorizedProbe, async function(
            resp && resp.lighthouseScanStatus
                ? resp.lighthouseScanStatus
                : null;
-        data.lighthouseScores =
-            resp && resp.lighthouseScores ? resp.lighthouseScores : null;
+        data.performance = resp && resp.performance ? resp.performance : null;
+        data.accessibility =
+            resp && resp.accessibility ? resp.accessibility : null;
+        data.bestPractices =
+            resp && resp.bestPractices ? resp.bestPractices : null;
+        data.seo = resp && resp.seo ? resp.seo : null;
+        data.pwa = resp && resp.pwa ? resp.pwa : null;
+        data.data = resp && resp.data ? resp.data : null;

        if (data.lighthouseScanStatus) {
-            await ProbeService.saveLighthouseScan(data);
+            if (data.lighthouseScanStatus === 'scanning') {
+                await MonitorService.updateOneBy(
+                    { _id: data.monitorId },
+                    {
+                        lighthouseScanStatus: data.lighthouseScanStatus,
+                    }
+                );
+            } else {
+                await MonitorService.updateOneBy(
+                    { _id: data.monitorId },
+                    {
+                        lighthouseScannedAt: Date.now(),
+                        lighthouseScanStatus: data.lighthouseScanStatus, // scanned || failed
+                        lighthouseScannedBy: data.probeId,
+                    }
+                );
+            }
+        } else {
+            if (data.data) {
+                log = await ProbeService.saveLighthouseLog(data);
+            } else {
+                log = await ProbeService.saveMonitorLog(data);
+            }
        }
-        if (!data.lighthouseScores) {
-            const log = await ProbeService.saveMonitorLog(data);
-            return sendItemResponse(req, response, log);
-        }
-        return sendEmptyResponse(req, response);
+
+        return sendItemResponse(req, response, log);
    } catch (error) {
        return sendErrorResponse(req, response, error);
    }
--- a/backend/backend/models/lighthouseLog.js
+++ b/backend/backend/models/lighthouseLog.js
@@ -0,0 +1,18 @@
+const mongoose = require('../config/db');
+
+const Schema = mongoose.Schema;
+const lighthouseLogSchema = new Schema({
+    monitorId: { type: String, ref: 'Monitor' }, // which monitor does this belong to.
+    probeId: { type: String, ref: 'Probe' }, // which probe does this belong to.
+    data: Object,
+    performance: Number,
+    accessibility: Number,
+    bestPractices: Number,
+    seo: Number,
+    pwa: Number,
+    createdAt: {
+        type: Date,
+        default: Date.now,
+    },
+});
+module.exports = mongoose.model('LighthouseLog', lighthouseLogSchema);
--- a/backend/backend/services/lighthouseLogService.js
+++ b/backend/backend/services/lighthouseLogService.js
@@ -0,0 +1,135 @@
+module.exports = {
+    create: async function(data) {
+        try {
+            const Log = new LighthouseLogModel();
+
+            Log.monitorId = data.monitorId;
+            Log.probeId = data.probeId;
+            Log.data = data.data;
+            Log.performance = data.performance;
+            Log.accessibility = data.accessibility;
+            Log.bestPractices = data.bestPractices;
+            Log.seo = data.seo;
+            Log.pwa = data.pwa;
+
+            const savedLog = await Log.save();
+
+            await this.sendLighthouseLog(savedLog);
+
+            if (data.probeId && data.monitorId)
+                await probeService.sendProbe(data.probeId, data.monitorId);
+
+            return savedLog;
+        } catch (error) {
+            ErrorService.log('lighthouseLogService.create', error);
+            throw error;
+        }
+    },
+
+    updateOneBy: async function(query, data) {
+        try {
+            if (!query) {
+                query = {};
+            }
+
+            const monitorLog = await LighthouseLogModel.findOneAndUpdate(
+                query,
+                { $set: data },
+                {
+                    new: true,
+                }
+            );
+
+            return monitorLog;
+        } catch (error) {
+            ErrorService.log('lighthouseLogService.updateOneBy', error);
+            throw error;
+        }
+    },
+
+    async findBy(query, limit, skip) {
+        try {
+            if (!skip) skip = 0;
+
+            if (!limit) limit = 0;
+
+            if (typeof skip === 'string') {
+                skip = parseInt(skip);
+            }
+
+            if (typeof limit === 'string') {
+                limit = parseInt(limit);
+            }
+
+            if (!query) {
+                query = {};
+            }
+
+            const monitorLogs = await LighthouseLogModel.find(query)
+                .sort([['createdAt', -1]])
+                .limit(limit)
+                .skip(skip)
+                .populate('probeId');
+
+            return monitorLogs;
+        } catch (error) {
+            ErrorService.log('lighthouseLogService.findBy', error);
+            throw error;
+        }
+    },
+
+    async findOneBy(query) {
+        try {
+            if (!query) {
+                query = {};
+            }
+
+            const monitorLog = await LighthouseLogModel.findOne(query).populate(
+                'probeId'
+            );
+
+            return monitorLog;
+        } catch (error) {
+            ErrorService.log('lighthouseLogService.findOneBy', error);
+            throw error;
+        }
+    },
+
+    async countBy(query) {
+        try {
+            if (!query) {
+                query = {};
+            }
+
+            const count = await LighthouseLogModel.countDocuments(query);
+
+            return count;
+        } catch (error) {
+            ErrorService.log('lighthouseLogService.countBy', error);
+            throw error;
+        }
+    },
+
+    async sendLighthouseLog(data) {
+        try {
+            const monitor = await MonitorService.findOneBy({
+                _id: data.monitorId,
+            });
+            if (monitor && monitor.projectId && monitor.projectId._id) {
+                await RealTimeService.updateLighthouseLog(
+                    data,
+                    monitor.projectId._id
+                );
+            }
+        } catch (error) {
+            ErrorService.log('lighthouseLogService.sendLighthouseLog', error);
+            throw error;
+        }
+    },
+};
+
+const LighthouseLogModel = require('../models/lighthouseLog');
+const MonitorService = require('./monitorService');
+const RealTimeService = require('./realTimeService');
+const probeService = require('./probeService');
+const ErrorService = require('./errorService');
--- a/backend/backend/services/probeService.js
+++ b/backend/backend/services/probeService.js
@@ -169,26 +169,10 @@ module.exports = {
        }
    },

-    saveLighthouseScan: async function(data) {
+    saveLighthouseLog: async function(data) {
        try {
-            if (data.lighthouseScores) {
-                await MonitorService.updateOneBy(
-                    { _id: data.monitorId },
-                    {
-                        lighthouseScannedAt: Date.now(),
-                        lighthouseScanStatus: data.lighthouseScanStatus, // scanned
-                        lighthouseScannedBy: data.probeId,
-                        lighthouseScores: data.lighthouseScores,
-                    }
-                );
-            } else {
-                await MonitorService.updateOneBy(
-                    { _id: data.monitorId },
-                    {
-                        lighthouseScanStatus: data.lighthouseScanStatus, // scanning || failed
-                    }
-                );
-            }
+            const log = await LighthouseLogService.create(data);
+            return log;
        } catch (error) {
            ErrorService.log('ProbeService.saveLighthouseScan', error);
            throw error;
@@ -1896,6 +1880,7 @@ const uuidv1 = require('uuid/v1');
 const MonitorService = require('./monitorService');
 const MonitorStatusService = require('./monitorStatusService');
 const MonitorLogService = require('./monitorLogService');
+const LighthouseLogService = require('./lighthouseLogService');
 const IncidentService = require('./incidentService');
 const IncidentTimelineService = require('./incidentTimelineService');
 const moment = require('moment');
--- a/backend/backend/services/realTimeService.js
+++ b/backend/backend/services/realTimeService.js
@@ -293,6 +293,30 @@ module.exports = {
        }
    },

+    updateLighthouseLog: async (data, projectId) => {
+        try {
+            if (!global || !global.io) {
+                return;
+            }
+
+            const project = await ProjectService.findOneBy({ _id: projectId });
+            const parentProjectId = project
+                ? project.parentProjectId
+                    ? project.parentProjectId._id
+                    : project._id
+                : projectId;
+
+            global.io.emit(`updateLighthouseLog-${parentProjectId}`, {
+                projectId,
+                monitorId: data.monitorId,
+                data,
+            });
+        } catch (error) {
+            ErrorService.log('realTimeService.updateLighthouseLog', error);
+            throw error;
+        }
+    },
+
    updateMonitorStatus: async (data, projectId) => {
        try {
            if (!global || !global.io) {
--- a/probe/Dockerfile
+++ b/probe/Dockerfile
@@ -9,7 +9,7 @@ FROM node:12.16.1
 ENV PRODUCTION=true
 ENV CHROME_PATH=/usr/bin/google-chrome

-# Install Chromium.
+# Install Chrome.
 RUN \
  wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
  echo "deb http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list && \
--- a/probe/package-lock.json
+++ b/probe/package-lock.json
@@ -363,16 +363,6 @@
                "concat-map": "0.0.1"
            }
        },
-        "buffer-crc32": {
-            "version": "0.2.13",
-            "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
-            "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI="
-        },
-        "buffer-from": {
-            "version": "1.1.1",
-            "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
-            "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A=="
-        },
        "builtin-modules": {
            "version": "3.1.0",
            "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.1.0.tgz",
@@ -474,54 +464,6 @@
                }
            }
        },
-        "chromium": {
-            "version": "2.1.2",
-            "resolved": "https://registry.npmjs.org/chromium/-/chromium-2.1.2.tgz",
-            "integrity": "sha512-JncmUwvumDbaWyIrU/1zMxTu89Zb0KIMdnBsXfm+lyTseE2ONmlIJC56cgmU6AxKLIhnDSI0/5TMf4Gg2ghv7A==",
-            "requires": {
-                "debug": "^4.1.0",
-                "extract-zip": "^1.6.7",
-                "got": "^7.1.0",
-                "tmp": "0.0.33",
-                "tunnel": "^0.0.6"
-            },
-            "dependencies": {
-                "debug": {
-                    "version": "4.1.1",
-                    "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
-                    "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
-                    "requires": {
-                        "ms": "^2.1.1"
-                    }
-                },
-                "got": {
-                    "version": "7.1.0",
-                    "resolved": "https://registry.npmjs.org/got/-/got-7.1.0.tgz",
-                    "integrity": "sha512-Y5WMo7xKKq1muPsxD+KmrR8DH5auG7fBdDVueZwETwV6VytKyU9OX/ddpq2/1hp1vIPvVb4T81dKQz3BivkNLw==",
-                    "requires": {
-                        "decompress-response": "^3.2.0",
-                        "duplexer3": "^0.1.4",
-                        "get-stream": "^3.0.0",
-                        "is-plain-obj": "^1.1.0",
-                        "is-retry-allowed": "^1.0.0",
-                        "is-stream": "^1.0.0",
-                        "isurl": "^1.0.0-alpha5",
-                        "lowercase-keys": "^1.0.0",
-                        "p-cancelable": "^0.3.0",
-                        "p-timeout": "^1.1.1",
-                        "safe-buffer": "^5.0.1",
-                        "timed-out": "^4.0.0",
-                        "url-parse-lax": "^1.0.0",
-                        "url-to-options": "^1.0.1"
-                    }
-                },
-                "ms": {
-                    "version": "2.1.2",
-                    "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
-                    "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
-                }
-            }
-        },
        "ci-info": {
            "version": "1.6.0",
            "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-1.6.0.tgz",
@@ -636,22 +578,16 @@
                "delayed-stream": "~1.0.0"
            }
        },
+        "commander": {
+            "version": "2.20.3",
+            "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
+            "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ=="
+        },
        "concat-map": {
            "version": "0.0.1",
            "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
            "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
        },
-        "concat-stream": {
-            "version": "1.6.2",
-            "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
-            "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
-            "requires": {
-                "buffer-from": "^1.0.0",
-                "inherits": "^2.0.3",
-                "readable-stream": "^2.2.2",
-                "typedarray": "^0.0.6"
-            }
-        },
        "configstore": {
            "version": "3.1.2",
            "resolved": "https://registry.npmjs.org/configstore/-/configstore-3.1.2.tgz",
@@ -814,14 +750,6 @@
            "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",
            "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA="
        },
-        "decompress-response": {
-            "version": "3.3.0",
-            "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz",
-            "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=",
-            "requires": {
-                "mimic-response": "^1.0.0"
-            }
-        },
        "deep-extend": {
            "version": "0.6.0",
            "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
@@ -1070,27 +998,6 @@
                "tmp": "^0.0.33"
            }
        },
-        "extract-zip": {
-            "version": "1.7.0",
-            "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-1.7.0.tgz",
-            "integrity": "sha512-xoh5G1W/PB0/27lXgMQyIhP5DSY/LhoCsOyZgb+6iMmRtCwVBo55uKaMoEYrDCKQhWvqEip5ZPKAc6eFNyf/MA==",
-            "requires": {
-                "concat-stream": "^1.6.2",
-                "debug": "^2.6.9",
-                "mkdirp": "^0.5.4",
-                "yauzl": "^2.10.0"
-            },
-            "dependencies": {
-                "debug": {
-                    "version": "2.6.9",
-                    "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
-                    "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
-                    "requires": {
-                        "ms": "2.0.0"
-                    }
-                }
-            }
-        },
        "extsprintf": {
            "version": "1.3.0",
            "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
@@ -1111,14 +1018,6 @@
            "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz",
            "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I="
        },
-        "fd-slicer": {
-            "version": "1.1.0",
-            "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz",
-            "integrity": "sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4=",
-            "requires": {
-                "pend": "~1.2.0"
-            }
-        },
        "figures": {
            "version": "2.0.0",
            "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz",
@@ -1304,19 +1203,6 @@
            "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
            "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
        },
-        "has-symbol-support-x": {
-            "version": "1.4.2",
-            "resolved": "https://registry.npmjs.org/has-symbol-support-x/-/has-symbol-support-x-1.4.2.tgz",
-            "integrity": "sha512-3ToOva++HaW+eCpgqZrCfN51IPB+7bJNVT6CUATzueB5Heb8o6Nam0V3HG5dlDvZU1Gn5QLcbahiKw/XVk5JJw=="
-        },
-        "has-to-string-tag-x": {
-            "version": "1.4.1",
-            "resolved": "https://registry.npmjs.org/has-to-string-tag-x/-/has-to-string-tag-x-1.4.1.tgz",
-            "integrity": "sha512-vdbKfmw+3LoOYVr+mtxHaX5a96+0f3DljYd8JOqvOLsf5mw2Otda2qCDT9qRqLAhrjyQ0h7ual5nOiASpsGNFw==",
-            "requires": {
-                "has-symbol-support-x": "^1.4.1"
-            }
-        },
        "he": {
            "version": "1.2.0",
            "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz",
@@ -1507,11 +1393,6 @@
            "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz",
            "integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8="
        },
-        "is-object": {
-            "version": "1.0.1",
-            "resolved": "https://registry.npmjs.org/is-object/-/is-object-1.0.1.tgz",
-            "integrity": "sha1-iVJojF7C/9awPsyF52ngKQMINHA="
-        },
        "is-path-inside": {
            "version": "1.0.1",
            "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-1.0.1.tgz",
@@ -1520,11 +1401,6 @@
                "path-is-inside": "^1.0.1"
            }
        },
-        "is-plain-obj": {
-            "version": "1.1.0",
-            "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz",
-            "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4="
-        },
        "is-redirect": {
            "version": "1.0.0",
            "resolved": "https://registry.npmjs.org/is-redirect/-/is-redirect-1.0.0.tgz",
@@ -1553,11 +1429,6 @@
                "is-docker": "^2.0.0"
            }
        },
-        "isarray": {
-            "version": "1.0.0",
-            "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
-            "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
-        },
        "isexe": {
            "version": "2.0.0",
            "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
@@ -1568,15 +1439,6 @@
            "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
            "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo="
        },
-        "isurl": {
-            "version": "1.0.0",
-            "resolved": "https://registry.npmjs.org/isurl/-/isurl-1.0.0.tgz",
-            "integrity": "sha512-1P/yWsxPlDtn7QeRD+ULKQPaIaN6yF368GZ2vDfv0AL0NwpStafjWCDDdn0k8wgFMWpVAqG7oJhxHnlud42i9w==",
-            "requires": {
-                "has-to-string-tag-x": "^1.2.0",
-                "is-object": "^1.0.1"
-            }
-        },
        "jpeg-js": {
            "version": "0.1.2",
            "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.1.2.tgz",
@@ -1961,11 +1823,6 @@
            "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz",
            "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ=="
        },
-        "mimic-response": {
-            "version": "1.0.1",
-            "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz",
-            "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ=="
-        },
        "minimatch": {
            "version": "3.0.4",
            "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
@@ -2127,11 +1984,6 @@
            "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
            "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ="
        },
-        "p-cancelable": {
-            "version": "0.3.0",
-            "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-0.3.0.tgz",
-            "integrity": "sha512-RVbZPLso8+jFeq1MfNvgXtCRED2raz/dKpacfTNxsx6pLEpEomM7gah6VeHSYV3+vo0OAi4MkArtQcWWXuQoyw=="
-        },
        "p-finally": {
            "version": "1.0.0",
            "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
@@ -2155,14 +2007,6 @@
                "p-limit": "^2.2.0"
            }
        },
-        "p-timeout": {
-            "version": "1.2.1",
-            "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-1.2.1.tgz",
-            "integrity": "sha1-XrOzU7f86Z8QGhA4iAuwVOu+o4Y=",
-            "requires": {
-                "p-finally": "^1.0.0"
-            }
-        },
        "p-try": {
            "version": "2.2.0",
            "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
@@ -2241,11 +2085,6 @@
                "through": "~2.3"
            }
        },
-        "pend": {
-            "version": "1.2.0",
-            "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
-            "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA="
-        },
        "performance-now": {
            "version": "2.1.0",
            "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
@@ -2270,11 +2109,6 @@
            "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-1.0.4.tgz",
            "integrity": "sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw="
        },
-        "process-nextick-args": {
-            "version": "2.0.1",
-            "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
-            "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
-        },
        "proxy-addr": {
            "version": "2.0.5",
            "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.5.tgz",
@@ -2371,20 +2205,6 @@
                }
            }
        },
-        "readable-stream": {
-            "version": "2.3.7",
-            "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
-            "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
-            "requires": {
-                "core-util-is": "~1.0.0",
-                "inherits": "~2.0.3",
-                "isarray": "~1.0.0",
-                "process-nextick-args": "~2.0.0",
-                "safe-buffer": "~5.1.1",
-                "string_decoder": "~1.1.1",
-                "util-deprecate": "~1.0.1"
-            }
-        },
        "registry-auth-token": {
            "version": "3.4.0",
            "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-3.4.0.tgz",
@@ -2513,6 +2333,11 @@
            "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
            "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
        },
+        "sax": {
+            "version": "1.2.4",
+            "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
+            "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
+        },
        "semver": {
            "version": "5.7.1",
            "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
@@ -2614,6 +2439,27 @@
            "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz",
            "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA=="
        },
+        "sitemap-stream-parser": {
+            "version": "1.7.0",
+            "resolved": "https://registry.npmjs.org/sitemap-stream-parser/-/sitemap-stream-parser-1.7.0.tgz",
+            "integrity": "sha512-aGNRTohb0G9uhrS04C6NlTBRdCK7XzWpWEXKN5cUjiYkhbea+g6FWm3Js24Kw8EM+ryeZLM5fCPnPEEufwW4Hw==",
+            "requires": {
+                "async": "^2.6.1",
+                "commander": "^2.15.1",
+                "request": "^2.87.0",
+                "sax": "^1.2.4"
+            },
+            "dependencies": {
+                "async": {
+                    "version": "2.6.3",
+                    "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz",
+                    "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==",
+                    "requires": {
+                        "lodash": "^4.17.14"
+                    }
+                }
+            }
+        },
        "source-map": {
            "version": "0.5.7",
            "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz",
@@ -2690,14 +2536,6 @@
                "strip-ansi": "^4.0.0"
            }
        },
-        "string_decoder": {
-            "version": "1.1.1",
-            "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
-            "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
-            "requires": {
-                "safe-buffer": "~5.1.0"
-            }
-        },
        "strip-ansi": {
            "version": "4.0.0",
            "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
@@ -2782,11 +2620,6 @@
                }
            }
        },
-        "tunnel": {
-            "version": "0.0.6",
-            "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz",
-            "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg=="
-        },
        "tunnel-agent": {
            "version": "0.6.0",
            "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
@@ -2809,11 +2642,6 @@
                "mime-types": "~2.1.24"
            }
        },
-        "typedarray": {
-            "version": "0.0.6",
-            "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
-            "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c="
-        },
        "tz-offset": {
            "version": "0.0.1",
            "resolved": "https://registry.npmjs.org/tz-offset/-/tz-offset-0.0.1.tgz",
@@ -2880,16 +2708,6 @@
                "prepend-http": "^1.0.1"
            }
        },
-        "url-to-options": {
-            "version": "1.0.1",
-            "resolved": "https://registry.npmjs.org/url-to-options/-/url-to-options-1.0.1.tgz",
-            "integrity": "sha1-FQWgOiiaSMvXpDTvuu7FBV9WM6k="
-        },
-        "util-deprecate": {
-            "version": "1.0.2",
-            "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
-            "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
-        },
        "utils-merge": {
            "version": "1.0.1",
            "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
@@ -3169,15 +2987,6 @@
                "camelcase": "^5.0.0",
                "decamelize": "^1.2.0"
            }
-        },
-        "yauzl": {
-            "version": "2.10.0",
-            "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz",
-            "integrity": "sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk=",
-            "requires": {
-                "buffer-crc32": "~0.2.3",
-                "fd-slicer": "~1.1.0"
-            }
        }
    }
 }
--- a/probe/package.json
+++ b/probe/package.json
@@ -31,6 +31,7 @@
        "moment": "^2.24.0",
        "node-cron": "^2.0.3",
        "node-fetch": "^2.6.0",
+        "sitemap-stream-parser": "^1.7.0",
        "winston": "^2.4.0",
        "winston-slack-transport": "^2.0.0"
    },
--- a/probe/utils/lighthouse.js
+++ b/probe/utils/lighthouse.js
@@ -18,7 +18,8 @@ function launchChromeAndRunLighthouse(
 process.on('message', url => {
    launchChromeAndRunLighthouse(url)
        .then(results => {
-            const scores = {
+            const result = {
+                data: { url },
                performance: Math.ceil(
                    results.categories.performance.score * 100
                ),
@@ -31,9 +32,10 @@ process.on('message', url => {
                seo: Math.ceil(results.categories.seo.score * 100),
                pwa: Math.ceil(results.categories.pwa.score * 100),
            };
-            process.send(scores);
+            process.send(result);
        })
        .catch(error => {
+            process.send({ data: { url }, error });
            ErrorService.log('launchChromeAndRunLighthouse', error);
        });
 });
--- a/probe/workers/urlMonitors.js
+++ b/probe/workers/urlMonitors.js
@@ -5,8 +5,7 @@ const fetch = require('node-fetch');
 const sslCert = require('get-ssl-certificate');
 const { fork } = require('child_process');
 const moment = require('moment');
-
-const minuteStartTime = Math.floor(Math.random() * 50);
+const sitemap = require('sitemap-stream-parser');

 // it collects all monitors then ping them one by one to store their response
 // checks if the website of the url in the monitors is up or down
@@ -31,10 +30,50 @@ module.exports = {
                        (!monitor.lighthouseScanStatus ||
                            monitor.lighthouseScanStatus !== 'scanning')
                    ) {
-                        resp.lighthouseScanStatus = 'scanning';
-                        setTimeout(() => {
-                            lighthouseFetch(monitor);
-                        }, minuteStartTime * 1000);
+                        await ApiService.ping(monitor._id, {
+                            monitor,
+                            resp: { lighthouseScanStatus: 'scanning' },
+                        });
+
+                        const urlObject = new URL(monitor.data.url);
+                        const sites = [];
+
+                        sitemap.parseSitemaps(
+                            `${urlObject.origin}/sitemap.xml`,
+                            url => {
+                                sites.push(url);
+                            },
+                            async err => {
+                                if (err || sites.length === 0)
+                                    sites.push(monitor.data.url);
+
+                                let resp = {};
+                                for (const url of sites) {
+                                    try {
+                                        resp = await lighthouseFetch(
+                                            monitor,
+                                            url
+                                        );
+
+                                        await ApiService.ping(monitor._id, {
+                                            monitor,
+                                            resp,
+                                        });
+                                    } catch (error) {
+                                        resp = error;
+                                        ErrorService.log(
+                                            'lighthouseFetch',
+                                            error.error
+                                        );
+                                    }
+                                }
+
+                                await ApiService.ping(monitor._id, {
+                                    monitor,
+                                    resp: { lighthouseScanStatus: 'scanned' },
+                                });
+                            }
+                        );
                    }

                    await ApiService.ping(monitor._id, {
@@ -92,25 +131,29 @@ const pingfetch = async url => {
    return { res, resp };
 };

-const lighthouseFetch = monitor => {
-    const lighthouseWorker = fork('./utils/lighthouse');
-    const handler = setTimeout(async () => {
-        await processLighthouseScan('failed');
-    }, 30000);
+const lighthouseFetch = (monitor, url) => {
+    return new Promise((resolve, reject) => {
+        const lighthouseWorker = fork('./utils/lighthouse');
+        const timeoutHandler = setTimeout(async () => {
+            await processLighthouseScan({
+                data: { url },
+                error: { message: 'TIMEOUT' },
+            });
+        }, 30000);

-    lighthouseWorker.send(monitor.data.url);
-    lighthouseWorker.on('message', async scores => {
-        await processLighthouseScan('scanned', scores);
-    });
-
-    async function processLighthouseScan(status, scores) {
-        clearTimeout(handler);
-        lighthouseWorker.removeAllListeners();
-
-        const resp = { lighthouseScanStatus: status, lighthouseScores: scores };
-        await ApiService.ping(monitor._id, {
-            monitor,
-            resp,
+        lighthouseWorker.send(url);
+        lighthouseWorker.on('message', async result => {
+            await processLighthouseScan(result);
        });
-    }
+
+        async function processLighthouseScan(result) {
+            clearTimeout(timeoutHandler);
+            lighthouseWorker.removeAllListeners();
+            if (result.error) {
+                reject({ status: 'failed', ...result });
+            } else {
+                resolve({ status: 'scanned', ...result });
+            }
+        }
+    });
 };