From d7d32ff1283c77f3ccc406cf49d7591d599d6e66 Mon Sep 17 00:00:00 2001 From: Ibukun Dairo Date: Wed, 3 Jun 2020 18:13:06 +0100 Subject: [PATCH] Add sitemap check [skip ci] --- backend/backend/api/probe.js | 44 ++- backend/backend/models/lighthouseLog.js | 18 ++ .../backend/services/lighthouseLogService.js | 135 ++++++++++ backend/backend/services/probeService.js | 23 +- backend/backend/services/realTimeService.js | 24 ++ probe/Dockerfile | 2 +- probe/package-lock.json | 253 +++--------------- probe/package.json | 1 + probe/utils/lighthouse.js | 6 +- probe/workers/urlMonitors.js | 93 +++++-- 10 files changed, 320 insertions(+), 279 deletions(-) create mode 100644 backend/backend/models/lighthouseLog.js create mode 100644 backend/backend/services/lighthouseLogService.js diff --git a/backend/backend/api/probe.js b/backend/backend/api/probe.js index 6f2b0c923d..9fcc7651ab 100755 --- a/backend/backend/api/probe.js +++ b/backend/backend/api/probe.js @@ -15,7 +15,6 @@ const isAuthorizedProbe = require('../middlewares/probeAuthorization') const sendErrorResponse = require('../middlewares/response').sendErrorResponse; const sendItemResponse = require('../middlewares/response').sendItemResponse; const sendListResponse = require('../middlewares/response').sendListResponse; -const sendEmptyResponse = require('../middlewares/response').sendEmptyResponse; const getUser = require('../middlewares/user').getUser; const { isAuthorized } = require('../middlewares/authorization'); @@ -80,7 +79,7 @@ router.post('/ping/:monitorId', isAuthorizedProbe, async function( ) { try { const { monitor, res, resp, type } = req.body; - let status; + let status, log; if (type === 'api' || type === 'url') { const validUp = await (monitor && @@ -130,17 +129,42 @@ router.post('/ping/:monitorId', isAuthorizedProbe, async function( resp && resp.lighthouseScanStatus ? resp.lighthouseScanStatus : null; - data.lighthouseScores = - resp && resp.lighthouseScores ? resp.lighthouseScores : null; + data.performance = resp && resp.performance ? resp.performance : null; + data.accessibility = + resp && resp.accessibility ? resp.accessibility : null; + data.bestPractices = + resp && resp.bestPractices ? resp.bestPractices : null; + data.seo = resp && resp.seo ? resp.seo : null; + data.pwa = resp && resp.pwa ? resp.pwa : null; + data.data = resp && resp.data ? resp.data : null; if (data.lighthouseScanStatus) { - await ProbeService.saveLighthouseScan(data); + if (data.lighthouseScanStatus === 'scanning') { + await MonitorService.updateOneBy( + { _id: data.monitorId }, + { + lighthouseScanStatus: data.lighthouseScanStatus, + } + ); + } else { + await MonitorService.updateOneBy( + { _id: data.monitorId }, + { + lighthouseScannedAt: Date.now(), + lighthouseScanStatus: data.lighthouseScanStatus, // scanned || failed + lighthouseScannedBy: data.probeId, + } + ); + } + } else { + if (data.data) { + log = await ProbeService.saveLighthouseLog(data); + } else { + log = await ProbeService.saveMonitorLog(data); + } } - if (!data.lighthouseScores) { - const log = await ProbeService.saveMonitorLog(data); - return sendItemResponse(req, response, log); - } - return sendEmptyResponse(req, response); + + return sendItemResponse(req, response, log); } catch (error) { return sendErrorResponse(req, response, error); } diff --git a/backend/backend/models/lighthouseLog.js b/backend/backend/models/lighthouseLog.js new file mode 100644 index 0000000000..43ac95d8a9 --- /dev/null +++ b/backend/backend/models/lighthouseLog.js @@ -0,0 +1,18 @@ +const mongoose = require('../config/db'); + +const Schema = mongoose.Schema; +const lighthouseLogSchema = new Schema({ + monitorId: { type: String, ref: 'Monitor' }, // which monitor does this belong to. + probeId: { type: String, ref: 'Probe' }, // which probe does this belong to. + data: Object, + performance: Number, + accessibility: Number, + bestPractices: Number, + seo: Number, + pwa: Number, + createdAt: { + type: Date, + default: Date.now, + }, +}); +module.exports = mongoose.model('LighthouseLog', lighthouseLogSchema); diff --git a/backend/backend/services/lighthouseLogService.js b/backend/backend/services/lighthouseLogService.js new file mode 100644 index 0000000000..fa8a240298 --- /dev/null +++ b/backend/backend/services/lighthouseLogService.js @@ -0,0 +1,135 @@ +module.exports = { + create: async function(data) { + try { + const Log = new LighthouseLogModel(); + + Log.monitorId = data.monitorId; + Log.probeId = data.probeId; + Log.data = data.data; + Log.performance = data.performance; + Log.accessibility = data.accessibility; + Log.bestPractices = data.bestPractices; + Log.seo = data.seo; + Log.pwa = data.pwa; + + const savedLog = await Log.save(); + + await this.sendLighthouseLog(savedLog); + + if (data.probeId && data.monitorId) + await probeService.sendProbe(data.probeId, data.monitorId); + + return savedLog; + } catch (error) { + ErrorService.log('lighthouseLogService.create', error); + throw error; + } + }, + + updateOneBy: async function(query, data) { + try { + if (!query) { + query = {}; + } + + const monitorLog = await LighthouseLogModel.findOneAndUpdate( + query, + { $set: data }, + { + new: true, + } + ); + + return monitorLog; + } catch (error) { + ErrorService.log('lighthouseLogService.updateOneBy', error); + throw error; + } + }, + + async findBy(query, limit, skip) { + try { + if (!skip) skip = 0; + + if (!limit) limit = 0; + + if (typeof skip === 'string') { + skip = parseInt(skip); + } + + if (typeof limit === 'string') { + limit = parseInt(limit); + } + + if (!query) { + query = {}; + } + + const monitorLogs = await LighthouseLogModel.find(query) + .sort([['createdAt', -1]]) + .limit(limit) + .skip(skip) + .populate('probeId'); + + return monitorLogs; + } catch (error) { + ErrorService.log('lighthouseLogService.findBy', error); + throw error; + } + }, + + async findOneBy(query) { + try { + if (!query) { + query = {}; + } + + const monitorLog = await LighthouseLogModel.findOne(query).populate( + 'probeId' + ); + + return monitorLog; + } catch (error) { + ErrorService.log('lighthouseLogService.findOneBy', error); + throw error; + } + }, + + async countBy(query) { + try { + if (!query) { + query = {}; + } + + const count = await LighthouseLogModel.countDocuments(query); + + return count; + } catch (error) { + ErrorService.log('lighthouseLogService.countBy', error); + throw error; + } + }, + + async sendLighthouseLog(data) { + try { + const monitor = await MonitorService.findOneBy({ + _id: data.monitorId, + }); + if (monitor && monitor.projectId && monitor.projectId._id) { + await RealTimeService.updateLighthouseLog( + data, + monitor.projectId._id + ); + } + } catch (error) { + ErrorService.log('lighthouseLogService.sendLighthouseLog', error); + throw error; + } + }, +}; + +const LighthouseLogModel = require('../models/lighthouseLog'); +const MonitorService = require('./monitorService'); +const RealTimeService = require('./realTimeService'); +const probeService = require('./probeService'); +const ErrorService = require('./errorService'); diff --git a/backend/backend/services/probeService.js b/backend/backend/services/probeService.js index 1a9dec983f..377d59db0d 100755 --- a/backend/backend/services/probeService.js +++ b/backend/backend/services/probeService.js @@ -169,26 +169,10 @@ module.exports = { } }, - saveLighthouseScan: async function(data) { + saveLighthouseLog: async function(data) { try { - if (data.lighthouseScores) { - await MonitorService.updateOneBy( - { _id: data.monitorId }, - { - lighthouseScannedAt: Date.now(), - lighthouseScanStatus: data.lighthouseScanStatus, // scanned - lighthouseScannedBy: data.probeId, - lighthouseScores: data.lighthouseScores, - } - ); - } else { - await MonitorService.updateOneBy( - { _id: data.monitorId }, - { - lighthouseScanStatus: data.lighthouseScanStatus, // scanning || failed - } - ); - } + const log = await LighthouseLogService.create(data); + return log; } catch (error) { ErrorService.log('ProbeService.saveLighthouseScan', error); throw error; @@ -1896,6 +1880,7 @@ const uuidv1 = require('uuid/v1'); const MonitorService = require('./monitorService'); const MonitorStatusService = require('./monitorStatusService'); const MonitorLogService = require('./monitorLogService'); +const LighthouseLogService = require('./lighthouseLogService'); const IncidentService = require('./incidentService'); const IncidentTimelineService = require('./incidentTimelineService'); const moment = require('moment'); diff --git a/backend/backend/services/realTimeService.js b/backend/backend/services/realTimeService.js index 43177c0c18..16757972ff 100755 --- a/backend/backend/services/realTimeService.js +++ b/backend/backend/services/realTimeService.js @@ -293,6 +293,30 @@ module.exports = { } }, + updateLighthouseLog: async (data, projectId) => { + try { + if (!global || !global.io) { + return; + } + + const project = await ProjectService.findOneBy({ _id: projectId }); + const parentProjectId = project + ? project.parentProjectId + ? project.parentProjectId._id + : project._id + : projectId; + + global.io.emit(`updateLighthouseLog-${parentProjectId}`, { + projectId, + monitorId: data.monitorId, + data, + }); + } catch (error) { + ErrorService.log('realTimeService.updateLighthouseLog', error); + throw error; + } + }, + updateMonitorStatus: async (data, projectId) => { try { if (!global || !global.io) { diff --git a/probe/Dockerfile b/probe/Dockerfile index fa36f01f21..5217eb01ce 100755 --- a/probe/Dockerfile +++ b/probe/Dockerfile @@ -9,7 +9,7 @@ FROM node:12.16.1 ENV PRODUCTION=true ENV CHROME_PATH=/usr/bin/google-chrome -# Install Chromium. +# Install Chrome. RUN \ wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \ echo "deb http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list && \ diff --git a/probe/package-lock.json b/probe/package-lock.json index a6792b2114..df62c2e630 100644 --- a/probe/package-lock.json +++ b/probe/package-lock.json @@ -363,16 +363,6 @@ "concat-map": "0.0.1" } }, - "buffer-crc32": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" - }, - "buffer-from": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" - }, "builtin-modules": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.1.0.tgz", @@ -474,54 +464,6 @@ } } }, - "chromium": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/chromium/-/chromium-2.1.2.tgz", - "integrity": "sha512-JncmUwvumDbaWyIrU/1zMxTu89Zb0KIMdnBsXfm+lyTseE2ONmlIJC56cgmU6AxKLIhnDSI0/5TMf4Gg2ghv7A==", - "requires": { - "debug": "^4.1.0", - "extract-zip": "^1.6.7", - "got": "^7.1.0", - "tmp": "0.0.33", - "tunnel": "^0.0.6" - }, - "dependencies": { - "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", - "requires": { - "ms": "^2.1.1" - } - }, - "got": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/got/-/got-7.1.0.tgz", - "integrity": "sha512-Y5WMo7xKKq1muPsxD+KmrR8DH5auG7fBdDVueZwETwV6VytKyU9OX/ddpq2/1hp1vIPvVb4T81dKQz3BivkNLw==", - "requires": { - "decompress-response": "^3.2.0", - "duplexer3": "^0.1.4", - "get-stream": "^3.0.0", - "is-plain-obj": "^1.1.0", - "is-retry-allowed": "^1.0.0", - "is-stream": "^1.0.0", - "isurl": "^1.0.0-alpha5", - "lowercase-keys": "^1.0.0", - "p-cancelable": "^0.3.0", - "p-timeout": "^1.1.1", - "safe-buffer": "^5.0.1", - "timed-out": "^4.0.0", - "url-parse-lax": "^1.0.0", - "url-to-options": "^1.0.1" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - } - } - }, "ci-info": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-1.6.0.tgz", @@ -636,22 +578,16 @@ "delayed-stream": "~1.0.0" } }, + "commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, - "concat-stream": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", - "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", - "requires": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^2.2.2", - "typedarray": "^0.0.6" - } - }, "configstore": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/configstore/-/configstore-3.1.2.tgz", @@ -814,14 +750,6 @@ "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" }, - "decompress-response": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", - "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=", - "requires": { - "mimic-response": "^1.0.0" - } - }, "deep-extend": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", @@ -1070,27 +998,6 @@ "tmp": "^0.0.33" } }, - "extract-zip": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-1.7.0.tgz", - "integrity": "sha512-xoh5G1W/PB0/27lXgMQyIhP5DSY/LhoCsOyZgb+6iMmRtCwVBo55uKaMoEYrDCKQhWvqEip5ZPKAc6eFNyf/MA==", - "requires": { - "concat-stream": "^1.6.2", - "debug": "^2.6.9", - "mkdirp": "^0.5.4", - "yauzl": "^2.10.0" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - } - } - }, "extsprintf": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", @@ -1111,14 +1018,6 @@ "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" }, - "fd-slicer": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", - "integrity": "sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4=", - "requires": { - "pend": "~1.2.0" - } - }, "figures": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", @@ -1304,19 +1203,6 @@ "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" }, - "has-symbol-support-x": { - "version": "1.4.2", - "resolved": "https://registry.npmjs.org/has-symbol-support-x/-/has-symbol-support-x-1.4.2.tgz", - "integrity": "sha512-3ToOva++HaW+eCpgqZrCfN51IPB+7bJNVT6CUATzueB5Heb8o6Nam0V3HG5dlDvZU1Gn5QLcbahiKw/XVk5JJw==" - }, - "has-to-string-tag-x": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/has-to-string-tag-x/-/has-to-string-tag-x-1.4.1.tgz", - "integrity": "sha512-vdbKfmw+3LoOYVr+mtxHaX5a96+0f3DljYd8JOqvOLsf5mw2Otda2qCDT9qRqLAhrjyQ0h7ual5nOiASpsGNFw==", - "requires": { - "has-symbol-support-x": "^1.4.1" - } - }, "he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", @@ -1507,11 +1393,6 @@ "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", "integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8=" }, - "is-object": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-object/-/is-object-1.0.1.tgz", - "integrity": "sha1-iVJojF7C/9awPsyF52ngKQMINHA=" - }, "is-path-inside": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-1.0.1.tgz", @@ -1520,11 +1401,6 @@ "path-is-inside": "^1.0.1" } }, - "is-plain-obj": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", - "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=" - }, "is-redirect": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-redirect/-/is-redirect-1.0.0.tgz", @@ -1553,11 +1429,6 @@ "is-docker": "^2.0.0" } }, - "isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -1568,15 +1439,6 @@ "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, - "isurl": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isurl/-/isurl-1.0.0.tgz", - "integrity": "sha512-1P/yWsxPlDtn7QeRD+ULKQPaIaN6yF368GZ2vDfv0AL0NwpStafjWCDDdn0k8wgFMWpVAqG7oJhxHnlud42i9w==", - "requires": { - "has-to-string-tag-x": "^1.2.0", - "is-object": "^1.0.1" - } - }, "jpeg-js": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.1.2.tgz", @@ -1961,11 +1823,6 @@ "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==" }, - "mimic-response": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", - "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==" - }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", @@ -2127,11 +1984,6 @@ "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=" }, - "p-cancelable": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-0.3.0.tgz", - "integrity": "sha512-RVbZPLso8+jFeq1MfNvgXtCRED2raz/dKpacfTNxsx6pLEpEomM7gah6VeHSYV3+vo0OAi4MkArtQcWWXuQoyw==" - }, "p-finally": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", @@ -2155,14 +2007,6 @@ "p-limit": "^2.2.0" } }, - "p-timeout": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-1.2.1.tgz", - "integrity": "sha1-XrOzU7f86Z8QGhA4iAuwVOu+o4Y=", - "requires": { - "p-finally": "^1.0.0" - } - }, "p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", @@ -2241,11 +2085,6 @@ "through": "~2.3" } }, - "pend": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", - "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=" - }, "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", @@ -2270,11 +2109,6 @@ "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-1.0.4.tgz", "integrity": "sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw=" }, - "process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" - }, "proxy-addr": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.5.tgz", @@ -2371,20 +2205,6 @@ } } }, - "readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, "registry-auth-token": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-3.4.0.tgz", @@ -2513,6 +2333,11 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "sax": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" + }, "semver": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", @@ -2614,6 +2439,27 @@ "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz", "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==" }, + "sitemap-stream-parser": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/sitemap-stream-parser/-/sitemap-stream-parser-1.7.0.tgz", + "integrity": "sha512-aGNRTohb0G9uhrS04C6NlTBRdCK7XzWpWEXKN5cUjiYkhbea+g6FWm3Js24Kw8EM+ryeZLM5fCPnPEEufwW4Hw==", + "requires": { + "async": "^2.6.1", + "commander": "^2.15.1", + "request": "^2.87.0", + "sax": "^1.2.4" + }, + "dependencies": { + "async": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz", + "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==", + "requires": { + "lodash": "^4.17.14" + } + } + } + }, "source-map": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", @@ -2690,14 +2536,6 @@ "strip-ansi": "^4.0.0" } }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } - }, "strip-ansi": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", @@ -2782,11 +2620,6 @@ } } }, - "tunnel": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", - "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==" - }, "tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", @@ -2809,11 +2642,6 @@ "mime-types": "~2.1.24" } }, - "typedarray": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" - }, "tz-offset": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/tz-offset/-/tz-offset-0.0.1.tgz", @@ -2880,16 +2708,6 @@ "prepend-http": "^1.0.1" } }, - "url-to-options": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/url-to-options/-/url-to-options-1.0.1.tgz", - "integrity": "sha1-FQWgOiiaSMvXpDTvuu7FBV9WM6k=" - }, - "util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" - }, "utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", @@ -3169,15 +2987,6 @@ "camelcase": "^5.0.0", "decamelize": "^1.2.0" } - }, - "yauzl": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", - "integrity": "sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk=", - "requires": { - "buffer-crc32": "~0.2.3", - "fd-slicer": "~1.1.0" - } } } } diff --git a/probe/package.json b/probe/package.json index 5c900e6ec2..ea4d9259b2 100755 --- a/probe/package.json +++ b/probe/package.json @@ -31,6 +31,7 @@ "moment": "^2.24.0", "node-cron": "^2.0.3", "node-fetch": "^2.6.0", + "sitemap-stream-parser": "^1.7.0", "winston": "^2.4.0", "winston-slack-transport": "^2.0.0" }, diff --git a/probe/utils/lighthouse.js b/probe/utils/lighthouse.js index c6d631152e..3c50464c05 100644 --- a/probe/utils/lighthouse.js +++ b/probe/utils/lighthouse.js @@ -18,7 +18,8 @@ function launchChromeAndRunLighthouse( process.on('message', url => { launchChromeAndRunLighthouse(url) .then(results => { - const scores = { + const result = { + data: { url }, performance: Math.ceil( results.categories.performance.score * 100 ), @@ -31,9 +32,10 @@ process.on('message', url => { seo: Math.ceil(results.categories.seo.score * 100), pwa: Math.ceil(results.categories.pwa.score * 100), }; - process.send(scores); + process.send(result); }) .catch(error => { + process.send({ data: { url }, error }); ErrorService.log('launchChromeAndRunLighthouse', error); }); }); diff --git a/probe/workers/urlMonitors.js b/probe/workers/urlMonitors.js index 8a0195a189..eeeacb5c0f 100755 --- a/probe/workers/urlMonitors.js +++ b/probe/workers/urlMonitors.js @@ -5,8 +5,7 @@ const fetch = require('node-fetch'); const sslCert = require('get-ssl-certificate'); const { fork } = require('child_process'); const moment = require('moment'); - -const minuteStartTime = Math.floor(Math.random() * 50); +const sitemap = require('sitemap-stream-parser'); // it collects all monitors then ping them one by one to store their response // checks if the website of the url in the monitors is up or down @@ -31,10 +30,50 @@ module.exports = { (!monitor.lighthouseScanStatus || monitor.lighthouseScanStatus !== 'scanning') ) { - resp.lighthouseScanStatus = 'scanning'; - setTimeout(() => { - lighthouseFetch(monitor); - }, minuteStartTime * 1000); + await ApiService.ping(monitor._id, { + monitor, + resp: { lighthouseScanStatus: 'scanning' }, + }); + + const urlObject = new URL(monitor.data.url); + const sites = []; + + sitemap.parseSitemaps( + `${urlObject.origin}/sitemap.xml`, + url => { + sites.push(url); + }, + async err => { + if (err || sites.length === 0) + sites.push(monitor.data.url); + + let resp = {}; + for (const url of sites) { + try { + resp = await lighthouseFetch( + monitor, + url + ); + + await ApiService.ping(monitor._id, { + monitor, + resp, + }); + } catch (error) { + resp = error; + ErrorService.log( + 'lighthouseFetch', + error.error + ); + } + } + + await ApiService.ping(monitor._id, { + monitor, + resp: { lighthouseScanStatus: 'scanned' }, + }); + } + ); } await ApiService.ping(monitor._id, { @@ -92,25 +131,29 @@ const pingfetch = async url => { return { res, resp }; }; -const lighthouseFetch = monitor => { - const lighthouseWorker = fork('./utils/lighthouse'); - const handler = setTimeout(async () => { - await processLighthouseScan('failed'); - }, 30000); +const lighthouseFetch = (monitor, url) => { + return new Promise((resolve, reject) => { + const lighthouseWorker = fork('./utils/lighthouse'); + const timeoutHandler = setTimeout(async () => { + await processLighthouseScan({ + data: { url }, + error: { message: 'TIMEOUT' }, + }); + }, 30000); - lighthouseWorker.send(monitor.data.url); - lighthouseWorker.on('message', async scores => { - await processLighthouseScan('scanned', scores); - }); - - async function processLighthouseScan(status, scores) { - clearTimeout(handler); - lighthouseWorker.removeAllListeners(); - - const resp = { lighthouseScanStatus: status, lighthouseScores: scores }; - await ApiService.ping(monitor._id, { - monitor, - resp, + lighthouseWorker.send(url); + lighthouseWorker.on('message', async result => { + await processLighthouseScan(result); }); - } + + async function processLighthouseScan(result) { + clearTimeout(timeoutHandler); + lighthouseWorker.removeAllListeners(); + if (result.error) { + reject({ status: 'failed', ...result }); + } else { + resolve({ status: 'scanned', ...result }); + } + } + }); };