Skip to content

Commit eae11f0

Browse files
simlarsensimlarsen
committed
refactor: Enhance probe initialization by adding worker management and improving logging
1 parent 065deeb commit eae11f0

File tree

2 files changed

+71
-47
lines changed

2 files changed

+71
-47
lines changed

‎Probe/Index.ts

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,14 @@
1-
import { PORT } from "./Config";
1+
import {
2+
PORT,
3+
PROBE_MONITOR_RETRY_LIMIT,
4+
PROBE_MONITORING_WORKERS,
5+
} from "./Config";
26
import "./Jobs/Alive";
3-
import "./Jobs/Monitor/FetchList";
7+
import FetchListAndProbe from "./Jobs/Monitor/FetchList";
48
import "./Jobs/Monitor/FetchMonitorTest";
59
import Register from "./Services/Register";
610
import { PromiseVoidFunction } from "Common/Types/FunctionTypes";
11+
import Sleep from "Common/Types/Sleep";
712
import logger from "Common/Server/Utils/Logger";
813
import App from "Common/Server/Utils/StartServer";
914
import Telemetry from "Common/Server/Utils/Telemetry";
@@ -33,6 +38,10 @@ const init: PromiseVoidFunction = async (): Promise<void> => {
3338
await App.addDefaultRoutes();
3439

3540
try {
41+
logger.debug(
42+
`This probe will retyr monitor for: ${PROBE_MONITOR_RETRY_LIMIT} times`,
43+
);
44+
3645
// Register this probe.
3746
await Register.registerProbe();
3847

@@ -44,6 +53,30 @@ const init: PromiseVoidFunction = async (): Promise<void> => {
4453
logger.error(err);
4554
throw err;
4655
}
56+
57+
try {
58+
let workers: number = 0;
59+
60+
while (workers < PROBE_MONITORING_WORKERS) {
61+
workers++;
62+
63+
const currentWorker: number = workers;
64+
65+
logger.debug(`Starting worker ${currentWorker}`);
66+
67+
new FetchListAndProbe("Worker " + currentWorker)
68+
.run()
69+
.catch((err: any) => {
70+
logger.error(`Worker ${currentWorker} failed: `);
71+
logger.error(err);
72+
});
73+
74+
await Sleep.sleep(1000);
75+
}
76+
} catch (err) {
77+
logger.error("Starting workers failed");
78+
logger.error(err);
79+
}
4780
} catch (err) {
4881
logger.error("App Init Failed:");
4982
logger.error(err);

‎Probe/Jobs/Monitor/FetchList.ts

Lines changed: 36 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,54 +1,19 @@
1-
import {
2-
PROBE_INGEST_URL,
3-
PROBE_MONITOR_FETCH_LIMIT,
4-
PROBE_MONITORING_WORKERS,
5-
} from "../../Config";
1+
import { PROBE_INGEST_URL, PROBE_MONITOR_FETCH_LIMIT } from "../../Config";
62
import MonitorUtil from "../../Utils/Monitors/Monitor";
73
import ProbeAPIRequest from "../../Utils/ProbeAPIRequest";
84
import BaseModel from "Common/Models/DatabaseModels/DatabaseBaseModel/DatabaseBaseModel";
95
import HTTPErrorResponse from "Common/Types/API/HTTPErrorResponse";
106
import HTTPMethod from "Common/Types/API/HTTPMethod";
117
import HTTPResponse from "Common/Types/API/HTTPResponse";
128
import URL from "Common/Types/API/URL";
9+
import OneUptimeDate from "Common/Types/Date";
1310
import APIException from "Common/Types/Exception/ApiException";
1411
import { JSONArray } from "Common/Types/JSON";
1512
import ProbeMonitorResponse from "Common/Types/Probe/ProbeMonitorResponse";
13+
import Sleep from "Common/Types/Sleep";
1614
import API from "Common/Utils/API";
1715
import logger from "Common/Server/Utils/Logger";
1816
import Monitor from "Common/Models/DatabaseModels/Monitor";
19-
import { EVERY_MINUTE } from "Common/Utils/CronTime";
20-
import BasicCron from "Common/Server/Utils/BasicCron";
21-
22-
BasicCron({
23-
jobName: "Probe:MonitorFetchList",
24-
options: {
25-
schedule: EVERY_MINUTE,
26-
runOnStartup: true,
27-
},
28-
runFunction: async () => {
29-
try {
30-
let workers: number = 0;
31-
32-
while (workers < PROBE_MONITORING_WORKERS) {
33-
workers++;
34-
35-
const currentWorker: number = workers;
36-
37-
logger.debug(`Starting worker ${currentWorker}`);
38-
39-
new FetchListAndProbe("Worker " + currentWorker)
40-
.run()
41-
.catch((err: any) => {
42-
logger.error(`Worker ${currentWorker} failed: `);
43-
logger.error(err);
44-
});
45-
}
46-
} catch (err) {
47-
logger.error("Starting workers failed");
48-
logger.error(err);
49-
}
50-
},
51-
});
5217

5318
export default class FetchListAndProbe {
5419
private workerName: string = "";
@@ -60,15 +25,41 @@ export default class FetchListAndProbe {
6025
public async run(): Promise<void> {
6126
logger.debug(`Running worker ${this.workerName}`);
6227

63-
try {
64-
logger.debug(`Probing monitors ${this.workerName}`);
28+
// eslint-disable-next-line no-constant-condition
29+
while (true) {
30+
try {
31+
// Sleep randomly between 1 and 5 seconds.
32+
// We do this to avoid all workers hitting the server at the same time and fetching the same monitors.
33+
const sleepTime: number = Math.floor(Math.random() * 5000) + 1000;
34+
logger.debug(
35+
`Worker ${this.workerName} is sleeping for ${sleepTime} seconds`,
36+
);
37+
await Sleep.sleep(Math.round(sleepTime) % 5000);
6538

66-
await this.fetchListAndProbe();
39+
const runTime: Date = OneUptimeDate.getCurrentDate();
6740

68-
logger.debug(`Probing monitors ${this.workerName} complete`);
69-
} catch (err) {
70-
logger.error(`Error in worker ${this.workerName}`);
71-
logger.error(err);
41+
logger.debug(`Probing monitors ${this.workerName}`);
42+
43+
await this.fetchListAndProbe();
44+
45+
logger.debug(`Probing monitors ${this.workerName} complete`);
46+
47+
// if rumTime + 5 seconds is in the future, then this fetchLst either errored out or had no monitors in the list. Either way, wait for 5 seconds and proceed.
48+
49+
const twoSecondsAdded: Date = OneUptimeDate.addRemoveSeconds(
50+
runTime,
51+
2,
52+
);
53+
54+
if (OneUptimeDate.isInTheFuture(twoSecondsAdded)) {
55+
logger.debug(`Worker ${this.workerName} is waiting for 2 seconds`);
56+
await Sleep.sleep(2000);
57+
}
58+
} catch (err) {
59+
logger.error(`Error in worker ${this.workerName}`);
60+
logger.error(err);
61+
await Sleep.sleep(2000);
62+
}
7263
}
7364
}
7465

0 commit comments

Comments
 (0)