feat: Add alerting service (#269)

Rishabh Singh · web-flow · commit 26643349bc0d · 2021-07-26T20:22:39.000+05:30
* Add alerting service

* .
diff --git a/docker/README.md b/docker/README.md
@@ -25,6 +25,12 @@ This will start all services required for Hypertrace. Once you see the service h
 
 If you are facing any issues with docker-compose setup, we have listed down common issues and resolutions [here](https://docs.hypertrace.org/troubleshooting/docker-compose/).
 
+#### To start alerting related serivces 
+
+Run ```docker-compose -f docker-compose.yml  -f docker-compose.alerting.yml up```
+
+Update alerting rule definition in alert-rules.json, notification-rules.json
+
 ### Ports
 
 Here are the default Hypertrace ports:
diff --git a/docker/configs/alert-rules.json b/docker/configs/alert-rules.json
@@ -0,0 +1,42 @@
+[
+    {
+      "id": "notification_rule_1",
+      "ruleName": "high_avg_latency",
+      "description": "Alert for high avg latency of payment service",
+      "eventConditionId": "event_condition_1",
+      "eventConditionType": "MetricAnomalyEventCondition",
+      "channelId": "channel-id-1",
+      "eventCondition" :  {
+        "metricSelection": {
+          "metricAttribute": {
+            "key": "numCalls",
+            "scope": "SERVICE"
+          },
+          "metricAggregationFunction": "METRIC_AGGREGATION_FUNCTION_TYPE_SUM",
+          "metricAggregationInterval": "PT15S",
+          "filter": {
+            "leafFilter": {
+              "lhsExpression": {
+                "attribute": {
+                  "key": "name",
+                  "scope": "SERVICE"
+                }
+              },
+              "valueOperator": "VALUE_OPERATOR_EQ",
+              "rhsExpression": {
+                "stringValue": "customer"
+              }
+            }
+          }
+        },
+        "violationCondition": [{
+          "staticThresholdCondition": {
+            "operator": "STATIC_THRESHOLD_OPERATOR_GT",
+            "value": 0.0,
+            "minimumViolationDuration": "PT5M",
+            "severity": "SEVERITY_CRITICAL"
+          }
+        }]
+      }
+    }
+  ]
diff --git a/docker/configs/notification-rules.json b/docker/configs/notification-rules.json
@@ -0,0 +1,13 @@
+[
+    {
+      "channelName": "channel-1",
+      "channelId": "channel-id-1",
+      "channelConfig": [
+        {
+          "channelConfigType": "WEBHOOK",
+          "url": "<add slack url here>",
+          "webhookFormat": "WEBHOOK_FORMAT_SLACK"
+        }
+      ]
+    }
+  ]
diff --git a/docker/docker-compose.alerting.yml b/docker/docker-compose.alerting.yml
@@ -0,0 +1,21 @@
+
+version: '2.4'
+services:
+
+  hypertrace-alert-engine:
+    image:  hypertrace/hypertrace-alert-engine
+    container_name: hypertrace-alert-engine
+    environment:
+        - ALERT_RULES_PATH=/app/resources/alert-rules.json
+        - NOTIFICATION_RULES_PATH=/app/resources/notification-rules.json
+        - QUERY_SERVICE_HOST_CONFIG=hypertrace
+        - QUERY_SERVICE_PORT_CONFIG=9001
+        - ATTRIBUTE_SERVICE_HOST_CONFIG=hypertrace
+        - ATTRIBUTE_SERVICE_PORT_CONFIG=9001
+    volumes:
+        - ../docker/configs/alert-rules.json:/app/resources/alert-rules.json:ro
+        - ../docker/configs/notification-rules.json:/app/resources/notification-rules.json:ro
+    depends_on:
+        hypertrace:
+        # service_started, not service_healthy as pinot and deps can take longer than 60s to start
+            condition: service_healthy  
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -3,7 +3,7 @@
 ## Note: Our stack is dependent on pinot and it is a cpu heavy during startup.
 ## The depends_on has a max wait time of 1 min, so if you don't have enough resources, you may have to re-run the same command.
 ## we are looking at improving this.
-version: "2.4"
+version: '2.4'
 services:
 
 
@@ -73,6 +73,7 @@ services:
       hypertrace:
         # service_started, not service_healthy as pinot and deps can take longer than 60s to start
         condition: service_started
+     
 
 # Third-party data services:
 
@@ -105,3 +106,5 @@ services:
     depends_on:
       kafka-zookeeper:
         condition: service_healthy
+
+        
