From fba000685a9459c0f79a51ac1c3db90869f26e2d Mon Sep 17 00:00:00 2001
From: Vladimir Vitkov <5081863+zeridon@users.noreply.github.com>
Date: Sun, 27 Oct 2019 22:10:55 +0200
Subject: [PATCH] Update monitoring configs

---
 monitoring/icinga2/.gitignore                 |   1 +
 monitoring/icinga2/conf.d/commands-new.conf   |  19 ++++
 .../icinga2/conf.d/hosts/router-vin.conf      |  18 +++
 monitoring/icinga2/conf.d/notifications.conf  |  30 +++--
 monitoring/icinga2/conf.d/services-new.conf   |  24 ++++
 monitoring/icinga2/conf.d/services.conf       |   2 +-
 .../slack-service-notification-command.conf   |  20 ++++
 .../conf.d/slack-service-notification.conf    |  10 ++
 monitoring/icinga2/constants.conf             |   3 +
 .../icinga2/features-enabled/graphite.conf    |  12 ++
 .../icinga2/features-enabled/perfdata.conf    |   6 +
 .../scripts/slack-service-notification.sh     | 107 ++++++++++++++++++
 .../icingaweb2/enabledModules/.gitignore      |   1 +
 monitoring/icingaweb2/enabledModules/.gitkeep |   0
 .../icingaweb2/modules/graphite/config.ini    |   4 +
 15 files changed, 245 insertions(+), 12 deletions(-)
 create mode 100644 monitoring/icinga2/.gitignore
 create mode 100644 monitoring/icinga2/conf.d/commands-new.conf
 create mode 100644 monitoring/icinga2/conf.d/hosts/router-vin.conf
 create mode 100644 monitoring/icinga2/conf.d/services-new.conf
 create mode 100644 monitoring/icinga2/conf.d/slack-service-notification-command.conf
 create mode 100644 monitoring/icinga2/conf.d/slack-service-notification.conf
 create mode 100644 monitoring/icinga2/features-enabled/graphite.conf
 create mode 100644 monitoring/icinga2/features-enabled/perfdata.conf
 create mode 100755 monitoring/icinga2/scripts/slack-service-notification.sh
 create mode 100644 monitoring/icingaweb2/enabledModules/.gitignore
 create mode 100644 monitoring/icingaweb2/enabledModules/.gitkeep
 create mode 100644 monitoring/icingaweb2/modules/graphite/config.ini

diff --git a/monitoring/icinga2/.gitignore b/monitoring/icinga2/.gitignore
new file mode 100644
index 0000000..8a4852e
--- /dev/null
+++ b/monitoring/icinga2/.gitignore
@@ -0,0 +1 @@
+features-available
diff --git a/monitoring/icinga2/conf.d/commands-new.conf b/monitoring/icinga2/conf.d/commands-new.conf
new file mode 100644
index 0000000..4e971c1
--- /dev/null
+++ b/monitoring/icinga2/conf.d/commands-new.conf
@@ -0,0 +1,19 @@
+object CheckCommand "check_net_int" {
+        import "by_ssh"
+        vars.by_ssh_timeout = "420"
+        timeout = "420"
+#          vars.by_ssh_command = "sudo /usr/lib64/nagios/plugins/check_net_int.sh -w $int_warn$ -c $int_crit$"
+          vars.by_ssh_command = "sudo /usr/lib/nagios/plugins/check_interface_load.sh -w 10000 -c 20000"
+          vars.int_warn=""
+          vars.int_crit=""
+}
+
+object CheckCommand "check_open_ports" {
+        import "by_ssh"
+        vars.by_ssh_timeout = "420"
+        timeout = "420"
+        vars.by_ssh_command = "sudo /usr/lib/nagios/plugins/check_network_ports.sh -p \"$ports$\" -H $host$"
+        vars.ports=""
+	vars.host=""
+}
+
diff --git a/monitoring/icinga2/conf.d/hosts/router-vin.conf b/monitoring/icinga2/conf.d/hosts/router-vin.conf
new file mode 100644
index 0000000..f0cac9f
--- /dev/null
+++ b/monitoring/icinga2/conf.d/hosts/router-vin.conf
@@ -0,0 +1,18 @@
+object Host "vin-external.openfest.org" {
+  import "generic-host"
+
+  address = "185.117.82.237"
+  # check_command = "xxx_health"
+  # vars.extra_port_check = ""
+  vars.by_ssh_port = "22"
+  # vars.specification = "router"
+
+  vars.net_interfaces = [ "bond0", "bond0.20", "bond0.21", "bond0.22", "bond0.23", "bond0.24", "bond0.25", "bond0.26", "bond0.4001", "eno1", "enp1s0f4", "enp1s0f4d1" ]
+
+  vars.open_ports = "80 179 2004 53 22 7002 8443 443"
+
+  vars.notification["slack"] = {
+    groups = [ "icingaadmins"]
+  }
+}
+
diff --git a/monitoring/icinga2/conf.d/notifications.conf b/monitoring/icinga2/conf.d/notifications.conf
index ac65875..66a7e00 100644
--- a/monitoring/icinga2/conf.d/notifications.conf
+++ b/monitoring/icinga2/conf.d/notifications.conf
@@ -8,26 +8,34 @@
  * Check `hosts.conf` for an example.
  */
 
-apply Notification "mail-icingaadmin" to Host {
-  import "mail-host-notification"
-  user_groups = host.vars.notification.mail.groups
-  users = host.vars.notification.mail.users
+#apply Notification "mail-icingaadmin" to Host {
+#  import "mail-host-notification"
+#  user_groups = host.vars.notification.mail.groups
+#  users = host.vars.notification.mail.users
 
   //interval = 2h
 
   //vars.notification_logtosyslog = true
 
-  assign where host.vars.notification.mail
-}
+#  assign where host.vars.notification.mail
+#}
 
-apply Notification "mail-icingaadmin" to Service {
-  import "mail-service-notification"
-  user_groups = host.vars.notification.mail.groups
-  users = host.vars.notification.mail.users
+//apply Notification "mail-icingaadmin" to Service {
+//  import "mail-service-notification"
+//  user_groups = host.vars.notification.mail.groups
+//  users = host.vars.notification.mail.users
 
   //interval = 2h
 
   //vars.notification_logtosyslog = true
 
-  assign where host.vars.notification.mail
+//  assign where host.vars.notification.mail
+//}
+
+apply Notification "slack" to Service {
+  import "slack-service-notification"
+  user_groups = host.vars.notification.slack.groups
+  interval = 1m
+  vars.notification_logtosyslog = true
+  assign where host.vars.notification.slack
 }
diff --git a/monitoring/icinga2/conf.d/services-new.conf b/monitoring/icinga2/conf.d/services-new.conf
new file mode 100644
index 0000000..d71b6e9
--- /dev/null
+++ b/monitoring/icinga2/conf.d/services-new.conf
@@ -0,0 +1,24 @@
+apply Service "network_interfaces_traffic" {
+        import "generic-service"
+        check_interval = 5m
+        retry_interval = 5m
+        check_command = "check_net_int"
+        vars.int_warn="925"
+        vars.int_crit="950"
+
+        assign where host.vars.net_interfaces
+
+        vars.notif_prio = "high"
+}
+
+apply Service "network_open_ports" {
+        import "generic-service"
+        check_interval = 5m
+        retry_interval = 5m
+        check_command = "check_open_ports"
+        vars.ports=host.vars.open_ports
+	vars.host=host.address
+        assign where host.vars.open_ports
+
+        vars.notif_prio = "high"
+}
diff --git a/monitoring/icinga2/conf.d/services.conf b/monitoring/icinga2/conf.d/services.conf
index c8e1b3c..efc987f 100644
--- a/monitoring/icinga2/conf.d/services.conf
+++ b/monitoring/icinga2/conf.d/services.conf
@@ -102,7 +102,7 @@ apply Service "swap" {
 
   check_command = "swap"
 
-  assign where host.name == NodeName
+  assign where host.vars.swap_size
 }
 
 apply Service "users" {
diff --git a/monitoring/icinga2/conf.d/slack-service-notification-command.conf b/monitoring/icinga2/conf.d/slack-service-notification-command.conf
new file mode 100644
index 0000000..99c525e
--- /dev/null
+++ b/monitoring/icinga2/conf.d/slack-service-notification-command.conf
@@ -0,0 +1,20 @@
+object NotificationCommand "slack-service-notification" {
+  import "plugin-notification-command"
+
+  command = [ SysconfDir + "/icinga2/scripts/slack-service-notification.sh" ]
+
+  env = {
+    "NOTIFICATIONTYPE" = "$notification.type$"
+    "SERVICEDESC" = "$service.name$"
+    "HOSTALIAS" = "$host.display_name$",
+    "HOSTNAME" = "$host.name$",
+    "HOSTADDRESS" = "$address$",
+    "SERVICESTATE" = "$service.state$",
+    "LONGDATETIME" = "$icinga.long_date_time$",
+    "SERVICEOUTPUT" = "$service.output$",
+    "NOTIFICATIONAUTHORNAME" = "$notification.author$",
+    "NOTIFICATIONCOMMENT" = "$notification.comment$",
+    "HOSTDISPLAYNAME" = "$host.display_name$",
+    "SERVICEDISPLAYNAME" = "$service.display_name$",
+  }
+}
diff --git a/monitoring/icinga2/conf.d/slack-service-notification.conf b/monitoring/icinga2/conf.d/slack-service-notification.conf
new file mode 100644
index 0000000..4bbb6d4
--- /dev/null
+++ b/monitoring/icinga2/conf.d/slack-service-notification.conf
@@ -0,0 +1,10 @@
+template Notification "slack-service-notification" {
+  command = "slack-service-notification"
+
+  states = [ OK, Warning, Critical, Unknown ]
+  types = [ Problem, Acknowledgement, Recovery, Custom,
+            FlappingStart, FlappingEnd,
+            DowntimeStart, DowntimeEnd, DowntimeRemoved ]
+
+  period = "24x7"
+}
diff --git a/monitoring/icinga2/constants.conf b/monitoring/icinga2/constants.conf
index 2d32308..5ee66df 100644
--- a/monitoring/icinga2/constants.conf
+++ b/monitoring/icinga2/constants.conf
@@ -3,6 +3,9 @@
  * the other configuration files.
  */
 
+/* XXX: Nice comment */
+const SysconfDir = "etc"
+
 /* The directory which contains the plugins from the Monitoring Plugins project. */
 const PluginDir = "/usr/lib/nagios/plugins"
 
diff --git a/monitoring/icinga2/features-enabled/graphite.conf b/monitoring/icinga2/features-enabled/graphite.conf
new file mode 100644
index 0000000..8f084d2
--- /dev/null
+++ b/monitoring/icinga2/features-enabled/graphite.conf
@@ -0,0 +1,12 @@
+/**
+ * The GraphiteWriter type writes check result metrics and
+ * performance data to a graphite tcp socket.
+ */
+
+library "perfdata"
+
+object GraphiteWriter "graphite" {
+//  host = "127.0.0.1"
+//  port = 2003
+//  enable_send_thresholds = true
+}
diff --git a/monitoring/icinga2/features-enabled/perfdata.conf b/monitoring/icinga2/features-enabled/perfdata.conf
new file mode 100644
index 0000000..3ba8635
--- /dev/null
+++ b/monitoring/icinga2/features-enabled/perfdata.conf
@@ -0,0 +1,6 @@
+/**
+ * The PerfdataWriter type writes performance data files and rotates
+ * them in a regular interval.
+ */
+
+object PerfdataWriter "perfdata" { }
diff --git a/monitoring/icinga2/scripts/slack-service-notification.sh b/monitoring/icinga2/scripts/slack-service-notification.sh
new file mode 100755
index 0000000..f2cd140
--- /dev/null
+++ b/monitoring/icinga2/scripts/slack-service-notification.sh
@@ -0,0 +1,107 @@
+#!/bin/bash
+
+ICINGA_HOSTNAME="vin.openfest.org/icingaweb2"
+SLACK_WEBHOOK_URL="https://hooks.slack.com/services/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"
+SLACK_CHANNEL="#video"
+SLACK_BOTNAME="icinga2"
+
+
+if [ "$NOTIFICATIONTYPE" = "ACKNOWLEDGEMENT" ] || [ "$NOTIFICATIONTYPE" = "DOWNTIMESTART" ] || [ "$NOTIFICATIONTYPE" = "DOWNTIMEEND" ]
+then
+  COLOR="#FFB6C1"
+  read -d '' PAYLOAD << EOF
+{
+  "channel": "${SLACK_CHANNEL}",
+  "username": "${SLACK_BOTNAME}",
+  "attachments": [
+    {
+      "fallback": "${NOTIFICATIONTYPE}: ${SERVICESTATE}: ${HOSTDISPLAYNAME} - ${SERVICEDISPLAYNAME}",
+      "color": "${COLOR}",
+      "fields": [
+        {
+          "title": "${NOTIFICATIONTYPE}",
+          "value": "${NOTIFICATIONCOMMENT} - ${NOTIFICATIONAUTHORNAME}"
+        },
+        {
+          "title": "Service output",
+          "value": "${SERVICEOUTPUT}",
+          "short": false
+        },
+        {
+          "title": "Host",
+          "value": "<${ICINGA_HOSTNAME}/monitoring/host/services?host=${HOSTNAME}|${HOSTDISPLAYNAME}>",
+          "short": true
+        },
+        {
+          "title": "Service",
+          "value": "<${ICINGA_HOSTNAME}/monitoring/service/show?host=${HOSTNAME}&service=${SERVICEDESC}|${SERVICEDISPLAYNAME}>",
+          "short": true
+        },
+        {
+          "title": "State",
+          "value": "${SERVICESTATE}",
+          "short": true
+        }
+      ]
+    }
+  ]
+}
+EOF
+else
+
+#Set the message icon based on ICINGA service state
+if [ "$SERVICESTATE" = "CRITICAL" ]
+then
+    COLOR="danger"
+elif [ "$SERVICESTATE" = "WARNING" ]
+then
+    COLOR="warning"
+elif [ "$SERVICESTATE" = "OK" ]
+then
+    COLOR="good"
+elif [ "$SERVICESTATE" = "UNKNOWN" ]
+then
+    COLOR="#800080"
+else
+    COLOR=""
+fi
+
+#Send message to Slack
+read -d '' PAYLOAD << EOF
+{
+  "channel": "${SLACK_CHANNEL}",
+  "username": "${SLACK_BOTNAME}",
+  "attachments": [
+    {
+      "fallback": "${SERVICESTATE}: ${HOSTDISPLAYNAME} - ${SERVICEDISPLAYNAME}",
+      "color": "${COLOR}",
+      "fields": [
+        {
+          "title": "Service output",
+          "value": "${SERVICEOUTPUT}",
+          "short": false
+        },
+        {
+          "title": "Host",
+          "value": "<${ICINGA_HOSTNAME}/monitoring/host/services?host=${HOSTNAME}|${HOSTDISPLAYNAME}>",
+          "short": true
+        },
+        {
+          "title": "Service",
+          "value": "<${ICINGA_HOSTNAME}/monitoring/service/show?host=${HOSTNAME}&service=${SERVICEDESC}|${SERVICEDISPLAYNAME}>",
+          "short": true
+        },
+        {
+          "title": "State",
+          "value": "${SERVICESTATE}",
+          "short": true
+        }
+      ]
+    }
+  ]
+}
+EOF
+
+fi
+
+curl --connect-timeout 30 --max-time 60 -s -S -X POST -H 'Content-type: application/json' --data "${PAYLOAD}" "${SLACK_WEBHOOK_URL}"
diff --git a/monitoring/icingaweb2/enabledModules/.gitignore b/monitoring/icingaweb2/enabledModules/.gitignore
new file mode 100644
index 0000000..72e8ffc
--- /dev/null
+++ b/monitoring/icingaweb2/enabledModules/.gitignore
@@ -0,0 +1 @@
+*
diff --git a/monitoring/icingaweb2/enabledModules/.gitkeep b/monitoring/icingaweb2/enabledModules/.gitkeep
new file mode 100644
index 0000000..e69de29
diff --git a/monitoring/icingaweb2/modules/graphite/config.ini b/monitoring/icingaweb2/modules/graphite/config.ini
new file mode 100644
index 0000000..fb2ad1d
--- /dev/null
+++ b/monitoring/icingaweb2/modules/graphite/config.ini
@@ -0,0 +1,4 @@
+[graphite]
+metric_prefix = icinga
+url = https://vin.openfest.org:8443/render?
+args_template = "&target=$target$&source=0&width=300&height=120&hideAxes=true&lineWidth=1&hideLegend=true&hideAxes=false&colorList=049BAF&bgcolor=white&fgcolor=blue&minorLine=green"