diff --git a/monitoring/icinga2/.gitignore b/monitoring/icinga2/.gitignore new file mode 100644 index 0000000..8a4852e --- /dev/null +++ b/monitoring/icinga2/.gitignore @@ -0,0 +1 @@ +features-available diff --git a/monitoring/icinga2/conf.d/commands-new.conf b/monitoring/icinga2/conf.d/commands-new.conf new file mode 100644 index 0000000..4e971c1 --- /dev/null +++ b/monitoring/icinga2/conf.d/commands-new.conf @@ -0,0 +1,19 @@ +object CheckCommand "check_net_int" { + import "by_ssh" + vars.by_ssh_timeout = "420" + timeout = "420" +# vars.by_ssh_command = "sudo /usr/lib64/nagios/plugins/check_net_int.sh -w $int_warn$ -c $int_crit$" + vars.by_ssh_command = "sudo /usr/lib/nagios/plugins/check_interface_load.sh -w 10000 -c 20000" + vars.int_warn="" + vars.int_crit="" +} + +object CheckCommand "check_open_ports" { + import "by_ssh" + vars.by_ssh_timeout = "420" + timeout = "420" + vars.by_ssh_command = "sudo /usr/lib/nagios/plugins/check_network_ports.sh -p \"$ports$\" -H $host$" + vars.ports="" + vars.host="" +} + diff --git a/monitoring/icinga2/conf.d/hosts/router-vin.conf b/monitoring/icinga2/conf.d/hosts/router-vin.conf new file mode 100644 index 0000000..f0cac9f --- /dev/null +++ b/monitoring/icinga2/conf.d/hosts/router-vin.conf @@ -0,0 +1,18 @@ +object Host "vin-external.openfest.org" { + import "generic-host" + + address = "185.117.82.237" + # check_command = "xxx_health" + # vars.extra_port_check = "" + vars.by_ssh_port = "22" + # vars.specification = "router" + + vars.net_interfaces = [ "bond0", "bond0.20", "bond0.21", "bond0.22", "bond0.23", "bond0.24", "bond0.25", "bond0.26", "bond0.4001", "eno1", "enp1s0f4", "enp1s0f4d1" ] + + vars.open_ports = "80 179 2004 53 22 7002 8443 443" + + vars.notification["slack"] = { + groups = [ "icingaadmins"] + } +} + diff --git a/monitoring/icinga2/conf.d/notifications.conf b/monitoring/icinga2/conf.d/notifications.conf index ac65875..66a7e00 100644 --- a/monitoring/icinga2/conf.d/notifications.conf +++ b/monitoring/icinga2/conf.d/notifications.conf @@ -8,26 +8,34 @@ * Check `hosts.conf` for an example. */ -apply Notification "mail-icingaadmin" to Host { - import "mail-host-notification" - user_groups = host.vars.notification.mail.groups - users = host.vars.notification.mail.users +#apply Notification "mail-icingaadmin" to Host { +# import "mail-host-notification" +# user_groups = host.vars.notification.mail.groups +# users = host.vars.notification.mail.users //interval = 2h //vars.notification_logtosyslog = true - assign where host.vars.notification.mail -} +# assign where host.vars.notification.mail +#} -apply Notification "mail-icingaadmin" to Service { - import "mail-service-notification" - user_groups = host.vars.notification.mail.groups - users = host.vars.notification.mail.users +//apply Notification "mail-icingaadmin" to Service { +// import "mail-service-notification" +// user_groups = host.vars.notification.mail.groups +// users = host.vars.notification.mail.users //interval = 2h //vars.notification_logtosyslog = true - assign where host.vars.notification.mail +// assign where host.vars.notification.mail +//} + +apply Notification "slack" to Service { + import "slack-service-notification" + user_groups = host.vars.notification.slack.groups + interval = 1m + vars.notification_logtosyslog = true + assign where host.vars.notification.slack } diff --git a/monitoring/icinga2/conf.d/services-new.conf b/monitoring/icinga2/conf.d/services-new.conf new file mode 100644 index 0000000..d71b6e9 --- /dev/null +++ b/monitoring/icinga2/conf.d/services-new.conf @@ -0,0 +1,24 @@ +apply Service "network_interfaces_traffic" { + import "generic-service" + check_interval = 5m + retry_interval = 5m + check_command = "check_net_int" + vars.int_warn="925" + vars.int_crit="950" + + assign where host.vars.net_interfaces + + vars.notif_prio = "high" +} + +apply Service "network_open_ports" { + import "generic-service" + check_interval = 5m + retry_interval = 5m + check_command = "check_open_ports" + vars.ports=host.vars.open_ports + vars.host=host.address + assign where host.vars.open_ports + + vars.notif_prio = "high" +} diff --git a/monitoring/icinga2/conf.d/services.conf b/monitoring/icinga2/conf.d/services.conf index c8e1b3c..efc987f 100644 --- a/monitoring/icinga2/conf.d/services.conf +++ b/monitoring/icinga2/conf.d/services.conf @@ -102,7 +102,7 @@ apply Service "swap" { check_command = "swap" - assign where host.name == NodeName + assign where host.vars.swap_size } apply Service "users" { diff --git a/monitoring/icinga2/conf.d/slack-service-notification-command.conf b/monitoring/icinga2/conf.d/slack-service-notification-command.conf new file mode 100644 index 0000000..99c525e --- /dev/null +++ b/monitoring/icinga2/conf.d/slack-service-notification-command.conf @@ -0,0 +1,20 @@ +object NotificationCommand "slack-service-notification" { + import "plugin-notification-command" + + command = [ SysconfDir + "/icinga2/scripts/slack-service-notification.sh" ] + + env = { + "NOTIFICATIONTYPE" = "$notification.type$" + "SERVICEDESC" = "$service.name$" + "HOSTALIAS" = "$host.display_name$", + "HOSTNAME" = "$host.name$", + "HOSTADDRESS" = "$address$", + "SERVICESTATE" = "$service.state$", + "LONGDATETIME" = "$icinga.long_date_time$", + "SERVICEOUTPUT" = "$service.output$", + "NOTIFICATIONAUTHORNAME" = "$notification.author$", + "NOTIFICATIONCOMMENT" = "$notification.comment$", + "HOSTDISPLAYNAME" = "$host.display_name$", + "SERVICEDISPLAYNAME" = "$service.display_name$", + } +} diff --git a/monitoring/icinga2/conf.d/slack-service-notification.conf b/monitoring/icinga2/conf.d/slack-service-notification.conf new file mode 100644 index 0000000..4bbb6d4 --- /dev/null +++ b/monitoring/icinga2/conf.d/slack-service-notification.conf @@ -0,0 +1,10 @@ +template Notification "slack-service-notification" { + command = "slack-service-notification" + + states = [ OK, Warning, Critical, Unknown ] + types = [ Problem, Acknowledgement, Recovery, Custom, + FlappingStart, FlappingEnd, + DowntimeStart, DowntimeEnd, DowntimeRemoved ] + + period = "24x7" +} diff --git a/monitoring/icinga2/constants.conf b/monitoring/icinga2/constants.conf index 2d32308..5ee66df 100644 --- a/monitoring/icinga2/constants.conf +++ b/monitoring/icinga2/constants.conf @@ -3,6 +3,9 @@ * the other configuration files. */ +/* XXX: Nice comment */ +const SysconfDir = "etc" + /* The directory which contains the plugins from the Monitoring Plugins project. */ const PluginDir = "/usr/lib/nagios/plugins" diff --git a/monitoring/icinga2/features-enabled/graphite.conf b/monitoring/icinga2/features-enabled/graphite.conf new file mode 100644 index 0000000..8f084d2 --- /dev/null +++ b/monitoring/icinga2/features-enabled/graphite.conf @@ -0,0 +1,12 @@ +/** + * The GraphiteWriter type writes check result metrics and + * performance data to a graphite tcp socket. + */ + +library "perfdata" + +object GraphiteWriter "graphite" { +// host = "127.0.0.1" +// port = 2003 +// enable_send_thresholds = true +} diff --git a/monitoring/icinga2/features-enabled/perfdata.conf b/monitoring/icinga2/features-enabled/perfdata.conf new file mode 100644 index 0000000..3ba8635 --- /dev/null +++ b/monitoring/icinga2/features-enabled/perfdata.conf @@ -0,0 +1,6 @@ +/** + * The PerfdataWriter type writes performance data files and rotates + * them in a regular interval. + */ + +object PerfdataWriter "perfdata" { } diff --git a/monitoring/icinga2/scripts/slack-service-notification.sh b/monitoring/icinga2/scripts/slack-service-notification.sh new file mode 100755 index 0000000..f2cd140 --- /dev/null +++ b/monitoring/icinga2/scripts/slack-service-notification.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +ICINGA_HOSTNAME="vin.openfest.org/icingaweb2" +SLACK_WEBHOOK_URL="https://hooks.slack.com/services/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" +SLACK_CHANNEL="#video" +SLACK_BOTNAME="icinga2" + + +if [ "$NOTIFICATIONTYPE" = "ACKNOWLEDGEMENT" ] || [ "$NOTIFICATIONTYPE" = "DOWNTIMESTART" ] || [ "$NOTIFICATIONTYPE" = "DOWNTIMEEND" ] +then + COLOR="#FFB6C1" + read -d '' PAYLOAD << EOF +{ + "channel": "${SLACK_CHANNEL}", + "username": "${SLACK_BOTNAME}", + "attachments": [ + { + "fallback": "${NOTIFICATIONTYPE}: ${SERVICESTATE}: ${HOSTDISPLAYNAME} - ${SERVICEDISPLAYNAME}", + "color": "${COLOR}", + "fields": [ + { + "title": "${NOTIFICATIONTYPE}", + "value": "${NOTIFICATIONCOMMENT} - ${NOTIFICATIONAUTHORNAME}" + }, + { + "title": "Service output", + "value": "${SERVICEOUTPUT}", + "short": false + }, + { + "title": "Host", + "value": "<${ICINGA_HOSTNAME}/monitoring/host/services?host=${HOSTNAME}|${HOSTDISPLAYNAME}>", + "short": true + }, + { + "title": "Service", + "value": "<${ICINGA_HOSTNAME}/monitoring/service/show?host=${HOSTNAME}&service=${SERVICEDESC}|${SERVICEDISPLAYNAME}>", + "short": true + }, + { + "title": "State", + "value": "${SERVICESTATE}", + "short": true + } + ] + } + ] +} +EOF +else + +#Set the message icon based on ICINGA service state +if [ "$SERVICESTATE" = "CRITICAL" ] +then + COLOR="danger" +elif [ "$SERVICESTATE" = "WARNING" ] +then + COLOR="warning" +elif [ "$SERVICESTATE" = "OK" ] +then + COLOR="good" +elif [ "$SERVICESTATE" = "UNKNOWN" ] +then + COLOR="#800080" +else + COLOR="" +fi + +#Send message to Slack +read -d '' PAYLOAD << EOF +{ + "channel": "${SLACK_CHANNEL}", + "username": "${SLACK_BOTNAME}", + "attachments": [ + { + "fallback": "${SERVICESTATE}: ${HOSTDISPLAYNAME} - ${SERVICEDISPLAYNAME}", + "color": "${COLOR}", + "fields": [ + { + "title": "Service output", + "value": "${SERVICEOUTPUT}", + "short": false + }, + { + "title": "Host", + "value": "<${ICINGA_HOSTNAME}/monitoring/host/services?host=${HOSTNAME}|${HOSTDISPLAYNAME}>", + "short": true + }, + { + "title": "Service", + "value": "<${ICINGA_HOSTNAME}/monitoring/service/show?host=${HOSTNAME}&service=${SERVICEDESC}|${SERVICEDISPLAYNAME}>", + "short": true + }, + { + "title": "State", + "value": "${SERVICESTATE}", + "short": true + } + ] + } + ] +} +EOF + +fi + +curl --connect-timeout 30 --max-time 60 -s -S -X POST -H 'Content-type: application/json' --data "${PAYLOAD}" "${SLACK_WEBHOOK_URL}" diff --git a/monitoring/icingaweb2/enabledModules/.gitignore b/monitoring/icingaweb2/enabledModules/.gitignore new file mode 100644 index 0000000..72e8ffc --- /dev/null +++ b/monitoring/icingaweb2/enabledModules/.gitignore @@ -0,0 +1 @@ +* diff --git a/monitoring/icingaweb2/enabledModules/.gitkeep b/monitoring/icingaweb2/enabledModules/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/monitoring/icingaweb2/modules/graphite/config.ini b/monitoring/icingaweb2/modules/graphite/config.ini new file mode 100644 index 0000000..fb2ad1d --- /dev/null +++ b/monitoring/icingaweb2/modules/graphite/config.ini @@ -0,0 +1,4 @@ +[graphite] +metric_prefix = icinga +url = https://vin.openfest.org:8443/render? +args_template = "&target=$target$&source=0&width=300&height=120&hideAxes=true&lineWidth=1&hideLegend=true&hideAxes=false&colorList=049BAF&bgcolor=white&fgcolor=blue&minorLine=green"