From d162a3dbd807f12728b1c7e609b5477c99a3a921 Mon Sep 17 00:00:00 2001
From: paulfantom <pawel@krupa.net.pl>
Date: Wed, 22 Jan 2020 22:31:46 +0100
Subject: [PATCH] add a mechanism to distribute textfile collectors alongside
node_exporter
---
README.md | 33 +++++++++++-
defaults/main.yml | 11 ++++
handlers/main.yml | 8 +++
molecule/alternative/playbook.yml | 5 ++
molecule/alternative/prepare.yml | 18 ++++++-
.../alternative/tests/test_alternative.py | 19 ++++++-
molecule/default/playbook.yml | 1 +
molecule/default/tests/test_default.py | 3 +-
tasks/configure.yml | 52 +++++++++++++++----
tasks/preflight.yml | 17 ++++++
templates/cron.service.j2 | 9 ++++
templates/cron.timer.j2 | 14 +++++
12 files changed, 174 insertions(+), 16 deletions(-)
create mode 100644 templates/cron.service.j2
create mode 100644 templates/cron.timer.j2
diff --git a/README.md b/README.md
index 6a261fd..17e3757 100644
--- a/README.md
+++ b/README.md
@@ -27,11 +27,12 @@ All variables which can be overridden are stored in [defaults/main.yml](defaults
| `node_exporter_web_listen_address` | "0.0.0.0:9100" | Address on which node exporter will listen |
| `node_exporter_enabled_collectors` | [ systemd, textfile ] | List of additionally enabled collectors. It adds collectors to [those enabled by default](https://github.com/prometheus/node_exporter#enabled-by-default) |
| `node_exporter_disabled_collectors` | [] | List of disabled collectors. By default node_exporter disables collectors listed [here](https://github.com/prometheus/node_exporter#disabled-by-default). |
-| `node_exporter_textfile_dir` | "/var/lib/node_exporter" | Directory used by the [Textfile Collector](https://github.com/prometheus/node_exporter#textfile-collector). To get permissions to write metrics in this directory, users must be in `node-exp` system group.
+| `node_exporter_textfile_dir` | "/var/lib/node_exporter" | Directory used by the [Textfile Collector](https://github.com/prometheus/node_exporter#textfile-collector). To get permissions to write metrics in this directory, users must be in `node-exp` system group. |
+| `node_exporter_textfile_collectors` | [] | List of textfile collectors which should be copied and crontab setup for them. |
## Example
-### Playbook
+### Basic playbook
Use it in a playbook as follows:
```yaml
@@ -40,6 +41,34 @@ Use it in a playbook as follows:
- cloudalchemy.node-exporter
```
+### Advanced playbook
+Following playbook sets up few more advanced features of this role. Mostly:
+- Every role run checks and tries to update node_exporter binary to latest available version
+- node_exporter is run as `root` user, which allows to use `systemd` collector
+- `systemd` collector is enabled
+- enables cron job running two textfile collectors: [`yum.sh`](https://github.com/prometheus-community/node-exporter-textfile-collector-scripts/blob/master/yum.sh) and [`md_info_detail.sh`](https://github.com/prometheus-community/node-exporter-textfile-collector-scripts/blob/master/md_info_detail.sh) Textfile collector is also copied from deployer host with path specified in `src`.
+
+```yaml
+- hosts: all
+ roles:
+ - cloudalchemy.node-exporter
+ vars:
+ node_exporter_system_group: "root"
+ node_exporter_system_user: "root"
+ node_exporter_enabled_collectors:
+ - systemd
+ node_exporter_textfile_collectors:
+ - src: "/tmp/md_info_detail.sh"
+ special_time: "hourly"
+ - src: "/tmp/yum.sh"
+ user: node-exp
+ minute: "*"
+ hour: "*"
+ day: "*"
+ month: "*"
+ weekday: "*"
+```
+
### Demo site
We provide demo site for full monitoring solution based on prometheus and grafana. Repository with code and links to running instances is [available on github](https://github.com/cloudalchemy/demo-site) and site is hosted on [DigitalOcean](https://digitalocean.com).
diff --git a/defaults/main.yml b/defaults/main.yml
index f6f549a..dc2b75f 100644
--- a/defaults/main.yml
+++ b/defaults/main.yml
@@ -4,6 +4,17 @@ node_exporter_binary_local_dir: ""
node_exporter_web_listen_address: "0.0.0.0:9100"
node_exporter_textfile_dir: "/var/lib/node_exporter"
+node_exporter_textfile_collectors: []
+# node_exporter_textfile_collectors:
+# - src: "/tmp/apt.sh"
+# user: root
+# calendar: ""
+# active_sec: ""
+# boot_sec: ""
+# startup_sec: ""
+# unit_active_sec: ""
+# unit_inactive_sec: ""
+
node_exporter_enabled_collectors:
- systemd
diff --git a/handlers/main.yml b/handlers/main.yml
index 25551ee..2c4240c 100644
--- a/handlers/main.yml
+++ b/handlers/main.yml
@@ -5,3 +5,11 @@
daemon_reload: true
name: node_exporter
state: restarted
+
+- name: reload cron
+ become: true
+ systemd:
+ daemon_reload: true
+ name: "textfile-collector-{{ item.src | basename }}.timer"
+ state: restarted
+ with_items: "{{ node_exporter_textfile_collectors }}"
diff --git a/molecule/alternative/playbook.yml b/molecule/alternative/playbook.yml
index ce51111..22c455e 100644
--- a/molecule/alternative/playbook.yml
+++ b/molecule/alternative/playbook.yml
@@ -11,3 +11,8 @@
- entropy
node_exporter_disabled_collectors:
- diskstats
+ node_exporter_textfile_collectors:
+ - src: "/tmp/apt.sh"
+ user: root
+ calendar: "1h 30min"
+ boot_sec: 90
diff --git a/molecule/alternative/prepare.yml b/molecule/alternative/prepare.yml
index 2f75da1..742234c 100644
--- a/molecule/alternative/prepare.yml
+++ b/molecule/alternative/prepare.yml
@@ -1,5 +1,5 @@
---
-- name: Prepare
+- name: Prepare localhost
hosts: localhost
gather_facts: false
vars:
@@ -35,3 +35,19 @@
state: link
run_once: true
check_mode: false
+
+ - name: get collector scripts
+ become: false
+ get_url:
+ url: "{{ item }}"
+ dest: "/tmp/{{ item.split('/')[-1] }}"
+ with_items:
+ - https://raw.githubusercontent.com/prometheus-community/node-exporter-textfile-collector-scripts/master/apt.sh
+
+- name: Prepare instances
+ hosts: all
+ tasks:
+ - name: Install cron
+ package:
+ name: "{{ 'cronie' if (ansible_os_family | lower == 'redhat') else 'cron' }}"
+ state: present
diff --git a/molecule/alternative/tests/test_alternative.py b/molecule/alternative/tests/test_alternative.py
index 87cfc15..1ea16fe 100644
--- a/molecule/alternative/tests/test_alternative.py
+++ b/molecule/alternative/tests/test_alternative.py
@@ -5,13 +5,30 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+def test_files(host):
+ files = [
+ "/usr/local/bin/md_info_detail.sh",
+ "/usr/local/bin/apt.sh"
+ "/etc/systemd/system/textfile-collector-apt.sh.service"
+ "/etc/systemd/system/textfile-collector-apt.sh.timer"
+ "/etc/systemd/system/textfile-collector-md_info_detail.sh.service"
+ "/etc/systemd/system/textfile-collector-md_info_detail.sh.timer"
+ ]
+ for file in files:
+ f = host.file(file)
+ assert f.exists
+ assert f.is_file
+ assert oct(f.mode) == '0o755'
+
+
def test_directories(host):
dirs = [
"/var/lib/node_exporter"
]
for dir in dirs:
d = host.file(dir)
- assert not d.exists
+ assert d.is_directory
+ assert d.exists
def test_service(host):
diff --git a/molecule/default/playbook.yml b/molecule/default/playbook.yml
index 1e92855..36af894 100644
--- a/molecule/default/playbook.yml
+++ b/molecule/default/playbook.yml
@@ -5,3 +5,4 @@
- ansible-node-exporter
vars:
node_exporter_web_listen_address: "127.0.0.1:9100"
+ node_exporter_textfile_dir: ""
diff --git a/molecule/default/tests/test_default.py b/molecule/default/tests/test_default.py
index 7797588..ef1a546 100644
--- a/molecule/default/tests/test_default.py
+++ b/molecule/default/tests/test_default.py
@@ -11,8 +11,7 @@ def test_directories(host):
]
for dir in dirs:
d = host.file(dir)
- assert d.is_directory
- assert d.exists
+ assert not d.exists
def test_files(host):
diff --git a/tasks/configure.yml b/tasks/configure.yml
index 9e9f9bc..ec9d826 100644
--- a/tasks/configure.yml
+++ b/tasks/configure.yml
@@ -8,16 +8,6 @@
mode: 0644
notify: restart node_exporter
-- name: Create textfile collector dir
- file:
- path: "{{ node_exporter_textfile_dir }}"
- state: directory
- owner: "{{ _node_exporter_system_user }}"
- group: "{{ _node_exporter_system_group }}"
- recurse: true
- mode: u+rwX,g+rwX,o=rX
- when: node_exporter_textfile_dir | length > 0
-
- name: Allow Node Exporter port in SELinux on RedHat OS family
seport:
ports: "{{ node_exporter_web_listen_address.split(':')[-1] }}"
@@ -27,3 +17,45 @@
when:
- ansible_version.full is version_compare('2.4', '>=')
- ansible_selinux.status == "enabled"
+
+- block:
+ - name: Create textfile collector dir
+ file:
+ path: "{{ node_exporter_textfile_dir }}"
+ state: directory
+ owner: "{{ _node_exporter_system_user }}"
+ group: "{{ _node_exporter_system_group }}"
+ recurse: true
+ mode: u+rwX,g+rwX,o=rX
+
+ - name: Download textfile collector scripts
+ copy:
+ src: "{{ item.src }}"
+ dest: "{{ _node_exporter_binary_install_dir }}/{{ item.src | basename }}"
+ owner: root
+ group: root
+ mode: "0755"
+ with_items: "{{ node_exporter_textfile_collectors }}"
+
+ - name: Setup textfile collector script systemd cron service
+ template:
+ src: "cron.service.j2"
+ dest: "/etc/systemd/systemd/textfile-collector-{{ item.src | basename }}.service"
+ with_items: "{{ node_exporter_textfile_collectors }}"
+ notify: reload cron
+
+ - name: Setup textfile collector script systemd cron timer
+ template:
+ src: "cron.timer.j2"
+ dest: "/etc/systemd/systemd/textfile-collector-{{ item.src | basename }}.timer"
+ with_items: "{{ node_exporter_textfile_collectors }}"
+ notify: reload cron
+
+ - name: Enable textfile collector
+ systemd:
+ daemon_reload: true
+ name: "textfile-collector-{{ item.src | basename }}.timer"
+ enabled: true
+ state: started
+ with_items: "{{ node_exporter_textfile_collectors }}"
+ when: node_exporter_textfile_dir | length > 0
diff --git a/tasks/preflight.yml b/tasks/preflight.yml
index 79fff42..ea2ca87 100644
--- a/tasks/preflight.yml
+++ b/tasks/preflight.yml
@@ -27,6 +27,23 @@
- "item not in node_exporter_enabled_collectors"
with_items: "{{ node_exporter_disabled_collectors }}"
+- block:
+ - name: Check for "/etc/crontab"
+ stat:
+ path: "/etc/crontab"
+ register: __node_exportert_etc_crontab
+
+ - name: Assert /etc/crontab exists when using textfile collector scripts
+ assert:
+ that:
+ __node_exportert_etc_crontab.stat.exists
+
+ - name: Assert node_exporter_textfile_dir is not empty when using collector scripts
+ assert:
+ that:
+ - node_exporter_textfile_dir | length != 0
+ when: node_exporter_textfile_collectors | length != 0
+
- name: Check if node_exporter is installed
stat:
path: "{{ _node_exporter_binary_install_dir }}/node_exporter"
diff --git a/templates/cron.service.j2 b/templates/cron.service.j2
new file mode 100644
index 0000000..9830a5f
--- /dev/null
+++ b/templates/cron.service.j2
@@ -0,0 +1,9 @@
+[Unit]
+Description=Run Textfile Collector {{ item.src | basename }}
+
+[Service]
+User={{ item.user | default("root") }}
+ExecStart=/bin/sh -c '\
+ TEMP=$(mktemp -q) && \
+ {{ _node_exporter_binary_install_dir }}/{{ item.src | basename }} > $TEMP && \
+ mv $TEMP {{ node_exporter_textfile_dir }}/{{ item.src | basename }}.prom'
diff --git a/templates/cron.timer.j2 b/templates/cron.timer.j2
new file mode 100644
index 0000000..52dbdd5
--- /dev/null
+++ b/templates/cron.timer.j2
@@ -0,0 +1,14 @@
+[Unit]
+Description=Textfile Collector {{ item.src | basename }} timer
+
+[Timer]
+Unit=textfile-collector-{{ item.src | basename }}.service
+OnCalendar={{ item.calendar }}
+OnBootSec={{ item.boot_sec }}
+OnActiveSec={{ item.active_sec }}
+OnStartupSec={{ item.startup_sec }}
+OnUnitActiveSec={{ item.unit_active_sec }}
+OnUnitInactiveSec={{ item.unit_inactive_sec }}
+
+[Install]
+WantedBy=multi-user.target
--
GitLab