forked from c3d2/nix-config
add beats logging to hypervisor
This commit is contained in:
parent
09c7d8453e
commit
f609e50bfa
|
@ -54,4 +54,4 @@
|
|||
}
|
||||
}
|
||||
vars:
|
||||
beats_version: 7.2.0
|
||||
beats_version: 7.3.0
|
||||
|
|
61
ansible/roles/elastic.beats/.ci/jobs/defaults.yml
Normal file
61
ansible/roles/elastic.beats/.ci/jobs/defaults.yml
Normal file
|
@ -0,0 +1,61 @@
|
|||
---
|
||||
|
||||
##### GLOBAL METADATA
|
||||
|
||||
- meta:
|
||||
cluster: devops-ci
|
||||
|
||||
##### JOB DEFAULTS
|
||||
|
||||
- job:
|
||||
project-type: matrix
|
||||
logrotate:
|
||||
daysToKeep: 30
|
||||
numToKeep: 100
|
||||
parameters:
|
||||
- string:
|
||||
name: branch_specifier
|
||||
default: master
|
||||
description: the Git branch specifier to build (<branchName>, <tagName>,
|
||||
<commitId>, etc.)
|
||||
properties:
|
||||
- github:
|
||||
url: https://github.com/elastic/ansible-beats/
|
||||
- inject:
|
||||
properties-content: HOME=$JENKINS_HOME
|
||||
concurrent: true
|
||||
node: master
|
||||
scm:
|
||||
- git:
|
||||
name: origin
|
||||
credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
|
||||
reference-repo: /var/lib/jenkins/.git-references/ansible-beats.git
|
||||
branches:
|
||||
- ${branch_specifier}
|
||||
url: git@github.com:elastic/ansible-beats.git
|
||||
basedir: ansible-beats
|
||||
wipe-workspace: 'False'
|
||||
axes:
|
||||
- axis:
|
||||
type: slave
|
||||
name: label
|
||||
values:
|
||||
- linux
|
||||
- axis:
|
||||
name: OS
|
||||
filename: ansible-beats/test/matrix.yml
|
||||
type: yaml
|
||||
- axis:
|
||||
name: TEST_TYPE
|
||||
filename: ansible-beats/test/matrix.yml
|
||||
type: yaml
|
||||
wrappers:
|
||||
- ansicolor
|
||||
- timeout:
|
||||
type: absolute
|
||||
timeout: 360
|
||||
fail: true
|
||||
- timestamps
|
||||
publishers:
|
||||
- email:
|
||||
recipients: infra-root+build@elastic.co
|
|
@ -0,0 +1,19 @@
|
|||
---
|
||||
- job:
|
||||
name: elastic+ansible-beats+master
|
||||
display-name: elastic / ansible-beats - master
|
||||
description: Master branch testing with test kitchen
|
||||
triggers:
|
||||
- timed: H H(02-04) * * *
|
||||
builders:
|
||||
- shell: |-
|
||||
#!/usr/local/bin/runbld
|
||||
set -euo pipefail
|
||||
|
||||
export RBENV_VERSION='2.3.0'
|
||||
export PATH="$HOME/.rbenv/bin:$PATH"
|
||||
eval "$(rbenv init -)"
|
||||
rbenv local $RBENV_VERSION
|
||||
|
||||
make setup
|
||||
make verify PATTERN=$TEST_TYPE-$OS
|
|
@ -0,0 +1,31 @@
|
|||
---
|
||||
- job:
|
||||
name: elastic+ansible-beats+pull-request
|
||||
display-name: elastic / ansible-beats - pull-request
|
||||
description: Pull request testing with test kitchen
|
||||
parameters: []
|
||||
scm:
|
||||
- git:
|
||||
branches:
|
||||
- $ghprbActualCommit
|
||||
refspec: +refs/pull/*:refs/remotes/origin/pr/*
|
||||
triggers:
|
||||
- github-pull-request:
|
||||
github-hooks: true
|
||||
org-list:
|
||||
- elastic
|
||||
allow-whitelist-orgs-as-admins: true
|
||||
cancel-builds-on-update: true
|
||||
status-context: devops-ci
|
||||
builders:
|
||||
- shell: |-
|
||||
#!/usr/local/bin/runbld
|
||||
set -euo pipefail
|
||||
|
||||
export RBENV_VERSION='2.3.0'
|
||||
export PATH="$HOME/.rbenv/bin:$PATH"
|
||||
eval "$(rbenv init -)"
|
||||
rbenv local $RBENV_VERSION
|
||||
|
||||
make setup
|
||||
make verify PATTERN=$TEST_TYPE-$OS
|
8
ansible/roles/elastic.beats/.gitignore
vendored
Normal file
8
ansible/roles/elastic.beats/.gitignore
vendored
Normal file
|
@ -0,0 +1,8 @@
|
|||
.kitchen/
|
||||
*.pyc
|
||||
.vendor
|
||||
.bundle
|
||||
Converging
|
||||
TODO
|
||||
.idea/
|
||||
beats.iml
|
102
ansible/roles/elastic.beats/.kitchen.yml
Normal file
102
ansible/roles/elastic.beats/.kitchen.yml
Normal file
|
@ -0,0 +1,102 @@
|
|||
---
|
||||
driver:
|
||||
name: docker
|
||||
|
||||
provisioner:
|
||||
name: ansible_playbook
|
||||
hosts: localhost
|
||||
roles_path: ./
|
||||
require_ansible_repo: true
|
||||
ansible_verbose: true
|
||||
idempotency_test: true
|
||||
|
||||
platforms:
|
||||
- name: ubuntu-14.04
|
||||
driver_config:
|
||||
image: ubuntu:14.04
|
||||
privileged: true
|
||||
provision_command:
|
||||
- apt-get update && apt-get install -y software-properties-common && add-apt-repository -y ppa:ansible/ansible
|
||||
- apt-get update && apt-get -y -q install ansible python-apt python-pycurl
|
||||
use_sudo: false
|
||||
- name: ubuntu-16.04
|
||||
driver_config:
|
||||
image: ubuntu:16.04
|
||||
privileged: true
|
||||
provision_command:
|
||||
- apt-get update && apt-get install -y software-properties-common && add-apt-repository -y ppa:ansible/ansible
|
||||
- apt-get update && apt-get -y -q install ansible python-apt python-pycurl
|
||||
use_sudo: false
|
||||
- name: ubuntu-18.04
|
||||
driver_config:
|
||||
image: ubuntu:18.04
|
||||
privileged: true
|
||||
provision_command:
|
||||
- apt-get update && apt-get install -y software-properties-common && add-apt-repository -y ppa:ansible/ansible
|
||||
- apt-get update && apt-get -y -q install ansible python-apt python-pycurl
|
||||
- mkdir -p /run/sshd
|
||||
use_sudo: false
|
||||
- name: debian-8
|
||||
driver_config:
|
||||
image: debian:8
|
||||
privileged: true
|
||||
provision_command:
|
||||
- echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible.list
|
||||
- apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367
|
||||
- apt-get update && apt-get -y install ansible
|
||||
use_sudo: false
|
||||
- name: debian-9
|
||||
driver_config:
|
||||
image: debian:9
|
||||
privileged: true
|
||||
provision_command:
|
||||
- apt-get update && apt-get -y install gnupg2
|
||||
- echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible.list
|
||||
- apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367
|
||||
- apt-get update && apt-get -y install ansible
|
||||
use_sudo: false
|
||||
- name: centos-6
|
||||
driver_config:
|
||||
image: centos:6
|
||||
privileged: true
|
||||
provision_command:
|
||||
- yum -y install epel-release
|
||||
- yum -y install ansible
|
||||
use_sudo: false
|
||||
- name: centos-7
|
||||
driver_config:
|
||||
image: centos:7
|
||||
provision_command:
|
||||
- yum -y install ansible
|
||||
run_command: "/usr/sbin/init"
|
||||
privileged: true
|
||||
use_sudo: false
|
||||
suites:
|
||||
- name: standard
|
||||
provisioner:
|
||||
playbook: test/integration/standard.yml
|
||||
additional_copy_path:
|
||||
- "."
|
||||
run_list:
|
||||
attributes:
|
||||
- name: standard-6x
|
||||
provisioner:
|
||||
playbook: test/integration/standard-6x.yml
|
||||
additional_copy_path:
|
||||
- "."
|
||||
run_list:
|
||||
attributes:
|
||||
- name: multi
|
||||
provisioner:
|
||||
playbook: test/integration/multi.yml
|
||||
additional_copy_path:
|
||||
- "."
|
||||
run_list:
|
||||
attributes:
|
||||
- name: config
|
||||
provisioner:
|
||||
playbook: test/integration/config.yml
|
||||
additional_copy_path:
|
||||
- "."
|
||||
run_list:
|
||||
attributes:
|
6
ansible/roles/elastic.beats/Gemfile
Normal file
6
ansible/roles/elastic.beats/Gemfile
Normal file
|
@ -0,0 +1,6 @@
|
|||
source 'https://rubygems.org'
|
||||
|
||||
gem 'test-kitchen', '1.21.1'
|
||||
gem "kitchen-docker", '2.6.0'
|
||||
gem 'kitchen-ansible', '0.48.3'
|
||||
gem 'net-ssh', '4.2.0'
|
75
ansible/roles/elastic.beats/Gemfile.lock
Normal file
75
ansible/roles/elastic.beats/Gemfile.lock
Normal file
|
@ -0,0 +1,75 @@
|
|||
GEM
|
||||
remote: https://rubygems.org/
|
||||
specs:
|
||||
builder (3.2.3)
|
||||
erubis (2.7.0)
|
||||
ffi (1.9.24)
|
||||
gssapi (1.2.0)
|
||||
ffi (>= 1.0.1)
|
||||
gyoku (1.3.1)
|
||||
builder (>= 2.1.2)
|
||||
httpclient (2.8.3)
|
||||
kitchen-ansible (0.48.3)
|
||||
net-ssh (>= 3)
|
||||
test-kitchen (~> 1.4)
|
||||
kitchen-docker (2.6.0)
|
||||
test-kitchen (>= 1.0.0)
|
||||
little-plugger (1.1.4)
|
||||
logging (2.2.2)
|
||||
little-plugger (~> 1.1)
|
||||
multi_json (~> 1.10)
|
||||
mixlib-install (3.9.3)
|
||||
mixlib-shellout
|
||||
mixlib-versioning
|
||||
thor
|
||||
mixlib-shellout (2.3.2)
|
||||
mixlib-versioning (1.2.2)
|
||||
multi_json (1.13.1)
|
||||
net-scp (1.2.1)
|
||||
net-ssh (>= 2.6.5)
|
||||
net-ssh (4.2.0)
|
||||
net-ssh-gateway (1.3.0)
|
||||
net-ssh (>= 2.6.5)
|
||||
nori (2.6.0)
|
||||
rubyntlm (0.6.2)
|
||||
rubyzip (1.2.2)
|
||||
test-kitchen (1.21.1)
|
||||
mixlib-install (~> 3.6)
|
||||
mixlib-shellout (>= 1.2, < 3.0)
|
||||
net-scp (~> 1.1)
|
||||
net-ssh (>= 2.9, < 5.0)
|
||||
net-ssh-gateway (~> 1.2)
|
||||
thor (~> 0.19, < 0.19.2)
|
||||
winrm (~> 2.0)
|
||||
winrm-elevated (~> 1.0)
|
||||
winrm-fs (~> 1.1)
|
||||
thor (0.19.1)
|
||||
winrm (2.2.3)
|
||||
builder (>= 2.1.2)
|
||||
erubis (~> 2.7)
|
||||
gssapi (~> 1.2)
|
||||
gyoku (~> 1.0)
|
||||
httpclient (~> 2.2, >= 2.2.0.2)
|
||||
logging (>= 1.6.1, < 3.0)
|
||||
nori (~> 2.0)
|
||||
rubyntlm (~> 0.6.0, >= 0.6.1)
|
||||
winrm-elevated (1.1.0)
|
||||
winrm (~> 2.0)
|
||||
winrm-fs (~> 1.0)
|
||||
winrm-fs (1.2.0)
|
||||
erubis (~> 2.7)
|
||||
logging (>= 1.6.1, < 3.0)
|
||||
rubyzip (~> 1.1)
|
||||
winrm (~> 2.0)
|
||||
|
||||
PLATFORMS
|
||||
ruby
|
||||
|
||||
DEPENDENCIES
|
||||
kitchen-ansible (= 0.48.3)
|
||||
kitchen-docker (= 2.6.0)
|
||||
net-ssh (= 4.2.0)
|
||||
test-kitchen (= 1.21.1)
|
||||
|
||||
BUNDLED WITH
|
||||
1.16.1
|
31
ansible/roles/elastic.beats/Makefile
Normal file
31
ansible/roles/elastic.beats/Makefile
Normal file
|
@ -0,0 +1,31 @@
|
|||
default: build
|
||||
|
||||
SHELL:=/bin/bash -eux
|
||||
PATTERN := standard-ubuntu-1804
|
||||
|
||||
.PHONY: converge verify test login destroy list
|
||||
|
||||
setup:
|
||||
bundle install
|
||||
docker ps
|
||||
|
||||
converge:
|
||||
bundle exec kitchen converge $(PATTERN)
|
||||
|
||||
verify:
|
||||
bundle exec kitchen verify $(PATTERN)
|
||||
|
||||
test:
|
||||
bundle exec kitchen test $(PATTERN) --destroy=always
|
||||
|
||||
login:
|
||||
bundle exec kitchen login $(PATTERN)
|
||||
|
||||
destroy:
|
||||
bundle exec kitchen destroy $(PATTERN)
|
||||
|
||||
destroy-all:
|
||||
bundle exec kitchen destroy
|
||||
|
||||
list:
|
||||
bundle exec kitchen list
|
159
ansible/roles/elastic.beats/README.md
Normal file
159
ansible/roles/elastic.beats/README.md
Normal file
|
@ -0,0 +1,159 @@
|
|||
Beats Role
|
||||
=========
|
||||
|
||||
This role provides a generic means of installing Elastic supported beats using Ansible. Currently this includes:
|
||||
|
||||
- Filebeat
|
||||
- MetricBeat (TopBeat in 1.x)
|
||||
- Packetbeat
|
||||
|
||||
Requirements
|
||||
------------
|
||||
|
||||
This role requires:
|
||||
|
||||
- Ansible version > 2.0
|
||||
- Beats version >= 1.x
|
||||
- Target machines require access to an external repo, or means
|
||||
|
||||
The following OS are currently actively tested using kitchen:
|
||||
|
||||
- Centos 6 & 7
|
||||
- Debian 8 & 9
|
||||
- Ubuntu 14.04, 16.04 & 18.04
|
||||
|
||||
The above list is subject to change.
|
||||
|
||||
Role Variables
|
||||
--------------
|
||||
|
||||
Supported variables are as follows:
|
||||
|
||||
- **beat** (*MANDATORY*): Beat product. Supported values are: "filebeat", "metricbeat" & "packetbeat".
|
||||
- **beat_conf** (*MANDATORY*): Beat Configuration. Should be defined as a map (see [Example Playbook](#example-playbook))
|
||||
- **beats_version** (*Defaults to `7.0.0`*): Beats version.
|
||||
- **version_lock** (*Defaults to `false`*): Locks the installed version if set to true, thus preventing other processes from updating. This will not impact the roles ability to update the beat on subsequent runs (it unlocks and re-locks if required).
|
||||
- **use_repository** (*Defaults to `true`*): Use elastic repo for yum or apt if true. If false, a custom custom_package_url must be provided.
|
||||
- **start_service** (*Defaults to `true`*): service will be started if true, false otherwise.
|
||||
- **restart_on_change** (*Defaults to `true`*): Changes to configuration or installed versions, will result in a restart if true.
|
||||
- **daemon_args** (*Applicable to version 1.x of beats*): Allows run time params to be passed to beats.
|
||||
- **logging_conf** (*Defaults to `{"files":{"rotateeverybytes":10485760}}`*): Logging configuration. Should be defined as a map (see [Example Playbook](#example-playbook)). Map is serialized into logging section of beat config.
|
||||
- **shipper_conf** (*Applicable to version 1.x of beats*): Shipper configuration. Should be defined as a map (see [Example Playbook](#example-playbook)). Map is serialized into shipper section of beat config.
|
||||
- **output_conf** (*Defaults to `{"elasticsearch":{"hosts":["localhost:9200"]}}`): Output configuration. Should be defined as a map (see [Example Playbook](#example-playbook)). Map is serialized into output section of beat config.
|
||||
- **beats_pid_dir** (*Defaults to `/var/run`*): Location of beats pid file.
|
||||
- **beats_conf_dir** (*Defaults to `/etc/{beat}`*): Location of conf directory for beats configuration file.
|
||||
|
||||
Dependencies
|
||||
------------
|
||||
|
||||
- Ansible version > 2.0
|
||||
- Beats version >= 6.x
|
||||
|
||||
Example Playbook
|
||||
----------------
|
||||
|
||||
Example playbook is provided below. This installs Packetbeat and illustrates the need for configuration sections to be specified as maps.
|
||||
|
||||
- name: Example playbook for installing packetbeat
|
||||
hosts: localhost
|
||||
roles:
|
||||
- { role: beats, beat: "packetbeat",
|
||||
beat_conf: {
|
||||
"interfaces": {"device":"any"},
|
||||
"protocols": {
|
||||
"dns": {
|
||||
"ports": [53],
|
||||
"include_authorities":true
|
||||
},
|
||||
"http": {
|
||||
"ports": [80, 8080, 8000, 5000, 8002]
|
||||
},
|
||||
"memcache": {
|
||||
"ports": [11211]
|
||||
},
|
||||
"mysql": {
|
||||
"ports": [3306]
|
||||
},
|
||||
"pgsql": {
|
||||
"ports": [5432]
|
||||
},
|
||||
"redis": {
|
||||
"ports": [6379]
|
||||
},
|
||||
"thrift": {
|
||||
"ports": [9090]
|
||||
},
|
||||
"mongodb": {
|
||||
"ports": [27017]
|
||||
}
|
||||
}
|
||||
},
|
||||
output_conf : {
|
||||
"elasticsearch": {
|
||||
"hosts": ["localhost:9200"]
|
||||
}
|
||||
}
|
||||
}
|
||||
vars:
|
||||
use_repository: "true"
|
||||
|
||||
## Testing
|
||||
|
||||
This playbook uses [Kitchen](https://kitchen.ci/) for CI and local testing.
|
||||
|
||||
### Requirements
|
||||
|
||||
* Ruby
|
||||
* Bundler
|
||||
* Docker
|
||||
* Make
|
||||
|
||||
### Running the tests
|
||||
|
||||
To converge an Ubuntu 18.04 host
|
||||
```sh
|
||||
$ make converge
|
||||
```
|
||||
|
||||
To run the tests
|
||||
```sh
|
||||
$ make verify
|
||||
```
|
||||
|
||||
To list all of the different test suits
|
||||
```sh
|
||||
$ make list
|
||||
```
|
||||
|
||||
The default test suite is Ubuntu 18.04. If you want to test another suite you can override this with the `PATTERN` variable
|
||||
```sh
|
||||
$ make converge PATTERN=standard-centos-7
|
||||
```
|
||||
|
||||
The `PATTERN` is a kitchen pattern which can match multiple suites. To run all tests for CentOS
|
||||
```sh
|
||||
$ make converge PATTERN=centos-7
|
||||
```
|
||||
|
||||
When you are finished testing you can clean up everything with
|
||||
```sh
|
||||
$ make destroy-all
|
||||
```
|
||||
|
||||
License
|
||||
-------
|
||||
|
||||
Apache 2.0
|
||||
|
||||
Limitations
|
||||
------------
|
||||
|
||||
Multiple instances of the same beat cannot be installed on the same target server.
|
||||
|
||||
Author Information
|
||||
------------------
|
||||
|
||||
Dale McDiarmid
|
||||
For further assistance, outside of issues, please use discuss.elastic.co.
|
||||
|
||||
Community Contributions always appreciated and welcome! Please ensure all contributions include tests as appropriate.
|
1
ansible/roles/elastic.beats/ansible.cfg
Normal file
1
ansible/roles/elastic.beats/ansible.cfg
Normal file
|
@ -0,0 +1 @@
|
|||
[defaults]
|
13
ansible/roles/elastic.beats/defaults/main.yml
Normal file
13
ansible/roles/elastic.beats/defaults/main.yml
Normal file
|
@ -0,0 +1,13 @@
|
|||
---
|
||||
# defaults file for beats
|
||||
|
||||
beats_version: 7.0.0
|
||||
version_lock: false
|
||||
use_repository: true
|
||||
start_service: true
|
||||
restart_on_change: true
|
||||
daemon_args: ""
|
||||
logging_conf: {"files":{"rotateeverybytes":10485760}}
|
||||
output_conf: {"elasticsearch":{"hosts":["localhost:9200"]}}
|
||||
beats_pid_dir: "/var/run"
|
||||
beats_conf_dir: "/etc/{{beat}}"
|
8
ansible/roles/elastic.beats/filter_plugins/custom.py
Normal file
8
ansible/roles/elastic.beats/filter_plugins/custom.py
Normal file
|
@ -0,0 +1,8 @@
|
|||
__author__ = 'dale mcdiarmid'
|
||||
|
||||
def merge_config(config={}, key=''):
|
||||
return {key:config}
|
||||
|
||||
class FilterModule(object):
|
||||
def filters(self):
|
||||
return {'merge_config': merge_config}
|
6
ansible/roles/elastic.beats/handlers/main.yml
Normal file
6
ansible/roles/elastic.beats/handlers/main.yml
Normal file
|
@ -0,0 +1,6 @@
|
|||
---
|
||||
# handlers file for beats
|
||||
|
||||
- name: restart beat
|
||||
service: name={{ beat }} state=restarted enabled=yes
|
||||
when: start_service and restart_on_change and not beats_started.changed and beat_install.changed
|
2
ansible/roles/elastic.beats/meta/.galaxy_install_info
Normal file
2
ansible/roles/elastic.beats/meta/.galaxy_install_info
Normal file
|
@ -0,0 +1,2 @@
|
|||
install_date: Sat Jul 6 01:04:40 2019
|
||||
version: 7.0.0
|
28
ansible/roles/elastic.beats/meta/main.yml
Normal file
28
ansible/roles/elastic.beats/meta/main.yml
Normal file
|
@ -0,0 +1,28 @@
|
|||
---
|
||||
|
||||
allow_duplicates: yes
|
||||
|
||||
galaxy_info:
|
||||
role_name: beats
|
||||
author: Dale McDiarmid
|
||||
description: Beats for Linux
|
||||
company: "Elastic.co"
|
||||
license: "license (Apache)"
|
||||
min_ansible_version: 2.0
|
||||
platforms:
|
||||
- name: EL
|
||||
versions:
|
||||
- 6
|
||||
- 7
|
||||
- name: Debian
|
||||
versions:
|
||||
- all
|
||||
- name: Ubuntu
|
||||
versions:
|
||||
- all
|
||||
categories:
|
||||
- beats
|
||||
- elastic
|
||||
- elk
|
||||
- logging
|
||||
- monitoring
|
33
ansible/roles/elastic.beats/tasks/beats-config.yml
Normal file
33
ansible/roles/elastic.beats/tasks/beats-config.yml
Normal file
|
@ -0,0 +1,33 @@
|
|||
---
|
||||
# Configure Beats Node
|
||||
|
||||
- set_fact: pid_file={{ beats_pid_dir }}/{{beat}}.pid
|
||||
- set_fact: instance_default_file={{ default_file }}/{{beat}}
|
||||
- set_fact: conf_file={{ beats_conf_dir }}/{{beat}}.yml
|
||||
|
||||
- set_fact: beat_output_conf={{ output_conf | merge_config('output') }}
|
||||
- set_fact: beat_shipper_conf={{ shipper_conf | merge_config('shipper') }}
|
||||
when: shipper_conf is defined
|
||||
- set_fact: beat_logging_conf={{ logging_conf | merge_config('logging') }}
|
||||
|
||||
- stat: path={{beats_pid_dir}}
|
||||
register: pid_stat
|
||||
|
||||
- name: Create PID Directory
|
||||
file: path={{ beats_pid_dir }} state=directory
|
||||
when: pid_stat.stat.isdir is not defined or pid_stat.stat.islnk is not defined
|
||||
|
||||
#fail if pid and config directories are not links or not directories i.e files
|
||||
|
||||
- name: Create Config Directory
|
||||
file: path={{ beats_conf_dir }} state=directory
|
||||
|
||||
#Copy the default file
|
||||
- name: Copy Default File for Instance
|
||||
template: src=beat.j2 dest={{instance_default_file}} mode=0644 force=yes owner=root group=root
|
||||
notify: restart beat
|
||||
|
||||
#Copy templated config file
|
||||
- name: Copy Configuration File
|
||||
template: src=beat.yml.j2 dest={{conf_file}} mode=0644 force=yes owner=root group=root
|
||||
notify: restart beat
|
45
ansible/roles/elastic.beats/tasks/beats-debian.yml
Normal file
45
ansible/roles/elastic.beats/tasks/beats-debian.yml
Normal file
|
@ -0,0 +1,45 @@
|
|||
---
|
||||
|
||||
- name: Debian - Ensure apt-transport-https is installed
|
||||
apt: name=apt-transport-https state=present cache_valid_time=86400
|
||||
when: use_repository
|
||||
register: beat_install
|
||||
notify: restart beat
|
||||
|
||||
- name: Debian - Add Beats repository key
|
||||
apt_key: url="http://packages.elasticsearch.org/GPG-KEY-elasticsearch" state=present
|
||||
when: use_repository
|
||||
|
||||
- name: Debian - add beats repository
|
||||
apt_repository: repo="deb {{ repo_url }} stable main" state=present
|
||||
when: use_repository
|
||||
|
||||
- name: Debian - unhold {{beat}} version for install
|
||||
command: apt-mark unhold {{beat}}
|
||||
changed_when: False
|
||||
|
||||
- name: Debian - Ensure {{beat}} is installed
|
||||
apt: name={{beat}}{% if beats_version is defined and beats_version != "" %}={{ beats_version }}{% endif %} state=present cache_valid_time=86400
|
||||
when: use_repository
|
||||
register: beat_install
|
||||
notify: restart beat
|
||||
|
||||
- name: Debian - hold {{beat}} version
|
||||
command: apt-mark hold {{beat}}
|
||||
when: version_lock
|
||||
changed_when: False
|
||||
|
||||
- set_fact: os_arch="i386"
|
||||
|
||||
- set_fact: os_arch="amd64"
|
||||
when: ansible_architecture == "x86_64"
|
||||
|
||||
- name: Debian - Download {{beat}} from url
|
||||
get_url: url={% if custom_package_url is defined %}{{ custom_package_url }}{% else %}{{ beats_package_url }}/{{beat}}/{{beat}}_{{ beats_version }}_{{os_arch}}.deb{% endif %} dest=/tmp/{{beat}}_{{ beats_version }}_{{os_arch}}.deb validate_certs=no
|
||||
when: not use_repository
|
||||
|
||||
- name: Debian - Ensure {{beat}} is installed from downloaded package
|
||||
apt: deb=/tmp/{{beat}}_{{ beats_version }}_{{os_arch}}.deb
|
||||
when: not use_repository
|
||||
register: beat_install
|
||||
notify: restart beat
|
9
ansible/roles/elastic.beats/tasks/beats-param-check.yml
Normal file
9
ansible/roles/elastic.beats/tasks/beats-param-check.yml
Normal file
|
@ -0,0 +1,9 @@
|
|||
---
|
||||
|
||||
- fail: msg="beat must be specified and cannot be blank e.g. filebeat"
|
||||
when: beat is not defined or beat == ''
|
||||
|
||||
- fail: msg="beat_conf must be specified"
|
||||
when: beat_conf is not defined
|
||||
|
||||
- set_fact: "beats_major_version={{ beats_version[0] }}.x"
|
33
ansible/roles/elastic.beats/tasks/beats-redhat.yml
Normal file
33
ansible/roles/elastic.beats/tasks/beats-redhat.yml
Normal file
|
@ -0,0 +1,33 @@
|
|||
---
|
||||
|
||||
- name: Ensure libselinux-python on CentOS 6.x
|
||||
yum: name=libselinux-python state=present update_cache=yes
|
||||
when: ( ansible_distribution == "CentOS" ) and ( ansible_distribution_major_version == "6" )
|
||||
|
||||
- name: Redhat - add beats repository
|
||||
template: src=beats.repo.j2 dest=/etc/yum.repos.d/beats.repo
|
||||
when: use_repository
|
||||
|
||||
- name: RedHat - install yum-version-lock
|
||||
yum: name=yum-plugin-versionlock state=present update_cache=yes
|
||||
|
||||
- name: RedHat - unlock {{beat}} for install
|
||||
shell: yum versionlock delete {{beat}} || true
|
||||
changed_when: False
|
||||
|
||||
- name: RedHat - Ensure {{beat}} is installed
|
||||
yum: name={{beat}}{% if beats_version is defined and beats_version != "" %}-{{ beats_version }}{% endif %} state=present update_cache=yes
|
||||
when: use_repository
|
||||
register: beat_install
|
||||
notify: restart beat
|
||||
|
||||
- name: RedHat - lock {{beat}} version
|
||||
shell: yum versionlock add {{beat}}{% if beats_version is defined and beats_version != "" %}-{{ beats_version }}{% endif %}
|
||||
when: version_lock
|
||||
changed_when: False
|
||||
|
||||
- name: RedHat - Install {{beat}} from url
|
||||
yum: name={% if custom_package_url is defined %}{{ custom_package_url }}{% else %}{{ beats_package_url }}/{{beat}}-{{ beats_version }}-{{ansible_architecture}}.rpm{% endif %} state=present
|
||||
when: not use_repository
|
||||
register: beat_install
|
||||
notify: restart beat
|
20
ansible/roles/elastic.beats/tasks/beats.yml
Normal file
20
ansible/roles/elastic.beats/tasks/beats.yml
Normal file
|
@ -0,0 +1,20 @@
|
|||
---
|
||||
# Install OS specific beats
|
||||
|
||||
- name: Include specific Beats
|
||||
include: beats-debian.yml
|
||||
when: ansible_os_family == 'Debian'
|
||||
|
||||
- name: Include specific Beats
|
||||
include: beats-redhat.yml
|
||||
when: ansible_os_family == 'RedHat'
|
||||
|
||||
#Configuration file for beats
|
||||
- name: Beats configuration
|
||||
include: beats-config.yml
|
||||
|
||||
# Make sure the service is started, and restart if necessary
|
||||
- name: Start beats service
|
||||
service: name={{ beat }} state=started enabled=yes
|
||||
when: start_service
|
||||
register: beats_started
|
10
ansible/roles/elastic.beats/tasks/main.yml
Normal file
10
ansible/roles/elastic.beats/tasks/main.yml
Normal file
|
@ -0,0 +1,10 @@
|
|||
---
|
||||
# tasks file for beats
|
||||
|
||||
- name: check-parameters
|
||||
include: beats-param-check.yml
|
||||
|
||||
- name: os-specific vars
|
||||
include_vars: "{{ansible_os_family}}.yml"
|
||||
|
||||
- include: beats.yml
|
7
ansible/roles/elastic.beats/templates/beat.j2
Normal file
7
ansible/roles/elastic.beats/templates/beat.j2
Normal file
|
@ -0,0 +1,7 @@
|
|||
################################
|
||||
# {{beat}}
|
||||
################################
|
||||
|
||||
# Beats PID File
|
||||
PIDFILE={{pid_file}}
|
||||
DAEMON_ARGS="-c {{conf_file}} {{daemon_args}}"
|
20
ansible/roles/elastic.beats/templates/beat.yml.j2
Normal file
20
ansible/roles/elastic.beats/templates/beat.yml.j2
Normal file
|
@ -0,0 +1,20 @@
|
|||
################### {{beat}} Configuration #########################
|
||||
|
||||
############################# {{beat}} ######################################
|
||||
{{beat_conf | to_nice_yaml}}
|
||||
|
||||
###############################################################################
|
||||
############################# Libbeat Config ##################################
|
||||
# Base config file used by all other beats for using libbeat features
|
||||
|
||||
############################# Output ##########################################
|
||||
|
||||
{{ beat_output_conf | to_nice_yaml }}
|
||||
|
||||
{% if shipper_conf is defined %}############################# Shipper #########################################
|
||||
|
||||
{{ beat_shipper_conf | to_nice_yaml }}
|
||||
{% endif %}
|
||||
############################# Logging #########################################
|
||||
|
||||
{{ beat_logging_conf | to_nice_yaml }}
|
6
ansible/roles/elastic.beats/templates/beats.repo.j2
Normal file
6
ansible/roles/elastic.beats/templates/beats.repo.j2
Normal file
|
@ -0,0 +1,6 @@
|
|||
[beats]
|
||||
name=Elastic Beats Repository
|
||||
baseurl={{ repo_url }}
|
||||
enabled=1
|
||||
gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch
|
||||
gpgcheck=1
|
44
ansible/roles/elastic.beats/test/integration/config.yml
Normal file
44
ansible/roles/elastic.beats/test/integration/config.yml
Normal file
|
@ -0,0 +1,44 @@
|
|||
---
|
||||
#Install specific version here
|
||||
- name: wrapper playbook for kitchen testing "beats"
|
||||
hosts: localhost
|
||||
roles:
|
||||
- { role: "ansible-beats", beat: "packetbeat", version_lock: true,
|
||||
beat_conf: {
|
||||
"interfaces": {"device":"any"},
|
||||
"protocols": {
|
||||
"dns": {
|
||||
"ports": [53],
|
||||
"include_authorities":true
|
||||
},
|
||||
"http": {
|
||||
"ports": [80, 8080, 8000, 5000, 8002]
|
||||
},
|
||||
"memcache": {
|
||||
"ports": [11211]
|
||||
},
|
||||
"mysql": {
|
||||
"ports": [3306]
|
||||
},
|
||||
"pgsql": {
|
||||
"ports": [5432]
|
||||
},
|
||||
"redis": {
|
||||
"ports": [6379]
|
||||
},
|
||||
"thrift": {
|
||||
"ports": [9090]
|
||||
},
|
||||
"mongodb": {
|
||||
"ports": [27017]
|
||||
}
|
||||
}
|
||||
},
|
||||
output_conf : {
|
||||
"elasticsearch": {
|
||||
"hosts": ["localhost:9200"]
|
||||
}
|
||||
}
|
||||
}
|
||||
vars:
|
||||
use_repository: "true"
|
|
@ -0,0 +1,49 @@
|
|||
require 'spec_helper'
|
||||
|
||||
describe 'Config Tests' do
|
||||
|
||||
describe service('packetbeat') do
|
||||
it { should be_running }
|
||||
end
|
||||
|
||||
describe package('packetbeat') do
|
||||
it { should be_installed }
|
||||
end
|
||||
|
||||
describe file('/etc/packetbeat/packetbeat.yml') do
|
||||
it { should be_file }
|
||||
it { should be_owned_by 'root' }
|
||||
end
|
||||
|
||||
describe file('/etc/packetbeat/packetbeat.yml') do
|
||||
it { should contain 'logging:' }
|
||||
it { should contain 'output:' }
|
||||
it { should contain 'protocols:' }
|
||||
it { should contain 'dns:' }
|
||||
it { should contain 'memcache:' }
|
||||
it { should contain 'http:' }
|
||||
it { should contain 'mongodb:' }
|
||||
it { should contain 'mysql:' }
|
||||
it { should contain 'pgsql:' }
|
||||
it { should contain 'redis:' }
|
||||
it { should contain 'thrift:' }
|
||||
it { should contain 'interfaces:' }
|
||||
it { should contain 'device: any' }
|
||||
end
|
||||
|
||||
describe file('/etc/init.d/packetbeat') do
|
||||
it { should exist }
|
||||
end
|
||||
|
||||
if os[:family] == 'redhat'
|
||||
describe command('yum versionlock list | grep packetbeat') do
|
||||
its(:stdout) { should match /packetbeat/ }
|
||||
end
|
||||
elsif ['debian', 'ubuntu'].include?(os[:family])
|
||||
describe command('sudo apt-mark showhold | grep packetbeat') do
|
||||
its(:stdout) { should match /packetbeat/ }
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
source 'https://rubygems.org'
|
||||
|
||||
gem 'rspec-retry'
|
|
@ -0,0 +1,11 @@
|
|||
require 'serverspec'
|
||||
set :backend, :exec
|
||||
|
||||
require 'rspec/retry'
|
||||
|
||||
RSpec.configure do |config|
|
||||
# show retry status in spec process
|
||||
config.verbose_retry = true
|
||||
# show exception that triggers a retry if verbose_retry is set to true
|
||||
config.display_try_failure_messages = true
|
||||
end
|
8
ansible/roles/elastic.beats/test/integration/multi.yml
Normal file
8
ansible/roles/elastic.beats/test/integration/multi.yml
Normal file
|
@ -0,0 +1,8 @@
|
|||
---
|
||||
- name: wrapper playbook for kitchen testing "beats"
|
||||
hosts: localhost
|
||||
roles:
|
||||
- { role: "ansible-beats", beat: "filebeat", beat_conf: {"filebeat": {"inputs":[{"paths":["/var/log/*.log"],"type":"log"}]} } }
|
||||
- { role: "ansible-beats", beat: "metricbeat", beat_conf: { "metricbeat.modules": [{"module": "system", "metricsets": ["cpu", "filesystem", "network", "process"], "enabled": true, "period": "10s", "processes":[".*"], "cpu_ticks": false } ] } }
|
||||
vars:
|
||||
use_repository: "true"
|
|
@ -0,0 +1,57 @@
|
|||
require 'spec_helper'
|
||||
|
||||
describe 'Multi Tests' do
|
||||
|
||||
describe service('filebeat') do
|
||||
it { should be_running }
|
||||
end
|
||||
|
||||
describe package('filebeat') do
|
||||
it { should be_installed }
|
||||
end
|
||||
|
||||
describe file('/etc/filebeat/filebeat.yml') do
|
||||
it { should be_file }
|
||||
it { should be_owned_by 'root' }
|
||||
end
|
||||
|
||||
describe file('/etc/filebeat/filebeat.yml') do
|
||||
it { should contain 'filebeat:' }
|
||||
it { should contain 'logging:' }
|
||||
it { should contain 'output:' }
|
||||
end
|
||||
|
||||
describe file('/etc/init.d/filebeat') do
|
||||
it { should exist }
|
||||
end
|
||||
|
||||
describe service('metricbeat') do
|
||||
it { should be_running }
|
||||
end
|
||||
|
||||
describe package('metricbeat') do
|
||||
it { should be_installed }
|
||||
end
|
||||
|
||||
describe file('/etc/metricbeat/metricbeat.yml') do
|
||||
it { should be_file }
|
||||
it { should be_owned_by 'root' }
|
||||
end
|
||||
|
||||
describe file('/etc/metricbeat/metricbeat.yml') do
|
||||
it { should contain 'module: system' }
|
||||
it { should contain 'metricsets:' }
|
||||
it { should contain 'period: 10s' }
|
||||
it { should contain 'processes:' }
|
||||
it { should contain 'cpu_ticks:' }
|
||||
it { should contain 'logging:' }
|
||||
it { should contain 'output:' }
|
||||
end
|
||||
|
||||
describe file('/etc/init.d/metricbeat') do
|
||||
it { should exist }
|
||||
end
|
||||
|
||||
|
||||
end
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
---
|
||||
- name: wrapper playbook for kitchen testing "beats"
|
||||
hosts: localhost
|
||||
roles:
|
||||
- { role: "ansible-beats", beat: "filebeat", beat_conf: {"filebeat": {"prospectors":[{"paths":["/var/log/*.log"],"input_type":"log"}], "registry_file": "/var/lib/filebeat/registry"} } }
|
||||
vars:
|
||||
beats_version: 6.7.1
|
||||
use_repository: "true"
|
|
@ -0,0 +1,39 @@
|
|||
require 'spec_helper'
|
||||
|
||||
describe 'Standard Tests' do
|
||||
|
||||
describe service('filebeat') do
|
||||
it { should be_running }
|
||||
end
|
||||
|
||||
describe package('filebeat') do
|
||||
it { should be_installed }
|
||||
end
|
||||
|
||||
describe file('/etc/filebeat/filebeat.yml') do
|
||||
it { should be_file }
|
||||
it { should be_owned_by 'root' }
|
||||
end
|
||||
|
||||
describe file('/etc/filebeat/filebeat.yml') do
|
||||
it { should contain 'filebeat:' }
|
||||
it { should contain 'logging:' }
|
||||
it { should contain 'output:' }
|
||||
end
|
||||
|
||||
describe file('/etc/init.d/filebeat') do
|
||||
it { should exist }
|
||||
end
|
||||
|
||||
if os[:family] == 'redhat'
|
||||
describe command('yum versionlock list | grep filebeat') do
|
||||
its(:stdout) { should_not match /filebeat/ }
|
||||
end
|
||||
elsif ['debian', 'ubuntu'].include?(os[:family])
|
||||
describe command('sudo apt-mark showhold | grep filebeat') do
|
||||
its(:stdout) { should_not match /filebeat/ }
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
---
|
||||
- name: wrapper playbook for kitchen testing "beats"
|
||||
hosts: localhost
|
||||
roles:
|
||||
- { role: "ansible-beats", beat: "filebeat", beat_conf: {"filebeat": {"inputs":[{"paths":["/var/log/*.log"],"type":"log"}]} } }
|
||||
vars:
|
||||
use_repository: "true"
|
|
@ -0,0 +1,39 @@
|
|||
require 'spec_helper'
|
||||
|
||||
describe 'Standard Tests' do
|
||||
|
||||
describe service('filebeat') do
|
||||
it { should be_running }
|
||||
end
|
||||
|
||||
describe package('filebeat') do
|
||||
it { should be_installed }
|
||||
end
|
||||
|
||||
describe file('/etc/filebeat/filebeat.yml') do
|
||||
it { should be_file }
|
||||
it { should be_owned_by 'root' }
|
||||
end
|
||||
|
||||
describe file('/etc/filebeat/filebeat.yml') do
|
||||
it { should contain 'filebeat:' }
|
||||
it { should contain 'logging:' }
|
||||
it { should contain 'output:' }
|
||||
end
|
||||
|
||||
describe file('/etc/init.d/filebeat') do
|
||||
it { should exist }
|
||||
end
|
||||
|
||||
if os[:family] == 'redhat'
|
||||
describe command('yum versionlock list | grep filebeat') do
|
||||
its(:stdout) { should_not match /filebeat/ }
|
||||
end
|
||||
elsif ['debian', 'ubuntu'].include?(os[:family])
|
||||
describe command('sudo apt-mark showhold | grep filebeat') do
|
||||
its(:stdout) { should_not match /filebeat/ }
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
13
ansible/roles/elastic.beats/test/matrix.yml
Normal file
13
ansible/roles/elastic.beats/test/matrix.yml
Normal file
|
@ -0,0 +1,13 @@
|
|||
OS:
|
||||
- ubuntu-1404
|
||||
- ubuntu-1604
|
||||
- ubuntu-1804
|
||||
- debian-8
|
||||
- debian-9
|
||||
- centos-6
|
||||
- centos-7
|
||||
TEST_TYPE:
|
||||
- standard
|
||||
- standard-6x
|
||||
- multi
|
||||
- config
|
5
ansible/roles/elastic.beats/vars/Debian.yml
Normal file
5
ansible/roles/elastic.beats/vars/Debian.yml
Normal file
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
|
||||
default_file: "/etc/default"
|
||||
init_script: "/etc/init.d"
|
||||
repo_url: "https://artifacts.elastic.co/packages/{{ beats_major_version }}/apt"
|
5
ansible/roles/elastic.beats/vars/RedHat.yml
Normal file
5
ansible/roles/elastic.beats/vars/RedHat.yml
Normal file
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
|
||||
default_file: "/etc/sysconfig"
|
||||
init_script: "/etc/init.d"
|
||||
repo_url: "https://artifacts.elastic.co/packages/{{ beats_major_version }}/yum"
|
5
ansible/roles/elastic.beats/vars/main.yml
Normal file
5
ansible/roles/elastic.beats/vars/main.yml
Normal file
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
# vars file for beats
|
||||
|
||||
beats_package_url: "https://download.elastic.co/beats"
|
||||
es_conf_dir: "/etc"
|
Loading…
Reference in New Issue
Block a user