diff --git a/ansible/hypervisor.yml b/ansible/hypervisor.yml index 444256c1..51ec9196 100644 --- a/ansible/hypervisor.yml +++ b/ansible/hypervisor.yml @@ -54,4 +54,4 @@ } } vars: - beats_version: 7.2.0 + beats_version: 7.3.0 diff --git a/ansible/roles/elastic.beats/.ci/jobs/defaults.yml b/ansible/roles/elastic.beats/.ci/jobs/defaults.yml new file mode 100644 index 00000000..fb1838cb --- /dev/null +++ b/ansible/roles/elastic.beats/.ci/jobs/defaults.yml @@ -0,0 +1,61 @@ +--- + +##### GLOBAL METADATA + +- meta: + cluster: devops-ci + +##### JOB DEFAULTS + +- job: + project-type: matrix + logrotate: + daysToKeep: 30 + numToKeep: 100 + parameters: + - string: + name: branch_specifier + default: master + description: the Git branch specifier to build (<branchName>, <tagName>, + <commitId>, etc.) + properties: + - github: + url: https://github.com/elastic/ansible-beats/ + - inject: + properties-content: HOME=$JENKINS_HOME + concurrent: true + node: master + scm: + - git: + name: origin + credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba + reference-repo: /var/lib/jenkins/.git-references/ansible-beats.git + branches: + - ${branch_specifier} + url: git@github.com:elastic/ansible-beats.git + basedir: ansible-beats + wipe-workspace: 'False' + axes: + - axis: + type: slave + name: label + values: + - linux + - axis: + name: OS + filename: ansible-beats/test/matrix.yml + type: yaml + - axis: + name: TEST_TYPE + filename: ansible-beats/test/matrix.yml + type: yaml + wrappers: + - ansicolor + - timeout: + type: absolute + timeout: 360 + fail: true + - timestamps + publishers: + - email: + recipients: infra-root+build@elastic.co diff --git a/ansible/roles/elastic.beats/.ci/jobs/elastic+ansible-beats+master.yml b/ansible/roles/elastic.beats/.ci/jobs/elastic+ansible-beats+master.yml new file mode 100644 index 00000000..5c0979e1 --- /dev/null +++ b/ansible/roles/elastic.beats/.ci/jobs/elastic+ansible-beats+master.yml @@ -0,0 +1,19 @@ +--- +- job: + name: elastic+ansible-beats+master + display-name: elastic / ansible-beats - master + description: Master branch testing with test kitchen + triggers: + - timed: H H(02-04) * * * + builders: + - shell: |- + #!/usr/local/bin/runbld + set -euo pipefail + + export RBENV_VERSION='2.3.0' + export PATH="$HOME/.rbenv/bin:$PATH" + eval "$(rbenv init -)" + rbenv local $RBENV_VERSION + + make setup + make verify PATTERN=$TEST_TYPE-$OS diff --git a/ansible/roles/elastic.beats/.ci/jobs/elastic+ansible-beats+pull-request.yml b/ansible/roles/elastic.beats/.ci/jobs/elastic+ansible-beats+pull-request.yml new file mode 100644 index 00000000..55edb8a6 --- /dev/null +++ b/ansible/roles/elastic.beats/.ci/jobs/elastic+ansible-beats+pull-request.yml @@ -0,0 +1,31 @@ +--- +- job: + name: elastic+ansible-beats+pull-request + display-name: elastic / ansible-beats - pull-request + description: Pull request testing with test kitchen + parameters: [] + scm: + - git: + branches: + - $ghprbActualCommit + refspec: +refs/pull/*:refs/remotes/origin/pr/* + triggers: + - github-pull-request: + github-hooks: true + org-list: + - elastic + allow-whitelist-orgs-as-admins: true + cancel-builds-on-update: true + status-context: devops-ci + builders: + - shell: |- + #!/usr/local/bin/runbld + set -euo pipefail + + export RBENV_VERSION='2.3.0' + export PATH="$HOME/.rbenv/bin:$PATH" + eval "$(rbenv init -)" + rbenv local $RBENV_VERSION + + make setup + make verify PATTERN=$TEST_TYPE-$OS diff --git a/ansible/roles/elastic.beats/.gitignore b/ansible/roles/elastic.beats/.gitignore new file mode 100644 index 00000000..6bc9fb39 --- /dev/null +++ b/ansible/roles/elastic.beats/.gitignore @@ -0,0 +1,8 @@ +.kitchen/ +*.pyc +.vendor +.bundle +Converging +TODO +.idea/ +beats.iml \ No newline at end of file diff --git a/ansible/roles/elastic.beats/.kitchen.yml b/ansible/roles/elastic.beats/.kitchen.yml new file mode 100644 index 00000000..cd35ebf5 --- /dev/null +++ b/ansible/roles/elastic.beats/.kitchen.yml @@ -0,0 +1,102 @@ +--- +driver: + name: docker + +provisioner: + name: ansible_playbook + hosts: localhost + roles_path: ./ + require_ansible_repo: true + ansible_verbose: true + idempotency_test: true + +platforms: + - name: ubuntu-14.04 + driver_config: + image: ubuntu:14.04 + privileged: true + provision_command: + - apt-get update && apt-get install -y software-properties-common && add-apt-repository -y ppa:ansible/ansible + - apt-get update && apt-get -y -q install ansible python-apt python-pycurl + use_sudo: false + - name: ubuntu-16.04 + driver_config: + image: ubuntu:16.04 + privileged: true + provision_command: + - apt-get update && apt-get install -y software-properties-common && add-apt-repository -y ppa:ansible/ansible + - apt-get update && apt-get -y -q install ansible python-apt python-pycurl + use_sudo: false + - name: ubuntu-18.04 + driver_config: + image: ubuntu:18.04 + privileged: true + provision_command: + - apt-get update && apt-get install -y software-properties-common && add-apt-repository -y ppa:ansible/ansible + - apt-get update && apt-get -y -q install ansible python-apt python-pycurl + - mkdir -p /run/sshd + use_sudo: false + - name: debian-8 + driver_config: + image: debian:8 + privileged: true + provision_command: + - echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible.list + - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367 + - apt-get update && apt-get -y install ansible + use_sudo: false + - name: debian-9 + driver_config: + image: debian:9 + privileged: true + provision_command: + - apt-get update && apt-get -y install gnupg2 + - echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" > /etc/apt/sources.list.d/ansible.list + - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367 + - apt-get update && apt-get -y install ansible + use_sudo: false + - name: centos-6 + driver_config: + image: centos:6 + privileged: true + provision_command: + - yum -y install epel-release + - yum -y install ansible + use_sudo: false + - name: centos-7 + driver_config: + image: centos:7 + provision_command: + - yum -y install ansible + run_command: "/usr/sbin/init" + privileged: true + use_sudo: false +suites: + - name: standard + provisioner: + playbook: test/integration/standard.yml + additional_copy_path: + - "." + run_list: + attributes: + - name: standard-6x + provisioner: + playbook: test/integration/standard-6x.yml + additional_copy_path: + - "." + run_list: + attributes: + - name: multi + provisioner: + playbook: test/integration/multi.yml + additional_copy_path: + - "." + run_list: + attributes: + - name: config + provisioner: + playbook: test/integration/config.yml + additional_copy_path: + - "." + run_list: + attributes: \ No newline at end of file diff --git a/ansible/roles/elastic.beats/Gemfile b/ansible/roles/elastic.beats/Gemfile new file mode 100644 index 00000000..fbac7392 --- /dev/null +++ b/ansible/roles/elastic.beats/Gemfile @@ -0,0 +1,6 @@ +source 'https://rubygems.org' + +gem 'test-kitchen', '1.21.1' +gem "kitchen-docker", '2.6.0' +gem 'kitchen-ansible', '0.48.3' +gem 'net-ssh', '4.2.0' \ No newline at end of file diff --git a/ansible/roles/elastic.beats/Gemfile.lock b/ansible/roles/elastic.beats/Gemfile.lock new file mode 100644 index 00000000..62376572 --- /dev/null +++ b/ansible/roles/elastic.beats/Gemfile.lock @@ -0,0 +1,75 @@ +GEM + remote: https://rubygems.org/ + specs: + builder (3.2.3) + erubis (2.7.0) + ffi (1.9.24) + gssapi (1.2.0) + ffi (>= 1.0.1) + gyoku (1.3.1) + builder (>= 2.1.2) + httpclient (2.8.3) + kitchen-ansible (0.48.3) + net-ssh (>= 3) + test-kitchen (~> 1.4) + kitchen-docker (2.6.0) + test-kitchen (>= 1.0.0) + little-plugger (1.1.4) + logging (2.2.2) + little-plugger (~> 1.1) + multi_json (~> 1.10) + mixlib-install (3.9.3) + mixlib-shellout + mixlib-versioning + thor + mixlib-shellout (2.3.2) + mixlib-versioning (1.2.2) + multi_json (1.13.1) + net-scp (1.2.1) + net-ssh (>= 2.6.5) + net-ssh (4.2.0) + net-ssh-gateway (1.3.0) + net-ssh (>= 2.6.5) + nori (2.6.0) + rubyntlm (0.6.2) + rubyzip (1.2.2) + test-kitchen (1.21.1) + mixlib-install (~> 3.6) + mixlib-shellout (>= 1.2, < 3.0) + net-scp (~> 1.1) + net-ssh (>= 2.9, < 5.0) + net-ssh-gateway (~> 1.2) + thor (~> 0.19, < 0.19.2) + winrm (~> 2.0) + winrm-elevated (~> 1.0) + winrm-fs (~> 1.1) + thor (0.19.1) + winrm (2.2.3) + builder (>= 2.1.2) + erubis (~> 2.7) + gssapi (~> 1.2) + gyoku (~> 1.0) + httpclient (~> 2.2, >= 2.2.0.2) + logging (>= 1.6.1, < 3.0) + nori (~> 2.0) + rubyntlm (~> 0.6.0, >= 0.6.1) + winrm-elevated (1.1.0) + winrm (~> 2.0) + winrm-fs (~> 1.0) + winrm-fs (1.2.0) + erubis (~> 2.7) + logging (>= 1.6.1, < 3.0) + rubyzip (~> 1.1) + winrm (~> 2.0) + +PLATFORMS + ruby + +DEPENDENCIES + kitchen-ansible (= 0.48.3) + kitchen-docker (= 2.6.0) + net-ssh (= 4.2.0) + test-kitchen (= 1.21.1) + +BUNDLED WITH + 1.16.1 diff --git a/ansible/roles/elastic.beats/Makefile b/ansible/roles/elastic.beats/Makefile new file mode 100644 index 00000000..35febc4b --- /dev/null +++ b/ansible/roles/elastic.beats/Makefile @@ -0,0 +1,31 @@ +default: build + +SHELL:=/bin/bash -eux +PATTERN := standard-ubuntu-1804 + +.PHONY: converge verify test login destroy list + +setup: + bundle install + docker ps + +converge: + bundle exec kitchen converge $(PATTERN) + +verify: + bundle exec kitchen verify $(PATTERN) + +test: + bundle exec kitchen test $(PATTERN) --destroy=always + +login: + bundle exec kitchen login $(PATTERN) + +destroy: + bundle exec kitchen destroy $(PATTERN) + +destroy-all: + bundle exec kitchen destroy + +list: + bundle exec kitchen list diff --git a/ansible/roles/elastic.beats/README.md b/ansible/roles/elastic.beats/README.md new file mode 100644 index 00000000..b220e2b0 --- /dev/null +++ b/ansible/roles/elastic.beats/README.md @@ -0,0 +1,159 @@ +Beats Role +========= + +This role provides a generic means of installing Elastic supported beats using Ansible. Currently this includes: + +- Filebeat +- MetricBeat (TopBeat in 1.x) +- Packetbeat + +Requirements +------------ + +This role requires: + +- Ansible version > 2.0 +- Beats version >= 1.x +- Target machines require access to an external repo, or means + +The following OS are currently actively tested using kitchen: + +- Centos 6 & 7 +- Debian 8 & 9 +- Ubuntu 14.04, 16.04 & 18.04 + +The above list is subject to change. + +Role Variables +-------------- + +Supported variables are as follows: + +- **beat** (*MANDATORY*): Beat product. Supported values are: "filebeat", "metricbeat" & "packetbeat". +- **beat_conf** (*MANDATORY*): Beat Configuration. Should be defined as a map (see [Example Playbook](#example-playbook)) +- **beats_version** (*Defaults to `7.0.0`*): Beats version. +- **version_lock** (*Defaults to `false`*): Locks the installed version if set to true, thus preventing other processes from updating. This will not impact the roles ability to update the beat on subsequent runs (it unlocks and re-locks if required). +- **use_repository** (*Defaults to `true`*): Use elastic repo for yum or apt if true. If false, a custom custom_package_url must be provided. +- **start_service** (*Defaults to `true`*): service will be started if true, false otherwise. +- **restart_on_change** (*Defaults to `true`*): Changes to configuration or installed versions, will result in a restart if true. +- **daemon_args** (*Applicable to version 1.x of beats*): Allows run time params to be passed to beats. +- **logging_conf** (*Defaults to `{"files":{"rotateeverybytes":10485760}}`*): Logging configuration. Should be defined as a map (see [Example Playbook](#example-playbook)). Map is serialized into logging section of beat config. +- **shipper_conf** (*Applicable to version 1.x of beats*): Shipper configuration. Should be defined as a map (see [Example Playbook](#example-playbook)). Map is serialized into shipper section of beat config. +- **output_conf** (*Defaults to `{"elasticsearch":{"hosts":["localhost:9200"]}}`): Output configuration. Should be defined as a map (see [Example Playbook](#example-playbook)). Map is serialized into output section of beat config. +- **beats_pid_dir** (*Defaults to `/var/run`*): Location of beats pid file. +- **beats_conf_dir** (*Defaults to `/etc/{beat}`*): Location of conf directory for beats configuration file. + +Dependencies +------------ + +- Ansible version > 2.0 +- Beats version >= 6.x + +Example Playbook +---------------- + +Example playbook is provided below. This installs Packetbeat and illustrates the need for configuration sections to be specified as maps. + + - name: Example playbook for installing packetbeat + hosts: localhost + roles: + - { role: beats, beat: "packetbeat", + beat_conf: { + "interfaces": {"device":"any"}, + "protocols": { + "dns": { + "ports": [53], + "include_authorities":true + }, + "http": { + "ports": [80, 8080, 8000, 5000, 8002] + }, + "memcache": { + "ports": [11211] + }, + "mysql": { + "ports": [3306] + }, + "pgsql": { + "ports": [5432] + }, + "redis": { + "ports": [6379] + }, + "thrift": { + "ports": [9090] + }, + "mongodb": { + "ports": [27017] + } + } + }, + output_conf : { + "elasticsearch": { + "hosts": ["localhost:9200"] + } + } + } + vars: + use_repository: "true" + +## Testing + +This playbook uses [Kitchen](https://kitchen.ci/) for CI and local testing. + +### Requirements + +* Ruby +* Bundler +* Docker +* Make + +### Running the tests + +To converge an Ubuntu 18.04 host +```sh +$ make converge +``` + +To run the tests +```sh +$ make verify +``` + +To list all of the different test suits +```sh +$ make list +``` + +The default test suite is Ubuntu 18.04. If you want to test another suite you can override this with the `PATTERN` variable +```sh +$ make converge PATTERN=standard-centos-7 +``` + +The `PATTERN` is a kitchen pattern which can match multiple suites. To run all tests for CentOS +```sh +$ make converge PATTERN=centos-7 +``` + +When you are finished testing you can clean up everything with +```sh +$ make destroy-all +``` + +License +------- + +Apache 2.0 + +Limitations +------------ + +Multiple instances of the same beat cannot be installed on the same target server. + +Author Information +------------------ + +Dale McDiarmid +For further assistance, outside of issues, please use discuss.elastic.co. + +Community Contributions always appreciated and welcome! Please ensure all contributions include tests as appropriate. diff --git a/ansible/roles/elastic.beats/ansible.cfg b/ansible/roles/elastic.beats/ansible.cfg new file mode 100644 index 00000000..d9a8c501 --- /dev/null +++ b/ansible/roles/elastic.beats/ansible.cfg @@ -0,0 +1 @@ +[defaults] \ No newline at end of file diff --git a/ansible/roles/elastic.beats/defaults/main.yml b/ansible/roles/elastic.beats/defaults/main.yml new file mode 100644 index 00000000..1e871b60 --- /dev/null +++ b/ansible/roles/elastic.beats/defaults/main.yml @@ -0,0 +1,13 @@ +--- +# defaults file for beats + +beats_version: 7.0.0 +version_lock: false +use_repository: true +start_service: true +restart_on_change: true +daemon_args: "" +logging_conf: {"files":{"rotateeverybytes":10485760}} +output_conf: {"elasticsearch":{"hosts":["localhost:9200"]}} +beats_pid_dir: "/var/run" +beats_conf_dir: "/etc/{{beat}}" \ No newline at end of file diff --git a/ansible/roles/elastic.beats/filter_plugins/custom.py b/ansible/roles/elastic.beats/filter_plugins/custom.py new file mode 100644 index 00000000..04b49155 --- /dev/null +++ b/ansible/roles/elastic.beats/filter_plugins/custom.py @@ -0,0 +1,8 @@ +__author__ = 'dale mcdiarmid' + +def merge_config(config={}, key=''): + return {key:config} + +class FilterModule(object): + def filters(self): + return {'merge_config': merge_config} \ No newline at end of file diff --git a/ansible/roles/elastic.beats/handlers/main.yml b/ansible/roles/elastic.beats/handlers/main.yml new file mode 100644 index 00000000..f69bcc39 --- /dev/null +++ b/ansible/roles/elastic.beats/handlers/main.yml @@ -0,0 +1,6 @@ +--- +# handlers file for beats + +- name: restart beat + service: name={{ beat }} state=restarted enabled=yes + when: start_service and restart_on_change and not beats_started.changed and beat_install.changed diff --git a/ansible/roles/elastic.beats/meta/.galaxy_install_info b/ansible/roles/elastic.beats/meta/.galaxy_install_info new file mode 100644 index 00000000..ed876590 --- /dev/null +++ b/ansible/roles/elastic.beats/meta/.galaxy_install_info @@ -0,0 +1,2 @@ +install_date: Sat Jul 6 01:04:40 2019 +version: 7.0.0 diff --git a/ansible/roles/elastic.beats/meta/main.yml b/ansible/roles/elastic.beats/meta/main.yml new file mode 100644 index 00000000..726a84cf --- /dev/null +++ b/ansible/roles/elastic.beats/meta/main.yml @@ -0,0 +1,28 @@ +--- + +allow_duplicates: yes + +galaxy_info: + role_name: beats + author: Dale McDiarmid + description: Beats for Linux + company: "Elastic.co" + license: "license (Apache)" + min_ansible_version: 2.0 + platforms: + - name: EL + versions: + - 6 + - 7 + - name: Debian + versions: + - all + - name: Ubuntu + versions: + - all + categories: + - beats + - elastic + - elk + - logging + - monitoring diff --git a/ansible/roles/elastic.beats/tasks/beats-config.yml b/ansible/roles/elastic.beats/tasks/beats-config.yml new file mode 100644 index 00000000..13e84f2a --- /dev/null +++ b/ansible/roles/elastic.beats/tasks/beats-config.yml @@ -0,0 +1,33 @@ +--- +# Configure Beats Node + +- set_fact: pid_file={{ beats_pid_dir }}/{{beat}}.pid +- set_fact: instance_default_file={{ default_file }}/{{beat}} +- set_fact: conf_file={{ beats_conf_dir }}/{{beat}}.yml + +- set_fact: beat_output_conf={{ output_conf | merge_config('output') }} +- set_fact: beat_shipper_conf={{ shipper_conf | merge_config('shipper') }} + when: shipper_conf is defined +- set_fact: beat_logging_conf={{ logging_conf | merge_config('logging') }} + +- stat: path={{beats_pid_dir}} + register: pid_stat + +- name: Create PID Directory + file: path={{ beats_pid_dir }} state=directory + when: pid_stat.stat.isdir is not defined or pid_stat.stat.islnk is not defined + +#fail if pid and config directories are not links or not directories i.e files + +- name: Create Config Directory + file: path={{ beats_conf_dir }} state=directory + +#Copy the default file +- name: Copy Default File for Instance + template: src=beat.j2 dest={{instance_default_file}} mode=0644 force=yes owner=root group=root + notify: restart beat + +#Copy templated config file +- name: Copy Configuration File + template: src=beat.yml.j2 dest={{conf_file}} mode=0644 force=yes owner=root group=root + notify: restart beat diff --git a/ansible/roles/elastic.beats/tasks/beats-debian.yml b/ansible/roles/elastic.beats/tasks/beats-debian.yml new file mode 100644 index 00000000..82a1befb --- /dev/null +++ b/ansible/roles/elastic.beats/tasks/beats-debian.yml @@ -0,0 +1,45 @@ +--- + +- name: Debian - Ensure apt-transport-https is installed + apt: name=apt-transport-https state=present cache_valid_time=86400 + when: use_repository + register: beat_install + notify: restart beat + +- name: Debian - Add Beats repository key + apt_key: url="http://packages.elasticsearch.org/GPG-KEY-elasticsearch" state=present + when: use_repository + +- name: Debian - add beats repository + apt_repository: repo="deb {{ repo_url }} stable main" state=present + when: use_repository + +- name: Debian - unhold {{beat}} version for install + command: apt-mark unhold {{beat}} + changed_when: False + +- name: Debian - Ensure {{beat}} is installed + apt: name={{beat}}{% if beats_version is defined and beats_version != "" %}={{ beats_version }}{% endif %} state=present cache_valid_time=86400 + when: use_repository + register: beat_install + notify: restart beat + +- name: Debian - hold {{beat}} version + command: apt-mark hold {{beat}} + when: version_lock + changed_when: False + +- set_fact: os_arch="i386" + +- set_fact: os_arch="amd64" + when: ansible_architecture == "x86_64" + +- name: Debian - Download {{beat}} from url + get_url: url={% if custom_package_url is defined %}{{ custom_package_url }}{% else %}{{ beats_package_url }}/{{beat}}/{{beat}}_{{ beats_version }}_{{os_arch}}.deb{% endif %} dest=/tmp/{{beat}}_{{ beats_version }}_{{os_arch}}.deb validate_certs=no + when: not use_repository + +- name: Debian - Ensure {{beat}} is installed from downloaded package + apt: deb=/tmp/{{beat}}_{{ beats_version }}_{{os_arch}}.deb + when: not use_repository + register: beat_install + notify: restart beat diff --git a/ansible/roles/elastic.beats/tasks/beats-param-check.yml b/ansible/roles/elastic.beats/tasks/beats-param-check.yml new file mode 100644 index 00000000..8b9aafb9 --- /dev/null +++ b/ansible/roles/elastic.beats/tasks/beats-param-check.yml @@ -0,0 +1,9 @@ +--- + +- fail: msg="beat must be specified and cannot be blank e.g. filebeat" + when: beat is not defined or beat == '' + +- fail: msg="beat_conf must be specified" + when: beat_conf is not defined + +- set_fact: "beats_major_version={{ beats_version[0] }}.x" \ No newline at end of file diff --git a/ansible/roles/elastic.beats/tasks/beats-redhat.yml b/ansible/roles/elastic.beats/tasks/beats-redhat.yml new file mode 100644 index 00000000..f79a2549 --- /dev/null +++ b/ansible/roles/elastic.beats/tasks/beats-redhat.yml @@ -0,0 +1,33 @@ +--- + +- name: Ensure libselinux-python on CentOS 6.x + yum: name=libselinux-python state=present update_cache=yes + when: ( ansible_distribution == "CentOS" ) and ( ansible_distribution_major_version == "6" ) + +- name: Redhat - add beats repository + template: src=beats.repo.j2 dest=/etc/yum.repos.d/beats.repo + when: use_repository + +- name: RedHat - install yum-version-lock + yum: name=yum-plugin-versionlock state=present update_cache=yes + +- name: RedHat - unlock {{beat}} for install + shell: yum versionlock delete {{beat}} || true + changed_when: False + +- name: RedHat - Ensure {{beat}} is installed + yum: name={{beat}}{% if beats_version is defined and beats_version != "" %}-{{ beats_version }}{% endif %} state=present update_cache=yes + when: use_repository + register: beat_install + notify: restart beat + +- name: RedHat - lock {{beat}} version + shell: yum versionlock add {{beat}}{% if beats_version is defined and beats_version != "" %}-{{ beats_version }}{% endif %} + when: version_lock + changed_when: False + +- name: RedHat - Install {{beat}} from url + yum: name={% if custom_package_url is defined %}{{ custom_package_url }}{% else %}{{ beats_package_url }}/{{beat}}-{{ beats_version }}-{{ansible_architecture}}.rpm{% endif %} state=present + when: not use_repository + register: beat_install + notify: restart beat diff --git a/ansible/roles/elastic.beats/tasks/beats.yml b/ansible/roles/elastic.beats/tasks/beats.yml new file mode 100644 index 00000000..ef936a28 --- /dev/null +++ b/ansible/roles/elastic.beats/tasks/beats.yml @@ -0,0 +1,20 @@ +--- +# Install OS specific beats + +- name: Include specific Beats + include: beats-debian.yml + when: ansible_os_family == 'Debian' + +- name: Include specific Beats + include: beats-redhat.yml + when: ansible_os_family == 'RedHat' + +#Configuration file for beats +- name: Beats configuration + include: beats-config.yml + +# Make sure the service is started, and restart if necessary +- name: Start beats service + service: name={{ beat }} state=started enabled=yes + when: start_service + register: beats_started diff --git a/ansible/roles/elastic.beats/tasks/main.yml b/ansible/roles/elastic.beats/tasks/main.yml new file mode 100644 index 00000000..69d69f46 --- /dev/null +++ b/ansible/roles/elastic.beats/tasks/main.yml @@ -0,0 +1,10 @@ +--- +# tasks file for beats + +- name: check-parameters + include: beats-param-check.yml + +- name: os-specific vars + include_vars: "{{ansible_os_family}}.yml" + +- include: beats.yml diff --git a/ansible/roles/elastic.beats/templates/beat.j2 b/ansible/roles/elastic.beats/templates/beat.j2 new file mode 100644 index 00000000..0321e8e3 --- /dev/null +++ b/ansible/roles/elastic.beats/templates/beat.j2 @@ -0,0 +1,7 @@ +################################ +# {{beat}} +################################ + +# Beats PID File +PIDFILE={{pid_file}} +DAEMON_ARGS="-c {{conf_file}} {{daemon_args}}" diff --git a/ansible/roles/elastic.beats/templates/beat.yml.j2 b/ansible/roles/elastic.beats/templates/beat.yml.j2 new file mode 100644 index 00000000..ea441d45 --- /dev/null +++ b/ansible/roles/elastic.beats/templates/beat.yml.j2 @@ -0,0 +1,20 @@ +################### {{beat}} Configuration ######################### + +############################# {{beat}} ###################################### +{{beat_conf | to_nice_yaml}} + +############################################################################### +############################# Libbeat Config ################################## +# Base config file used by all other beats for using libbeat features + +############################# Output ########################################## + +{{ beat_output_conf | to_nice_yaml }} + +{% if shipper_conf is defined %}############################# Shipper ######################################### + +{{ beat_shipper_conf | to_nice_yaml }} +{% endif %} +############################# Logging ######################################### + +{{ beat_logging_conf | to_nice_yaml }} diff --git a/ansible/roles/elastic.beats/templates/beats.repo.j2 b/ansible/roles/elastic.beats/templates/beats.repo.j2 new file mode 100644 index 00000000..c89503b6 --- /dev/null +++ b/ansible/roles/elastic.beats/templates/beats.repo.j2 @@ -0,0 +1,6 @@ +[beats] +name=Elastic Beats Repository +baseurl={{ repo_url }} +enabled=1 +gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch +gpgcheck=1 diff --git a/ansible/roles/elastic.beats/test/integration/config.yml b/ansible/roles/elastic.beats/test/integration/config.yml new file mode 100644 index 00000000..735a3680 --- /dev/null +++ b/ansible/roles/elastic.beats/test/integration/config.yml @@ -0,0 +1,44 @@ +--- +#Install specific version here +- name: wrapper playbook for kitchen testing "beats" + hosts: localhost + roles: + - { role: "ansible-beats", beat: "packetbeat", version_lock: true, + beat_conf: { + "interfaces": {"device":"any"}, + "protocols": { + "dns": { + "ports": [53], + "include_authorities":true + }, + "http": { + "ports": [80, 8080, 8000, 5000, 8002] + }, + "memcache": { + "ports": [11211] + }, + "mysql": { + "ports": [3306] + }, + "pgsql": { + "ports": [5432] + }, + "redis": { + "ports": [6379] + }, + "thrift": { + "ports": [9090] + }, + "mongodb": { + "ports": [27017] + } + } + }, + output_conf : { + "elasticsearch": { + "hosts": ["localhost:9200"] + } + } + } + vars: + use_repository: "true" \ No newline at end of file diff --git a/ansible/roles/elastic.beats/test/integration/config/serverspec/default_spec.rb b/ansible/roles/elastic.beats/test/integration/config/serverspec/default_spec.rb new file mode 100644 index 00000000..28477681 --- /dev/null +++ b/ansible/roles/elastic.beats/test/integration/config/serverspec/default_spec.rb @@ -0,0 +1,49 @@ +require 'spec_helper' + +describe 'Config Tests' do + + describe service('packetbeat') do + it { should be_running } + end + + describe package('packetbeat') do + it { should be_installed } + end + + describe file('/etc/packetbeat/packetbeat.yml') do + it { should be_file } + it { should be_owned_by 'root' } + end + + describe file('/etc/packetbeat/packetbeat.yml') do + it { should contain 'logging:' } + it { should contain 'output:' } + it { should contain 'protocols:' } + it { should contain 'dns:' } + it { should contain 'memcache:' } + it { should contain 'http:' } + it { should contain 'mongodb:' } + it { should contain 'mysql:' } + it { should contain 'pgsql:' } + it { should contain 'redis:' } + it { should contain 'thrift:' } + it { should contain 'interfaces:' } + it { should contain 'device: any' } + end + + describe file('/etc/init.d/packetbeat') do + it { should exist } + end + + if os[:family] == 'redhat' + describe command('yum versionlock list | grep packetbeat') do + its(:stdout) { should match /packetbeat/ } + end + elsif ['debian', 'ubuntu'].include?(os[:family]) + describe command('sudo apt-mark showhold | grep packetbeat') do + its(:stdout) { should match /packetbeat/ } + end + end + +end + diff --git a/ansible/roles/elastic.beats/test/integration/helpers/serverspec/Gemfile b/ansible/roles/elastic.beats/test/integration/helpers/serverspec/Gemfile new file mode 100644 index 00000000..e6aafc0b --- /dev/null +++ b/ansible/roles/elastic.beats/test/integration/helpers/serverspec/Gemfile @@ -0,0 +1,3 @@ +source 'https://rubygems.org' + +gem 'rspec-retry' diff --git a/ansible/roles/elastic.beats/test/integration/helpers/serverspec/spec_helper.rb b/ansible/roles/elastic.beats/test/integration/helpers/serverspec/spec_helper.rb new file mode 100644 index 00000000..1a1bfb52 --- /dev/null +++ b/ansible/roles/elastic.beats/test/integration/helpers/serverspec/spec_helper.rb @@ -0,0 +1,11 @@ +require 'serverspec' +set :backend, :exec + +require 'rspec/retry' + +RSpec.configure do |config| + # show retry status in spec process + config.verbose_retry = true + # show exception that triggers a retry if verbose_retry is set to true + config.display_try_failure_messages = true +end \ No newline at end of file diff --git a/ansible/roles/elastic.beats/test/integration/multi.yml b/ansible/roles/elastic.beats/test/integration/multi.yml new file mode 100644 index 00000000..11a3146b --- /dev/null +++ b/ansible/roles/elastic.beats/test/integration/multi.yml @@ -0,0 +1,8 @@ +--- +- name: wrapper playbook for kitchen testing "beats" + hosts: localhost + roles: + - { role: "ansible-beats", beat: "filebeat", beat_conf: {"filebeat": {"inputs":[{"paths":["/var/log/*.log"],"type":"log"}]} } } + - { role: "ansible-beats", beat: "metricbeat", beat_conf: { "metricbeat.modules": [{"module": "system", "metricsets": ["cpu", "filesystem", "network", "process"], "enabled": true, "period": "10s", "processes":[".*"], "cpu_ticks": false } ] } } + vars: + use_repository: "true" \ No newline at end of file diff --git a/ansible/roles/elastic.beats/test/integration/multi/serverspec/default_spec.rb b/ansible/roles/elastic.beats/test/integration/multi/serverspec/default_spec.rb new file mode 100644 index 00000000..086b6f28 --- /dev/null +++ b/ansible/roles/elastic.beats/test/integration/multi/serverspec/default_spec.rb @@ -0,0 +1,57 @@ +require 'spec_helper' + +describe 'Multi Tests' do + + describe service('filebeat') do + it { should be_running } + end + + describe package('filebeat') do + it { should be_installed } + end + + describe file('/etc/filebeat/filebeat.yml') do + it { should be_file } + it { should be_owned_by 'root' } + end + + describe file('/etc/filebeat/filebeat.yml') do + it { should contain 'filebeat:' } + it { should contain 'logging:' } + it { should contain 'output:' } + end + + describe file('/etc/init.d/filebeat') do + it { should exist } + end + + describe service('metricbeat') do + it { should be_running } + end + + describe package('metricbeat') do + it { should be_installed } + end + + describe file('/etc/metricbeat/metricbeat.yml') do + it { should be_file } + it { should be_owned_by 'root' } + end + + describe file('/etc/metricbeat/metricbeat.yml') do + it { should contain 'module: system' } + it { should contain 'metricsets:' } + it { should contain 'period: 10s' } + it { should contain 'processes:' } + it { should contain 'cpu_ticks:' } + it { should contain 'logging:' } + it { should contain 'output:' } + end + + describe file('/etc/init.d/metricbeat') do + it { should exist } + end + + +end + diff --git a/ansible/roles/elastic.beats/test/integration/standard-6x.yml b/ansible/roles/elastic.beats/test/integration/standard-6x.yml new file mode 100644 index 00000000..30d9c330 --- /dev/null +++ b/ansible/roles/elastic.beats/test/integration/standard-6x.yml @@ -0,0 +1,8 @@ +--- +- name: wrapper playbook for kitchen testing "beats" + hosts: localhost + roles: + - { role: "ansible-beats", beat: "filebeat", beat_conf: {"filebeat": {"prospectors":[{"paths":["/var/log/*.log"],"input_type":"log"}], "registry_file": "/var/lib/filebeat/registry"} } } + vars: + beats_version: 6.7.1 + use_repository: "true" \ No newline at end of file diff --git a/ansible/roles/elastic.beats/test/integration/standard-6x/serverspec/default_spec.rb b/ansible/roles/elastic.beats/test/integration/standard-6x/serverspec/default_spec.rb new file mode 100644 index 00000000..2aa16923 --- /dev/null +++ b/ansible/roles/elastic.beats/test/integration/standard-6x/serverspec/default_spec.rb @@ -0,0 +1,39 @@ +require 'spec_helper' + +describe 'Standard Tests' do + + describe service('filebeat') do + it { should be_running } + end + + describe package('filebeat') do + it { should be_installed } + end + + describe file('/etc/filebeat/filebeat.yml') do + it { should be_file } + it { should be_owned_by 'root' } + end + + describe file('/etc/filebeat/filebeat.yml') do + it { should contain 'filebeat:' } + it { should contain 'logging:' } + it { should contain 'output:' } + end + + describe file('/etc/init.d/filebeat') do + it { should exist } + end + + if os[:family] == 'redhat' + describe command('yum versionlock list | grep filebeat') do + its(:stdout) { should_not match /filebeat/ } + end + elsif ['debian', 'ubuntu'].include?(os[:family]) + describe command('sudo apt-mark showhold | grep filebeat') do + its(:stdout) { should_not match /filebeat/ } + end + end + +end + diff --git a/ansible/roles/elastic.beats/test/integration/standard.yml b/ansible/roles/elastic.beats/test/integration/standard.yml new file mode 100644 index 00000000..9ac2cdbb --- /dev/null +++ b/ansible/roles/elastic.beats/test/integration/standard.yml @@ -0,0 +1,7 @@ +--- +- name: wrapper playbook for kitchen testing "beats" + hosts: localhost + roles: + - { role: "ansible-beats", beat: "filebeat", beat_conf: {"filebeat": {"inputs":[{"paths":["/var/log/*.log"],"type":"log"}]} } } + vars: + use_repository: "true" \ No newline at end of file diff --git a/ansible/roles/elastic.beats/test/integration/standard/serverspec/default_spec.rb b/ansible/roles/elastic.beats/test/integration/standard/serverspec/default_spec.rb new file mode 100644 index 00000000..2aa16923 --- /dev/null +++ b/ansible/roles/elastic.beats/test/integration/standard/serverspec/default_spec.rb @@ -0,0 +1,39 @@ +require 'spec_helper' + +describe 'Standard Tests' do + + describe service('filebeat') do + it { should be_running } + end + + describe package('filebeat') do + it { should be_installed } + end + + describe file('/etc/filebeat/filebeat.yml') do + it { should be_file } + it { should be_owned_by 'root' } + end + + describe file('/etc/filebeat/filebeat.yml') do + it { should contain 'filebeat:' } + it { should contain 'logging:' } + it { should contain 'output:' } + end + + describe file('/etc/init.d/filebeat') do + it { should exist } + end + + if os[:family] == 'redhat' + describe command('yum versionlock list | grep filebeat') do + its(:stdout) { should_not match /filebeat/ } + end + elsif ['debian', 'ubuntu'].include?(os[:family]) + describe command('sudo apt-mark showhold | grep filebeat') do + its(:stdout) { should_not match /filebeat/ } + end + end + +end + diff --git a/ansible/roles/elastic.beats/test/matrix.yml b/ansible/roles/elastic.beats/test/matrix.yml new file mode 100644 index 00000000..2763b4aa --- /dev/null +++ b/ansible/roles/elastic.beats/test/matrix.yml @@ -0,0 +1,13 @@ +OS: + - ubuntu-1404 + - ubuntu-1604 + - ubuntu-1804 + - debian-8 + - debian-9 + - centos-6 + - centos-7 +TEST_TYPE: + - standard + - standard-6x + - multi + - config diff --git a/ansible/roles/elastic.beats/vars/Debian.yml b/ansible/roles/elastic.beats/vars/Debian.yml new file mode 100644 index 00000000..819955f9 --- /dev/null +++ b/ansible/roles/elastic.beats/vars/Debian.yml @@ -0,0 +1,5 @@ +--- + +default_file: "/etc/default" +init_script: "/etc/init.d" +repo_url: "https://artifacts.elastic.co/packages/{{ beats_major_version }}/apt" diff --git a/ansible/roles/elastic.beats/vars/RedHat.yml b/ansible/roles/elastic.beats/vars/RedHat.yml new file mode 100644 index 00000000..34a6787f --- /dev/null +++ b/ansible/roles/elastic.beats/vars/RedHat.yml @@ -0,0 +1,5 @@ +--- + +default_file: "/etc/sysconfig" +init_script: "/etc/init.d" +repo_url: "https://artifacts.elastic.co/packages/{{ beats_major_version }}/yum" \ No newline at end of file diff --git a/ansible/roles/elastic.beats/vars/main.yml b/ansible/roles/elastic.beats/vars/main.yml new file mode 100644 index 00000000..f696bfec --- /dev/null +++ b/ansible/roles/elastic.beats/vars/main.yml @@ -0,0 +1,5 @@ +--- +# vars file for beats + +beats_package_url: "https://download.elastic.co/beats" +es_conf_dir: "/etc"