diff --git a/clojurians_log/README.md b/clojurians_log/README.md index 553f30c..53da68e 100644 --- a/clojurians_log/README.md +++ b/clojurians_log/README.md @@ -14,7 +14,7 @@ This is a work in progress and is subject to significant changes over time. Contact either of the [maintainers](https://github.com/clojureverse/nebula#maintainers) for this. - Create your own SSH keypair on the Console: [guide](https://community.exoscale.com/documentation/compute/ssh-keypairs/) -- Install Terraform v0.11+. +- Install Terraform v0.12.6+. - Download the latest Exoscale terraform provider for your OS from [here](https://github.com/terraform-providers/terraform-provider-exoscale). - Decompress the archive and follow [the plugin installation](https://www.terraform.io/docs/configuration/providers.html#third-party-plugins). - Install [GPG](https://gnupg.org/download/) @@ -24,9 +24,19 @@ This is a work in progress and is subject to significant changes over time. - Run `export TF_VAR_exoscale_api_key="The Exoscale API key here"` - Run `export TF_VAR_exoscale_secret_key="The Exoscale secret key here"` - Run `export TF_VAR_exoscale_ssh_keypair_name="The Exoscale key pair name you created"` + - Run `export TF_VAR_cloudflare_email="Your cloudflare account"` + - Run `export TF_VAR_cloudlfare_api_key="Cloudflare 'global API key' (not to be confused with cloudflare tokens)"` + - Run `bin/terraform_apply` + - To destroy: `terraform destroy` + - To ssh into the instance + - `ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $(terraform output username)@$(terraform output ip_address)` + - You wil want to reboot the instance after provisioning the first time. (why?) + - You will need to update the IP address on the host name in the DNS as well. + +The `terraform_apply script does the following` - In the `playbooks/vars` dir: - Run `gpg --decrypt clojurians_log_secrets.yml.gpg > clojurians_log_secrets.yml` - - (Optional, for maintainers) Run: + - If you need to update the secret file later on you'll have to do this: ```bash gpg --encrypt --recipient your_email \ --recipient others_emails \ @@ -41,13 +51,7 @@ This is a work in progress and is subject to significant changes over time. -backend-config="secret_key=${TF_VAR_exoscale_secret_key}" ``` - Run `terraform plan -out plan` - - Make sure all looks good. - Run `terraform apply plan` - - To destroy: `terraform destroy` - - To ssh into the instance - - `ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $(terraform output username)@$(terraform output ip_address)` - - You wil want to reboot the instance after provisioning the first time. - - You will need to update the IP address on the host name in the DNS as well. ## How to use Vagrant diff --git a/clojurians_log/Vagrantfile b/clojurians_log/Vagrantfile index edd0cdc..f128fa0 100644 --- a/clojurians_log/Vagrantfile +++ b/clojurians_log/Vagrantfile @@ -76,7 +76,8 @@ Vagrant.configure("2") do |config| ansible.raw_arguments = Shellwords.shellsplit(ENV['ANSIBLE_ARGS']) if ENV['ANSIBLE_ARGS'] ansible.extra_vars = { ansible_python_interpreter: "/usr/bin/python3", clojurians_app_fqdn: "clojurians-log-staging.clojureverse.org", - use_demo_logs: true, # for only loading the demo logs on a vagrant host + use_demo_logs: true, # Load the demo logs on a vagrant host + acme_sh_default_staging: true # Use ACME default servers on a vagrant host acme_sh_default_force_issue: true, # Force ACME cert issue on a vagrant host } end diff --git a/clojurians_log/bin/terraform_apply b/clojurians_log/bin/terraform_apply new file mode 100755 index 0000000..4f03dd7 --- /dev/null +++ b/clojurians_log/bin/terraform_apply @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e + +if [[ -z "${TF_VAR_exoscale_ssh_keypair_name}" ]]; then + echo "TF_VAR_exoscale_ssh_keypair_name not set, aborting." + exit 1 +fi + +if [[ -z "${TF_VAR_exoscale_api_key}" ]]; then + echo "TF_VAR_exoscale_api_key not set, aborting." + exit 1 +fi + +if [[ -z "${TF_VAR_exoscale_secret_key}" ]]; then + echo "TF_VAR_exoscale_secret_key not set, aborting." + exit 1 +fi + +# File does not exist, or encrypted version is newer: decrypt +if [[ "playbooks/vars/clojurians_log_secrets.yml.gpg" -nt "playbooks/vars/clojurians_log_secrets.yml" ]]; then + gpg --decrypt playbooks/vars/clojurians_log_secrets.yml.gpg > playbooks/vars/clojurians_log_secrets.yml +fi + +if [[ ! -d ".terraform" ]]; then + terraform init \ + -backend-config="access_key=${TF_VAR_exoscale_api_key}" \ + -backend-config="secret_key=${TF_VAR_exoscale_secret_key}" +fi + +terraform plan -out plan +terraform apply plan diff --git a/clojurians_log/main.tf b/clojurians_log/main.tf index c4ca5b8..5d0d714 100644 --- a/clojurians_log/main.tf +++ b/clojurians_log/main.tf @@ -1,28 +1,42 @@ +locals { + instance_name = "${terraform.workspace == "default" ? "clojurians-log" : terraform.workspace}" +} + provider "exoscale" { version = "~> 0.10" key = "${var.exoscale_api_key}" secret = "${var.exoscale_secret_key}" } +provider "cloudflare" { + version = "~> 1.17" + + email = "${var.cloudflare_email}" + token = "${var.cloudflare_api_key}" +} + +provider "template" { + version = "~> 2.1" +} + data "template_file" "userdata" { template = "${file("userdata.sh.tmpl")}" vars = { - # For development / testing, use (uncomment) the line below - ansible_playbook_params = "--extra-vars \"ansible_python_interpreter=/usr/bin/python3 clojurians_app_fqdn=clojurians-log-staging.clojureverse.org use_demo_logs=true acme_sh_default_force_issue=true\"" - git_clone_params = "--single-branch --branch exoscale-staging" - # For normal production use, the values set in ansible are appropriate. - # So, can leave the following empty. - # ansible_playbook_params = "" - # git_clone_params = "" + # For development / testing, use (uncomment) the lines below + # ansible_playbook_params = "--extra-vars \"ansible_python_interpreter=/usr/bin/python3 clojurians_app_fqdn=clojurians-log-staging.clojureverse.org use_demo_logs=true acme_sh_default_staging=true acme_sh_default_force_issue=true\"" + # git_clone_params = "--single-branch --branch exoscale-deploy" + # The following can be used for production + ansible_playbook_params = "--extra-vars \"ansible_python_interpreter=/usr/bin/python3\" --extra-vars \"clojurians_app_fqdn=${local.instance_name}.clojureverse.org\"" + git_clone_params = "" } } resource "exoscale_compute" "clojurians_log" { - display_name = "clojurians-log" + display_name = "${local.instance_name}" template = "Linux Ubuntu 18.04 LTS 64-bit" - zone = "de-fra-1" - size = "Medium" - disk_size = 10 + zone = "ch-gva-2" + size = "Large" + disk_size = 50 key_pair = "${var.exoscale_ssh_keypair_name}" user_data = "${data.template_file.userdata.rendered}" @@ -34,6 +48,22 @@ resource "exoscale_compute" "clojurians_log" { output "ip_address" { value = exoscale_compute.clojurians_log.ip_address } + output "username" { value = exoscale_compute.clojurians_log.username } + +resource "cloudflare_record" "clojurians_log_internal" { + domain = "clojureverse.org" + name = "${local.instance_name}-internal" + value = "${exoscale_compute.clojurians_log.ip_address}" + type = "A" +} + +resource "cloudflare_record" "clojurians_log" { + domain = "clojureverse.org" + name = "${local.instance_name}" + value = "${exoscale_compute.clojurians_log.ip_address}" + type = "A" + proxied = true +} diff --git a/clojurians_log/playbooks/clojurians-log.yml b/clojurians_log/playbooks/clojurians-log.yml index 065bbf9..1bee2a9 100644 --- a/clojurians_log/playbooks/clojurians-log.yml +++ b/clojurians_log/playbooks/clojurians-log.yml @@ -22,10 +22,19 @@ clojurians_app_http_port: 4242 datomic_pro_version: 0.9.5561.56 datomic_object_cache_max: 2g + datomic_transactor_max_heap: 3584m + clojure_socket_repl_port: 50505 clojure_app_env_vars: | - JVM_OPTS="-Dclojure.server.myrepl={:port,50505,:accept,clojure.core.server/repl} -Xmx8g -Xms2g -Ddatomic.ObjectCacheMax=7g -Ddatomic.memcachedServers=127.0.0.1:11211" + JVM_OPTS="-Dclojure.server.myrepl={:port,{{ clojure_socket_repl_port }},:accept,clojure.core.server/repl} -Xmx3584m -Xms2g -Ddatomic.ObjectCacheMax=2g -Ddatomic.memcachedServers=127.0.0.1:11211" + # -Dcom.sun.management.jmxremote.port=17264 -Dcom.sun.management.jmxremote.rmi.port=17264 tasks: + - name: Set up librato first so we have eyes + import_role: + name: librato.librato + vars: + librato_enabled_plugins: ['nginx', 'memcached'] #jvm, postgresql + librato_logging_use_syslog: true # Keys first, then repos, then the dist-upgrade. - name: Add repository for ansible @@ -177,7 +186,7 @@ clojure_app_health_check_url: "http://localhost:{{clojurians_app_http_port}}/healthcheck" clojure_app_service_start_after: datomic.service - - name: Install the Clojurians Log app + - name: Install and configure the clojurians-log-app tags: clojurians-log import_role: name: plexus.clojurians-log @@ -237,6 +246,9 @@ state: "present" nginx_access_log: off extra_parameters: | + location = /basic_status { + stub_status; + } location / { root {{ clojure_app_app_dir }}/resources/public; try_files $uri @proxy_app; diff --git a/clojurians_log/playbooks/roles/librato.librato/.gitignore b/clojurians_log/playbooks/roles/librato.librato/.gitignore new file mode 100644 index 0000000..58d5086 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/.gitignore @@ -0,0 +1,5 @@ +tmp/ +.kitchen/ +.librarian/ +Ansiblefile.lock +hosts diff --git a/clojurians_log/playbooks/roles/librato.librato/.kitchen.yml b/clojurians_log/playbooks/roles/librato.librato/.kitchen.yml new file mode 100644 index 0000000..e96da29 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/.kitchen.yml @@ -0,0 +1,102 @@ +driver: + name: vagrant + +provisioner: + name: ansible_playbook + require_ansible_repo: true + ansible_verbose: true + require_chef_for_busser: false + require_ruby_for_busser: true + hosts: hosts + roles_path: ../ + +verifier: + ruby_bindir: '/usr/bin' + +platforms: + - name: centos-6 + driver: + box: centos/6 + - name: centos-7 + driver: + box: centos/7 + - name: fedora-23 + driver: + box: fedora/23-cloud-base +# Due to a bug in kitchen-ansible, statically set the ansible_platform to Amazon +# See: https://github.com/neillturner/kitchen-ansible/issues/216 + - name: amazonlinux-2016.03 + driver: + name: ec2 + image_id: ami-7172b611 + aws_ssh_key_id: test-kitchen + region: us-west-2 + availability_zone: a + instance_type: t2.large + associate_public_ip: true + interface: dns + transport: + username: ec2-user + ssh_key: ~/.ssh/id_rsa + provisioner: + ansible_platform: 'amazon' + require_chef_for_busser: true + require_ruby_for_busser: false + verifier: + ruby_bindir: '/opt/chef/embedded/bin' + - name: amazonlinux-2016.09 + driver: + name: ec2 + image_id: ami-5ec1673e + aws_ssh_key_id: test-kitchen + region: us-west-2 + availability_zone: a + instance_type: t2.large + associate_public_ip: true + interface: dns + transport: + username: ec2-user + ssh_key: ~/.ssh/id_rsa + provisioner: + ansible_platform: 'amazon' + require_chef_for_busser: true + require_ruby_for_busser: false + verifier: + ruby_bindir: '/opt/chef/embedded/bin' +# Ansible doesn't provide Debian repos so we use the omnibus installer to provide Ansible + - name: debian-7 + driver: + box: debian/wheezy64 + provisioner: + require_ansible_omnibus: true + require_ansible_repo: false + - name: debian-8 + driver: + box: debian/jessie64 + provisioner: + require_ansible_omnibus: true + require_ansible_repo: false +# Ubuntu 12.04 uses Ruby 1.8.7 while busser requires Ruby 1.9+, so we need to use Chef to run busser + - name: ubuntu-12.04 + driver: + box: ubuntu/precise64 + provisioner: + require_chef_for_busser: true + require_ruby_for_busser: false + verifier: + ruby_bindir: '/opt/chef/embedded/bin' + - name: ubuntu-14.04 + driver: + box: ubuntu/trusty64 + - name: ubuntu-15.04 + driver: + box: larryli/vivid64 + - name: ubuntu-15.10 + driver: + box: ubuntu/wily64 + - name: ubuntu-16.04 + driver: + box: ubuntu/xenial64 + +suites: + - name: default diff --git a/clojurians_log/playbooks/roles/librato.librato/Ansiblefile b/clojurians_log/playbooks/roles/librato.librato/Ansiblefile new file mode 100644 index 0000000..59c70e2 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/Ansiblefile @@ -0,0 +1,4 @@ +#!/usr/bin/env ruby +#^syntax detection + +site "https://galaxy.ansible.com/api/v1"; diff --git a/clojurians_log/playbooks/roles/librato.librato/Gemfile b/clojurians_log/playbooks/roles/librato.librato/Gemfile new file mode 100644 index 0000000..3b9bd23 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/Gemfile @@ -0,0 +1,5 @@ +source 'https://rubygems.org' + +gem 'kitchen-ansible' +gem 'kitchen-vagrant' +gem 'kitchen-verifier-serverspec' diff --git a/clojurians_log/playbooks/roles/librato.librato/README.md b/clojurians_log/playbooks/roles/librato.librato/README.md new file mode 100644 index 0000000..1b47390 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/README.md @@ -0,0 +1,559 @@ +# Ansible Role: Librato + +An Ansible role that installs and configures the Librato Agent and plugins + +## Requirements + +None + +## Dependencies + +None + +## Quickstart + +The bare minimum required to get started is to set the `email` and `token` variables and include the role: + +```yaml +- hosts: all + roles: + - { role: librato.librato, librato_email: 'you@domain.com', librato_token: 'your-token-here' } +``` + +By default, the following plugins are enabled: `cpu`, `df`, `disk`, `swap`, `memory`, `load`. + +To include a plugin, set any variables you need for it, then add it to the `librato_enabled_plugins` variable: + +```yaml +- hosts: all + vars: + librato_apache_path: '/my-status-endpoint' + librato_enabled_plugins: ['apache'] +``` + +### Supported Plugins + +* `cpu` +* `df` +* `memory` +* `load` +* `disk` +* `swap` +* `apache` +* `nginx` +* `nginx_plus` +* `jvm` +* `memcached` +* `varnish` +* `zookeeper` +* `docker` +* `elasticsearch` +* `mongodb` +* `postgresql` +* `mysql` +* `redis` +* `haproxy` + +## Usage + +### Using custom or upstream collectd plugins + +To use your own custom or upstream collectd plugin, simply have another module drop the config into `/opt/collectd/etc/collectd.conf.d/` and any custom plugins into `/opt/collectd/share/collectd/`. + +## Reference + +### Plugin: `apache` + - `librato_apache_protocol` + + **Type**: string + + The protocol to use. Defaults to `http`. Change this to `https` if you require SSL. + + - `librato_apache_host` + + **Type**: string + + The hostname to use. Defaults to `localhost`. + - `librato.apache.path` + + **Type**: string + + The path to the status page. Defaults to `/server-status`. `?auto` is automatically appended, so no need to include it. + + - `librato_apache_user` + + **Type**: string + + The username to use for password-protected status pages. Defaults to empty. + - `librato_apache_password` + + **Type**: string + + The password to use for password-protected status pages. Defaults to empty. + +### Plugin: `docker` + - `librato_docker_protocol` + + **Type**: string + + The protocol to use. Defaults to `http`. Change this to `https` if you require SSL. + + - `librato_docker_host` + + **Type**: string + + The Docker hostname. Defaults to `localhost`. + + - `librato_docker_port` + + **Type**: string + + The Docker port. Defaults to `2735` + +### Plugin: `elasticsearch` + + - `librato_elasticsearch_protocol` + + **Type**: string + + The protocol to use. Defaults to `http`. Change this to `https` if you require SSL. + + - `librato_elasticsearch_host` + + **Type**: string + + The ElasticSearch hostname. Defaults to `localhost`. + + - `librato_elasticsearch_port_` + + **Type**: string + + The ElasticSearch port. Defaults to `9200`. + + - `librato_elasticsearch_cluster_name` + + **Type**: string + + The ElasticSearch cluster name, if set. Defaults to `nil`. + + - `librato_elasticsearch_verbose` + + **Type**: true/false + + Verbosity trigger of the plugin. Defaults to `true`. + +### Plugin: `haproxy` + - `librato_haproxy_socket_file` + + **Type**: string + + The HAProxy socket file. Defaults to `/run/haproxy/admin.sock`. + + - `librato_haproxy_proxies` + + **Type**: array + + The default proxies to collect. Defaults to `server`, `frontend`, `backend`. + +### Plugin: `jvm` + - `librato_jvm_host` + + **Type**: string + + The JVM host. Defaults to `localhost`. + + - `librato_jvm_service_url` + + **Type**: string + + The JVM service URL to query. Defaults to `service:jmx:rmi:///jndi/rmi://localhost:17264/jmxrmi`. + + - `librato_jvm_mbeans` + + **Type**: mapping + + Additional mbeans to collect. Defaults to empty. + + Format of the mapping is: + ```yaml + librato_jvm_mbeans: [ + { + name: 'mbean name' + object_name: 'object name' + instance_prefix': 'instance prefix (optional)', + instance_from': 'instance from (optional)', + values: [ + { + type: 'value type', + table: true|false, + attribute: 'attribute' + instance_prefix: 'instance prefix (optional)', + instance_from: 'instance from (optional)' + } + ] + } + ] + ``` + +### Plugin: `memcached` + - `librato_memcached_host` + + **Type**: string + + The memcached hostname. Defaults to `localhost`. + + - `librato_memcached_port` + + **Type**: string + + The memcached port. Defaults to `11211`. + +### Plugin: `mongodb` + - `librato_mongodb_host` + + **Type**: string + + The MongoDB hostname. Defaults to `localhost`. + + - `librato_mongodb_port` + + **Type**: string + + The MongoDB port. Defaults to `27017`. + + - `librato_mongodb_user` + + **Type**: string + + The MongoDB username to connect with. Defaults to `nil`. + + - `librato_mongodb_password` + + **Type**: string + + The MongoDB password to connect with. Defaults to `nil`. + + - `librato_mongodb_databases` + + **Type**: array + + Databases to collect metrics for. Defaults to empty. `admin` database is automatically included in the array. + + - `librato_mongodb_name` + + **Type**: string + + Set the name of the plugin instance. Defaults to `mongodb`. + +### Plugin: `mysql` + - `librato_mysql_databases` + + **Type**: mapping + + Databases to collect metrics for. Defaults to empty. + + Format of the hash is: + ```yaml + databases: [ + { + name: 'mydb' + host: 'localhost' + port: 3306 + user: '' + password: '' + innodb_stats: true + slave_stats: false + } + ] + ``` + +### Plugin: `nginx` + - `librato_nginx_protocol` + + **Type**: string + + The protocol to use. Defaults to `http`. Change this to `https` if you require SSL. + + - `librato_nginx_host` + + **Type**: string + + The hostname to use. Defaults to `localhost`. + + - `librato_nginx_path` + + **Type**: string + + The path to the status page. Defaults to `/basic_status`. + +### Plugin: `nginx_plus` + - `librato_nginx_plus_protocol` + + **Type**: string + + The protocol to use. Defaults to `http`. Change this to `https` if you require SSL. + + - `librato_nginx_plus_host` + + **Type**: string + + The hostname to use. Defaults to `localhost`. + + - `librato_nginx_plus_path` + + **Type**: string + + The path to the status page. Defaults to `/status`. + + - `librato_nginx_plus_verbose` + + **Type**: string + + Verbosity on/off for the plugin. Defaults to `false`. + +### Plugin: `postgresql` + - `librato_postgresql_socket_file` + + **Type**: string + + The PostgreSQL socket file. Defaults to `/var/run/postgresql`. + + - `librato_postgresql_user` + + **Type**: string + + The PostgreSQL user to connect as. Defaults to `postgresql`. + + - `librato_postgresql_databases` + + **Type**: mapping + + The databases to collect metrics for. Defaults to empty. + + The format of the hash is: + ```yaml + databases = [ + { + name: 'mydb' + instance: 'baz' + host: 'localhost' + port: 5432 + user: '' + password: '' + ssl_mode: 'prefer' + } + ] + ``` + +### Plugin: `redis` + - `librato_redis_host` + + **Type**: string + + The Redis hostname. Defaults to `localhost`. + + - `librato_redis_port` + + **Type**: string + + The Redis port. Defaults to `6379`. + + - `librato_redis_timeout` + + **Type**: string + + The timeout for connecting to Redis in milliseconds. Defaults to `2000`. + +### Plugin: `varnish` + + Varnish has no configurable attributes. + +### Plugin: `zookeeper` + - `librato_zookeeper_host` + + **Type**: string + + The ZooKeeper hostname. Defaults to `localhost`. + + - `librato_zookeeper_port` + + **Type**: string + + The ZooKeeper port. Defaults to `2181`. + +### Global Variables + +- `librato_email` + + **Type**: string + + The email to use for sending metrics. Use in conjunction with `token`. This attribute is required and defaults to empty. + +- `librato_token` + + **Type**: string + + The API token to use for sending metrics. Use in conjunction with `email`. This attribute is required and defaults to empty. + +- `librato_version` + + **Type**: string + + The version of the Librato Agent to install. + +- `librato_repo_url` + + **Type**: string + + The base URL for the packages. Defaults to Librato's repo URL `https://packagecloud.io/librato/`. + +- `librato_repo_base` + + **Type**: string + + The repo base to use. Defaults to Librato's repo collection `librato-collectd`. + +- `librato_config_base` + + **Type**: string + + The base path for collectd's config files. Defaults to `/opt/collectd/etc`. + +- `librato_plugin_config_path` + + **Type**: string + + The path for collectd's plugin configs. Defaults to `/opt/collectd/etc/collectd.conf.d`. + +- `librato_hostname` + + **Type**: string + + The hostname to use for the node. Defaults to `nil` in favor of using `FQDNLookup`. + +- `librato_fqdn_lookup` + + **Type**: true/false + + Perform an FQDN lookup for the hostname. Defaults to `true`. + +- `librato_interval` + + **Type**: integer + + The global plugin polling interval in seconds. Defaults to `60`. + +- `librato_use_log_file` + + **Type**: true/false + + Write collectd logs to a file. Defaults to `true`. + +- `librato_use_syslog` + + **Type**: true/false + + Write collectd logs to syslog. Defaults to `false`. + +- `librato_use_logstash` + + **Type**: true/false + + Write collectd logs to a logstash-formatted file. Defaults to `false`. + +- `librato_log_file_log_level` + + **Type**: string + + The log level to use for `log_file`. Defaults to `info`. + +- `librato_log_file_filename` + + **Type**: string + + The filename to use for `log_file`. Defaults to `/opt/collectd/var/log/collectd.log`. + +- `librato_log_file_timestamp` + + **Type**: true/false + + Use timestamps in the log file or not. Defaults to `true`. + +- `librato_log_file_print_severity` + + **Type**: true/false + + Include severity levels in the log file or not. Defaults to `true`. + +- `librato_syslog_log_level` + + **Type**: string + + The log level to use for `syslog`. Defaults to `info`. + +- `librato_logstash_log_level` + + **Type**: string + + The log level to use for `logstash`. Defaults to `info`. + +- `librato_logstash_filename` + + **Type**: string + + The file name to use for `logstash`. Defaults to `/opt/collectd/var/log/collectd.json.log` + +- `librato_default_plugins` + + **Type**: array + + A list of default plugins to include. Defaults to: `cpu`, `df`, `disk`, `swap`, `memory`, `load`. + +- `librato_enabled_plugins` + + **Type**: array + + A list of plugins to enable. This is separate from the default plugins and is used to include plugins aside from those. Defaults to empty. + +### Supported Platforms + +* RHEL 6 / CentOS 6 +* RHEL 7 / CentOS 7 +* Fedora 23 +* Amazon Linux 2016.03 +* Ubuntu 12.04 +* Ubuntu 14.04 +* Ubuntu 15.04 +* Ubuntu 15.10 +* Ubuntu 16.04 +* Debian 7 +* Debian 8 + +### Supported Ansible Versions + +This role requires Ansible 2.2 or later. + +## Development + +### Testing + +Integration tests utilize `kitchen-ansible` and `serverspec`. To run the test suite: + +1. Run `bundle install` +2. Run `kitchen test` + +#### Testing Amazon Linux + +Testing Amazon Linux through `test-kitchen` requires a bit more setup: + +1. Ensure `kitchen-ec2` is installed: `gem install kitchen-ec2` +2. Update `.kitchen.yml` to have the correct AWS key ID you're going to use +3. Set `security_group_ids` in the driver section to include a security group accessible from your laptop. Not setting this will use the `default` security group. +4. Set `transport.ssh_key` to the path of your SSH key. It looks for `id_rsa` by default. + +## License + +MIT + +## Authors & Contributors +Mike Julian (@mjulian) diff --git a/clojurians_log/playbooks/roles/librato.librato/defaults/main.yml b/clojurians_log/playbooks/roles/librato.librato/defaults/main.yml new file mode 100644 index 0000000..07b2496 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/defaults/main.yml @@ -0,0 +1,73 @@ +librato_email: '' +librato_token: '' +librato_repo_url: 'https://packagecloud.io/librato/' +librato_repo_base: 'librato-collectd' +librato_deb_version: '5.5.0-librato51.251' +librato_rh_version: '5.5.0_librato51.251' +librato_config_base: '/opt/collectd/etc' +librato_plugin_config_path: '/opt/collectd/etc/collectd.conf.d' +librato_fqdn_lookup: true +librato_interval: 60 +librato_default_plugins: ['cpu', 'df', 'disk', 'swap', 'memory', 'load'] +librato_enabled_plugins: [] + +librato_logging_use_log_file: true +librato_logging_use_syslog: false +librato_logging_use_logstash: false +librato_logging_log_file_log_level: 'info' +librato_logging_log_file_filename: '/opt/collectd/var/log/collectd.log' +librato_logging_log_file_timestamp: true +librato_logging_log_file_print_severity: false +librato_logging_syslog_log_level: 'info' +librato_logging_logstash_log_level: 'info' +librato_logging_logstash_filename: '/opt/collectd/var/log/collectd.json.log' + +librato_apache_protocol: 'http' +librato_apache_host: 'localhost' +librato_apache_path: '/server-status' + +librato_nginx_protocol: 'http' +librato_nginx_host: 'localhost' +librato_nginx_path: '/basic_status' + +librato_nginx_plus_protocol: 'http' +librato_nginx_plus_host: 'localhost' +librato_nginx_plus_path: '/status' +librato_nginx_plus_verbose: false + +librato_memcached_host: 'localhost' +librato_memcached_port: '11211' + +librato_elasticsearch_protocol: 'http' +librato_elasticsearch_host: 'localhost' +librato_elasticsearch_port: '9200' +librato_elasticsearch_verbose: true + +librato_mongodb_host: 'localhost' +librato_mongodb_port: '27017' +librato_mongodb_databases: [] +librato_mongodb_name: 'mongodb' + +librato_zookeeper_host: 'localhost' +librato_zookeeper_port: '2181' + +librato_haproxy_socket_file: '/run/haproxy/admin.sock' +librato_haproxy_proxies: ['server', 'frontend', 'backend'] + +librato_redis_host: 'localhost' +librato_redis_port: '6379' +librato_redis_timeout: '2000' + +librato_mysql_databases: [] + +librato_postgresql_databases: [] +librato_postgresql_socket_file: '/var/run/postgresql' +librato_postgresql_user: 'postgresql' + +librato_docker_protocol: 'http' +librato_docker_host: 'localhost' +librato_docker_port: '2735' + +librato_jvm_host: 'localhost' +librato_jvm_service_url: 'service:jmx:rmi:///jndi/rmi://localhost:17264/jmxrmi' +librato_jvm_mbeans: [] diff --git a/clojurians_log/playbooks/roles/librato.librato/files/collectd-core.pref b/clojurians_log/playbooks/roles/librato.librato/files/collectd-core.pref new file mode 100644 index 0000000..17d7d98 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/files/collectd-core.pref @@ -0,0 +1,3 @@ +Package: collectd-core +Pin: release l=librato-collectd +Pin-Priority: 1001 diff --git a/clojurians_log/playbooks/roles/librato.librato/files/collectd.pref b/clojurians_log/playbooks/roles/librato.librato/files/collectd.pref new file mode 100644 index 0000000..c34a0cb --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/files/collectd.pref @@ -0,0 +1,3 @@ +Package: collectd +Pin: release l=librato-collectd +Pin-Priority: 1001 diff --git a/clojurians_log/playbooks/roles/librato.librato/files/cpu.conf b/clojurians_log/playbooks/roles/librato.librato/files/cpu.conf new file mode 100644 index 0000000..6a403d0 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/files/cpu.conf @@ -0,0 +1,4 @@ +LoadPlugin cpu + + ReportByCpu false + diff --git a/clojurians_log/playbooks/roles/librato.librato/files/df.conf b/clojurians_log/playbooks/roles/librato.librato/files/df.conf new file mode 100644 index 0000000..cd0f609 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/files/df.conf @@ -0,0 +1,15 @@ +LoadPlugin df + + FSType "ext2" + FSType "ext3" + FSType "ext4" + FSType "xfs" + FSType "jfs" + FSType "btrfs" + FSType "reiserfs" + FSType "vboxsf" + FSType "tmpfs" + ReportInodes false + ValuesAbsolute false + ValuesPercentage true + diff --git a/clojurians_log/playbooks/roles/librato.librato/files/disk.conf b/clojurians_log/playbooks/roles/librato.librato/files/disk.conf new file mode 100644 index 0000000..db9bddb --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/files/disk.conf @@ -0,0 +1 @@ +LoadPlugin disk diff --git a/clojurians_log/playbooks/roles/librato.librato/files/load.conf b/clojurians_log/playbooks/roles/librato.librato/files/load.conf new file mode 100644 index 0000000..5a65eda --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/files/load.conf @@ -0,0 +1 @@ +LoadPlugin load diff --git a/clojurians_log/playbooks/roles/librato.librato/files/memory.conf b/clojurians_log/playbooks/roles/librato.librato/files/memory.conf new file mode 100644 index 0000000..2a1db34 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/files/memory.conf @@ -0,0 +1,5 @@ +LoadPlugin memory + + ValuesAbsolute true + ValuesPercentage false + diff --git a/clojurians_log/playbooks/roles/librato.librato/files/swap.conf b/clojurians_log/playbooks/roles/librato.librato/files/swap.conf new file mode 100644 index 0000000..9ac8d80 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/files/swap.conf @@ -0,0 +1 @@ +LoadPlugin swap diff --git a/clojurians_log/playbooks/roles/librato.librato/handlers/main.yml b/clojurians_log/playbooks/roles/librato.librato/handlers/main.yml new file mode 100644 index 0000000..c055939 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: collectd + service: + name: collectd + state: restarted diff --git a/clojurians_log/playbooks/roles/librato.librato/meta/.galaxy_install_info b/clojurians_log/playbooks/roles/librato.librato/meta/.galaxy_install_info new file mode 100644 index 0000000..b2a7238 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/meta/.galaxy_install_info @@ -0,0 +1 @@ +{install_date: 'Fri Aug 23 08:21:22 2019', version: 1.0.3} diff --git a/clojurians_log/playbooks/roles/librato.librato/meta/main.yml b/clojurians_log/playbooks/roles/librato.librato/meta/main.yml new file mode 100644 index 0000000..7e99b35 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/meta/main.yml @@ -0,0 +1,30 @@ +--- +galaxy_info: + author: librato + description: Install and configure the Librato Agent + company: Librato + license: MIT + min_ansible_version: 2.2 + platforms: + - name: EL + versions: + - 6 + - 7 + - name: Fedora + versions: + - 23 + - name: Ubuntu + versions: + - trusty + - precise + - xenial + - wily + - vivid + - name: Debian + versions: + - jessie + - wheezy + galaxy_tags: + - librato + - collectd + - monitoring diff --git a/clojurians_log/playbooks/roles/librato.librato/requirements.yml b/clojurians_log/playbooks/roles/librato.librato/requirements.yml new file mode 100644 index 0000000..41a6d0d --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/requirements.yml @@ -0,0 +1,3 @@ +- librato.librato +- name: role-public + src: https://github.com/librato/ansible-librato.git diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/agent.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/agent.yml new file mode 100644 index 0000000..261f566 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/agent.yml @@ -0,0 +1,83 @@ +- name: Install package (Debian/Ubuntu) + apt: + name: collectd-core={{ librato_deb_version }} + state: present + notify: + - collectd + when: ansible_os_family == 'Debian' + +- name: Install package (RedHat/CentOS/Amazon) + yum: + name: collectd-core-{{ librato_rh_version }} + state: present + notify: + - collectd + when: + - ansible_os_family == 'RedHat' + - ansible_distribution != 'Fedora' + +- name: Install package (Fedora) + dnf: + name: collectd-core-{{ librato_rh_version }} + state: present + notify: + - collectd + when: ansible_distribution == 'Fedora' + +- name: setup_collectd_config + template: + src: collectd.conf.jinja + dest: "{{ librato_config_base }}/collectd.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd + +- name: setup_logging_plugin + template: + src: logging.conf.jinja + dest: "{{ librato_plugin_config_path }}/logging.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd + +- name: setup_librato_plugin + template: + src: librato.conf.jinja + dest: "{{ librato_plugin_config_path }}/librato.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd + +- name: Default plugins + copy: + src: "{{ item }}.conf" + dest: "{{ librato_plugin_config_path }}/{{ item }}.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd + with_items: "{{ librato_default_plugins }}" + +- name: Additional enabled plugins + template: + src: "{{ item }}.conf.jinja" + dest: "{{ librato_plugin_config_path }}/{{ item }}.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd + with_items: "{{ librato_enabled_plugins }}" + +- name: collectd_service + service: + name: collectd + state: started + enabled: yes diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/apache.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/apache.yml new file mode 100644 index 0000000..2a12b93 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/apache.yml @@ -0,0 +1,9 @@ +- name: Apache plugin + copy: + src: "apache.conf.jinja" + dest: "{{ plugin_config_path }}/apache.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/docker.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/docker.yml new file mode 100644 index 0000000..b58b91b --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/docker.yml @@ -0,0 +1,9 @@ +- name: Docker plugin + copy: + src: "docker.conf.jinja" + dest: "{{ plugin_config_path }}/docker.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/elasticsearch.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/elasticsearch.yml new file mode 100644 index 0000000..34d2643 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/elasticsearch.yml @@ -0,0 +1,9 @@ +- name: Elasticsearch plugin + copy: + src: "elasticsearch.conf.jinja" + dest: "{{ plugin_config_path }}/elasticsearch.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/haproxy.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/haproxy.yml new file mode 100644 index 0000000..601ffbb --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/haproxy.yml @@ -0,0 +1,9 @@ +- name: haproxy plugin + copy: + src: "haproxy.conf.jinja" + dest: "{{ plugin_config_path }}/haproxy.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/jvm.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/jvm.yml new file mode 100644 index 0000000..0d34b23 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/jvm.yml @@ -0,0 +1,9 @@ +- name: JVM plugin + copy: + src: "jvm.conf.jinja" + dest: "{{ plugin_config_path }}/jvm.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/main.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/main.yml new file mode 100644 index 0000000..9912c8b --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/main.yml @@ -0,0 +1,3 @@ +--- +- include: repo.yml tags=librato +- include: agent.yml tags=librato diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/memcached.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/memcached.yml new file mode 100644 index 0000000..1ea14f5 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/memcached.yml @@ -0,0 +1,9 @@ +- name: memcached plugin + copy: + src: "memcached.conf.jinja" + dest: "{{ plugin_config_path }}/memcached.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/mongodb.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/mongodb.yml new file mode 100644 index 0000000..c21ae90 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/mongodb.yml @@ -0,0 +1,9 @@ +- name: MongoDB plugin + copy: + src: "mongodb.conf.jinja" + dest: "{{ plugin_config_path }}/mongodb.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/mysql.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/mysql.yml new file mode 100644 index 0000000..7d07537 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/mysql.yml @@ -0,0 +1,9 @@ +- name: MySQL plugin + copy: + src: "mysql.conf.jinja" + dest: "{{ plugin_config_path }}/mysql.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/nginx.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/nginx.yml new file mode 100644 index 0000000..477b883 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/nginx.yml @@ -0,0 +1,9 @@ +- name: nginx plugin + copy: + src: "nginx.conf.jinja" + dest: "{{ plugin_config_path }}/nginx.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/nginx_plus.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/nginx_plus.yml new file mode 100644 index 0000000..9be1966 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/nginx_plus.yml @@ -0,0 +1,9 @@ +- name: nginx_plus plugin + copy: + src: "nginx_plus.conf.jinja" + dest: "{{ plugin_config_path }}/nginx_plus.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/postgresql.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/postgresql.yml new file mode 100644 index 0000000..fd73e42 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/postgresql.yml @@ -0,0 +1,9 @@ +- name: PostgreSQL plugin + copy: + src: "postgresql.conf.jinja" + dest: "{{ plugin_config_path }}/postgresql.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/redis.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/redis.yml new file mode 100644 index 0000000..399aad4 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/redis.yml @@ -0,0 +1,9 @@ +- name: Redis plugin + copy: + src: "redis.conf.jinja" + dest: "{{ plugin_config_path }}/redis.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/repo.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/repo.yml new file mode 100644 index 0000000..c3fc214 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/repo.yml @@ -0,0 +1,78 @@ +- name: Install EPEL repo (RedHat/CentOS/Amazon) + yum: + name: epel-release + state: present + when: + - ansible_os_family == 'RedHat' + - ansible_distribution != 'Fedora' + +- name: Install gpg key + apt_key: + url: https://packagecloud.io/librato/librato-collectd/gpgkey + id: B83E13B25EA683E71C17FFA0BDBDCC081E77728D + state: present + when: ansible_os_family == 'Debian' + +- name: Install apt-transport-https + apt: + name: apt-transport-https + state: present + when: ansible_os_family == 'Debian' + +- name: Create apt repo + apt_repository: + # no packages (currently) for bionic, hard coding to use xenial instead + # repo: 'deb {{ librato_repo_url }}{{ librato_repo_base }}/{{ ansible_distribution|lower }}/ {{ ansible_distribution_release }} main' + repo: 'deb {{ librato_repo_url }}{{ librato_repo_base }}/{{ ansible_distribution|lower }}/ xenial main' + state: present + when: ansible_os_family == 'Debian' + +- name: collectd_apt_pin + copy: + src: collectd.pref + dest: /etc/apt/preferences.d/collectd.pref + owner: root + group: root + mode: 0644 + when: ansible_os_family == 'Debian' + +- name: collectd-core_apt_pin + copy: + src: collectd-core.pref + dest: /etc/apt/preferences.d/collectd-core.pref + owner: root + group: root + mode: 0644 + when: ansible_os_family == 'Debian' + +- name: Run apt-get update + apt: + update_cache: yes + when: ansible_os_family == 'Debian' + +- name: Create yum repo + yum_repository: + name: 'librato_{{ librato_repo_base }}' + description: 'librato_{{ librato_repo_base }}' + baseurl: '{{ librato_repo_url }}{{ librato_repo_base }}/el/$releasever/$basearch' + gpgcheck: no + when: + - ansible_os_family == 'RedHat' + - ansible_distribution != 'Fedora' + - ansible_distribution != 'Amazon' + +- name: Create yum repo (Fedora) + yum_repository: + name: 'librato_{{ librato_repo_base }}' + description: 'librato_{{ librato_repo_base }}' + baseurl: '{{ librato_repo_url }}{{ librato_repo_base }}/fedora/$releasever/$basearch' + gpgcheck: no + when: ansible_distribution == 'Fedora' + +- name: Create yum repo (Amazon Linux) + yum_repository: + name: 'librato_{{ librato_repo_base }}' + description: 'librato_{{ librato_repo_base }}' + baseurl: '{{ librato_repo_url }}librato-amazonlinux-collectd/el/6/$basearch' + gpgcheck: no + when: ansible_distribution == 'Amazon' diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/varnish.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/varnish.yml new file mode 100644 index 0000000..d6883df --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/varnish.yml @@ -0,0 +1,9 @@ +- name: Varnish plugin + copy: + src: "varnish.conf.jinja" + dest: "{{ plugin_config_path }}/varnish.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/tasks/zookeeper.yml b/clojurians_log/playbooks/roles/librato.librato/tasks/zookeeper.yml new file mode 100644 index 0000000..d7d0db1 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/tasks/zookeeper.yml @@ -0,0 +1,9 @@ +- name: ZooKeeper plugin + copy: + src: "zookeeper.conf.jinja" + dest: "{{ plugin_config_path }}/zookeeper.conf" + owner: root + group: root + mode: 0644 + notify: + - collectd diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/apache.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/apache.conf.jinja new file mode 100644 index 0000000..4ea621e --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/apache.conf.jinja @@ -0,0 +1,12 @@ +LoadPlugin "apache" + + + URL "{{ librato_apache_protocol }}://{{ librato_apache_host }}{{ librato_apache_path }}?auto" +{% if librato_apache_user is defined %} + User "{{ librato_apache_user }}" +{% endif %} +{% if librato_apache_password is defined %} + Password "{{ librato_apache_password }}" +{% endif %} + + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/collectd.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/collectd.conf.jinja new file mode 100644 index 0000000..64b76d3 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/collectd.conf.jinja @@ -0,0 +1,23 @@ +# This file is managed by Ansible. Local changes will be overwritten. +# +# +# Config file for collectd(1). +# Please read collectd.conf(5) for a list of options. +# http://collectd.org/ +# + +############################################################################## +# Global # +#----------------------------------------------------------------------------# +# Global settings for the daemon. # +############################################################################## + +{% if librato_hostname is defined %} +Hostname "{{ librato_hostname }}" +{% endif %} +FQDNLookup {{ librato_fqdn_lookup }} +Interval {{ librato_interval }} + +# Include logging config before plugins collectd logs plugin log entries +Include "{{ librato_plugin_config_path }}/logging.conf" +Include "{{ librato_plugin_config_path }}/*.conf" diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/docker.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/docker.conf.jinja new file mode 100644 index 0000000..8692167 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/docker.conf.jinja @@ -0,0 +1,4 @@ +LoadPlugin exec + + Exec "nobody" "/opt/collectd/share/collectd/collectd-docker.py" "{{ librato_docker_protocol }}://{{ librato_docker_host }}:{{ librato_docker_port }}" + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/elasticsearch.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/elasticsearch.conf.jinja new file mode 100644 index 0000000..4daf697 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/elasticsearch.conf.jinja @@ -0,0 +1,17 @@ + + Globals true + + + + ModulePath "/opt/collectd/share/collectd/" + + Import "collectd-elasticsearch" + + + Url "{{ librato_elasticsearch_protocol }}://{{ librato_elasticsearch_host }}:{{ librato_elasticsearch_port }}" + {% if librato_elasticsearch_cluster_name is defined %} + Name "{{ librato_elasticsearch_cluster_name }}" + {% endif %} + Verbose {{ librato_elasticsearch_verbose }} + + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/haproxy.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/haproxy.conf.jinja new file mode 100644 index 0000000..e907445 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/haproxy.conf.jinja @@ -0,0 +1,12 @@ +LoadPlugin python + + ModulePath "/opt/collectd/share/collectd" + Import "collectd-haproxy" + + + Socket "{{ librato_haproxy_socket_file }}" + {% for proxy in librato_haproxy_proxies %} + ProxyMonitor "{{ proxy }}}" + {% endfor %} + + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/jvm.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/jvm.conf.jinja new file mode 100644 index 0000000..17784a3 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/jvm.conf.jinja @@ -0,0 +1,51 @@ +LoadPlugin java + +Include "/opt/collectd/share/collectd/jvm_default.conf" + + + + + Host "{{ librato_jvm_host }}" + ServiceURL "{{ librato_jvm_service_url }}" + Collect "memory_heap" + Collect "memory_nonheap" + Collect "classes" + Collect "threads" + Collect "compilation" + Collect "garbage_collector" + Collect "memory_pool" + {% if librato_jvm_mbeans is iterable %} + {% for mbean in librato_jvm_mbeans %} + Collect "{{ mbean.name }}" + {% endfor %} + {% endif %} + +{% if librato_jvm_mbeans is iterable %} + {% for mbean in librato_jvm_mbeans %} + + ObjectName "{{ mbean.object_name }}" + {% if 'instance_prefix' in mbean %} + InstancePrefix "{{ mbean.instance_prefix }}" + {% endif %} + {% if 'instance_from' in mbean %} + InstanceFrom "{{ mbean.instance_from }}" + {% endif %} + + {% for value in mbean['values'] %} + + Type "{{ value.type }}" + Table {{ value.table }} + Attribute "{{ value.attribute }}" + {% if 'instance_prefix' in value %} + InstancePrefix "{{ value.instance_prefix }}" + {% endif %} + {% if 'instance_from' in value %} + InstanceFrom "{{ value.instance_from }}" + {% endif %} + + {% endfor %} + + {% endfor %} +{% endif %} + + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/librato.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/librato.conf.jinja new file mode 100644 index 0000000..039eba3 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/librato.conf.jinja @@ -0,0 +1,11 @@ +LoadPlugin write_http + + + URL "https://collectd.librato.com/v1/measurements" + Format "JSON" + BufferSize 8192 + + User "{{ librato_email }}" + Password "{{ librato_token }}" + + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/logging.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/logging.conf.jinja new file mode 100644 index 0000000..e07178c --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/logging.conf.jinja @@ -0,0 +1,37 @@ +############################################################################## +# Logging # +#----------------------------------------------------------------------------# +# Plugins which provide logging functions should be loaded first, so log # +# messages generated when loading or configuring other plugins can be # +# accessed. # +############################################################################## + +{% if not librato_logging_use_log_file and not librato_logging_use_syslog and not librato_logging_use_logstash %} +### No logging configured +{% endif %} + +{% if librato_logging_use_log_file %} +LoadPlugin logfile + + + LogLevel {{ librato_logging_log_file_log_level }} + File "{{ librato_logging_log_file_filename }}" + Timestamp {{ librato_logging_log_file_timestamp }} + PrintSeverity {{ librato_logging_log_file_print_severity }} + +{% endif %} +{% if librato_logging_use_syslog %} +LoadPlugin syslog + + + LogLevel {{ librato_logging_syslog_log_level }} + +{% endif %} +{% if librato_logging_use_logstash %} +LoadPlugin log_logstash + + + LogLevel {{ librato_logging_logstash_log_level }} + File "{{ librato_logging_logstash_filename }}" + +{% endif %} diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/memcached.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/memcached.conf.jinja new file mode 100644 index 0000000..6460d2d --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/memcached.conf.jinja @@ -0,0 +1,7 @@ +LoadPlugin memcached + + + Host "{{ librato_memcached_host }}" + Port "{{ librato_memcached_port }}" + + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/mongodb.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/mongodb.conf.jinja new file mode 100644 index 0000000..1078511 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/mongodb.conf.jinja @@ -0,0 +1,23 @@ + + Globals true + + + + ModulePath "/opt/collectd/share/collectd" + ModulePath "/opt/collectd/share/collectd/pymongo.egg" + + Import "collectd-mongodb" + + + Host "{{ librato_mongodb_host }}" + Port "{{ librato_mongodb_port }}" + {% if librato_mongodb_user is defined %} + User "{{ librato_mongodb_user }}" + {% endif %} + {% if librato_mongodb_password is defined %} + Password "{{ librato_mongodb_password }}" + {% endif %} + Database "admin", {{ '\"' + librato_mongodb_databases|join('\", \"') + '\"' }} + Name "{{ librato_mongodb_name }}" + + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/mysql.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/mysql.conf.jinja new file mode 100644 index 0000000..1854e77 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/mysql.conf.jinja @@ -0,0 +1,23 @@ +LoadPlugin mysql + +{% for db in librato_mysql_databases %} + + Host "{{ db.host }}" + User "{{ db.user }}" + {% if 'password' in db %} + Password "{{ db.password }}" + {% endif %} + {% if 'port' in db %} + Port {{ db.port }} + {% else %} + Port 3306 + {% endif %} + {% if 'innodb_stats' in db %} + InnodbStats {{ db.innodb_stats }} + {% endif %} + {% if 'slave_stats' in db %} + SlaveStats {{ db.slave_stats }} + {% endif %} + +{% endfor %} + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/nginx.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/nginx.conf.jinja new file mode 100644 index 0000000..e1c6bfd --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/nginx.conf.jinja @@ -0,0 +1,4 @@ +LoadPlugin nginx + + URL "{{ librato_nginx_protocol }}://{{ librato_nginx_host }}{{ librato_nginx_path }}" + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/nginx_plus.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/nginx_plus.conf.jinja new file mode 100644 index 0000000..17688d8 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/nginx_plus.conf.jinja @@ -0,0 +1,11 @@ +LoadPlugin python + + ModulePath "/opt/collectd/share/collectd" + Import "collectd-nginx_plus" + LogTraces true + + + URL "{{ librato_nginx_plus_protocol }}://{{ librato_nginx_plus_host }}{{ librato_nginx_plus_path }}" + Verbose {{ librato_nginx_plus_verbose }} + + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/postgresql.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/postgresql.conf.jinja new file mode 100644 index 0000000..fd30dfd --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/postgresql.conf.jinja @@ -0,0 +1,34 @@ +LoadPlugin postgresql + + + Host "{{ librato_postgresql_socket_file }}" + User "{{ librato_postgresql_user }}" + Query connections + Query transactions + Query queries + Query query_plans + Query table_states + Query disk_io + Query disk_usage + +{% for db in librato_postgresql_databases %} + + Instance "{{ db.instance }}" + Host "{{ db.host }}" + {% if 'port' in db %} + Port "{{ db.port }}" + {% else %} + Port 5432 + {% endif %} + User "{{ db.user }}" + {% if 'password' in db %} + Password "{{ db.password }}" + {% endif %} + {% if 'ssl_mode' in db %} + SSLMode "{{ db.ssl_mode }}" + {% else %} + SSLMode "Prefer" + {% endif %} + +{% endfor %} + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/redis.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/redis.conf.jinja new file mode 100644 index 0000000..7b238c8 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/redis.conf.jinja @@ -0,0 +1,8 @@ +LoadPlugin redis + + + Host "{{ librato_redis_host }}" + Port "{{ librato_redis_port }}" + Timeout {{ librato_redis_timeout }} + + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/varnish.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/varnish.conf.jinja new file mode 100644 index 0000000..7b27c2f --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/varnish.conf.jinja @@ -0,0 +1,14 @@ +LoadPlugin varnish + + + CollectBackend true + CollectCache true + CollectConnections true + CollectFetch true + CollectObjects true + CollectSession true + CollectSHM true + CollectTotals true + CollectWorkers true + + diff --git a/clojurians_log/playbooks/roles/librato.librato/templates/zookeeper.conf.jinja b/clojurians_log/playbooks/roles/librato.librato/templates/zookeeper.conf.jinja new file mode 100644 index 0000000..bcbd12f --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/templates/zookeeper.conf.jinja @@ -0,0 +1,5 @@ +LoadPlugin zookeeper + + Host "{{ librato_zookeeper_host }}" + Port "{{ librato_zookeeper_port }}" + diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/default.yml b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/default.yml new file mode 100644 index 0000000..7265c37 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/default.yml @@ -0,0 +1,55 @@ +--- +- hosts: all + + vars: + librato_email: 'foo@bar.baz' + librato_token: '1234abcd' + librato_logging_use_log_file: true + librato_enabled_plugins: ['docker', 'haproxy', 'memcached', 'mysql', 'nginx', 'redis', 'varnish', 'apache', 'elasticsearch', 'jvm', 'mongodb', 'nginx_plus', 'postgresql', 'zookeeper'] + librato_jvm_mbeans: [ + { + 'name': 'foo', + 'object_name': 'objname', + 'instance_prefix': 'prefix', + 'instance_from': 'from', + 'values': [ + { + 'type': 'type', + 'table': true, + 'attribute': 'att' + }, + { + 'type': 'type', + 'table': true, + 'attribute': 'att' + } + ] + } + ] + librato_mongodb_databases: ['foo', 'bar', 'baz'] + librato_nginx_path: '/test-status' + librato_nginx_plus_path: '/test-status' + librato_postgresql_databases: [ + { + name: 'mydb', + instance: 'baz', + host: 'localhost', + port: 5432, + user: 'foo', + password: 'baz', + } + ] + librato_mysql_databases: [ + { + name: 'mydb', + host: 'localhost', + port: 3306, + user: 'foo', + password: 'baz', + innodb_stats: true, + } + ] + librato_apache_path: '/test-status' + + roles: + - ansible-librato diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/agent_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/agent_spec.rb new file mode 100644 index 0000000..38d7092 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/agent_spec.rb @@ -0,0 +1,28 @@ +require 'serverspec' +set :backend, :exec + +describe package('collectd-core') do + it { should be_installed } +end + +describe service('collectd') do + it { should be_running } +end + +describe file('/opt/collectd/etc/collectd.conf.d/librato.conf') do + it { should exist } + it { should contain 'User "foo@bar.baz"' } + it { should contain 'Password "1234abcd"' } +end + +describe file('/opt/collectd/var/log/collectd.log') do + it { should exist } + it { should contain 'Initialization complete' } +end + +%w(cpu df disk swap memory load).each do |plugin| + describe file("/opt/collectd/etc/collectd.conf.d/#{plugin}.conf") do + it { should exist } + it { should contain "LoadPlugin #{plugin}" } + end +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/apache_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/apache_spec.rb new file mode 100644 index 0000000..4c802d4 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/apache_spec.rb @@ -0,0 +1,8 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/apache.conf') do + it { should exist } + it { should contain 'LoadPlugin "apache"' } + it { should contain 'URL "http://localhost/test-status?auto"' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/docker_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/docker_spec.rb new file mode 100644 index 0000000..47181f7 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/docker_spec.rb @@ -0,0 +1,8 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/docker.conf') do + it { should exist } + it { should contain 'LoadPlugin exec' } + it { should contain 'Exec "nobody" "/opt/collectd/share/collectd/collectd-docker.py" "http://localhost:2735"' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/elasticsearch_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/elasticsearch_spec.rb new file mode 100644 index 0000000..5e6a1e3 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/elasticsearch_spec.rb @@ -0,0 +1,8 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/elasticsearch.conf') do + it { should exist } + it { should contain 'LoadPlugin "python"' } + it { should contain 'Url "http://localhost:9200"' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/haproxy_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/haproxy_spec.rb new file mode 100644 index 0000000..35dfc4f --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/haproxy_spec.rb @@ -0,0 +1,8 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/haproxy.conf') do + it { should exist } + it { should contain 'LoadPlugin python' } + it { should contain 'Socket "/run/haproxy/admin.sock"' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/jvm_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/jvm_spec.rb new file mode 100644 index 0000000..c501ef5 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/jvm_spec.rb @@ -0,0 +1,10 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/jvm.conf') do + it { should exist } + it { should contain 'LoadPlugin java' } + it { should contain 'ServiceURL "service:jmx:rmi:///jndi/rmi://localhost:17264/jmxrmi"' } + it { should contain 'Collect "foo"' } + it { should contain '' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/memcached_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/memcached_spec.rb new file mode 100644 index 0000000..9f85802 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/memcached_spec.rb @@ -0,0 +1,9 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/memcached.conf') do + it { should exist } + it { should contain 'LoadPlugin memcached' } + it { should contain 'Host "localhost"' } + it { should contain 'Port "11211"' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/mongodb_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/mongodb_spec.rb new file mode 100644 index 0000000..f9c9ab4 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/mongodb_spec.rb @@ -0,0 +1,10 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/mongodb.conf') do + it { should exist } + it { should contain 'LoadPlugin python' } + it { should contain 'Host "localhost"' } + it { should contain 'Port "27017"' } + it { should contain 'Database "admin", "foo", "bar", "baz"' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/mysql_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/mysql_spec.rb new file mode 100644 index 0000000..93209ce --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/mysql_spec.rb @@ -0,0 +1,10 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/mysql.conf') do + it { should exist } + it { should contain 'LoadPlugin mysql' } + it { should contain 'Database "mydb"' } + it { should contain 'User "foo"' } + it { should contain 'Password "baz"' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/nginx_plus_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/nginx_plus_spec.rb new file mode 100644 index 0000000..0e7ae2e --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/nginx_plus_spec.rb @@ -0,0 +1,8 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/nginx_plus.conf') do + it { should exist } + it { should contain 'LoadPlugin python' } + it { should contain 'URL "http://localhost/test-status"' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/nginx_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/nginx_spec.rb new file mode 100644 index 0000000..2ccd9f1 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/nginx_spec.rb @@ -0,0 +1,8 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/nginx.conf') do + it { should exist } + it { should contain 'LoadPlugin nginx' } + it { should contain 'URL "http://localhost/test-status"' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/postgresql_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/postgresql_spec.rb new file mode 100644 index 0000000..fe45180 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/postgresql_spec.rb @@ -0,0 +1,11 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/postgresql.conf') do + it { should exist } + it { should contain 'LoadPlugin postgresql' } + it { should contain 'Database mydb' } + it { should contain 'Database postgres' } + it { should contain 'Instance "baz"' } + it { should contain 'Host "/var/run/postgresql"' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/redis_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/redis_spec.rb new file mode 100644 index 0000000..df3b965 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/redis_spec.rb @@ -0,0 +1,9 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/redis.conf') do + it { should exist } + it { should contain 'LoadPlugin redis' } + it { should contain 'Host "localhost"' } + it { should contain 'Port "6379"' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/repo_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/repo_spec.rb new file mode 100644 index 0000000..1a6b679 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/repo_spec.rb @@ -0,0 +1 @@ +require 'serverspec' diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/varnish_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/varnish_spec.rb new file mode 100644 index 0000000..afc6781 --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/varnish_spec.rb @@ -0,0 +1,7 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/varnish.conf') do + it { should exist } + it { should contain 'LoadPlugin varnish' } +end diff --git a/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/zookeeper_spec.rb b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/zookeeper_spec.rb new file mode 100644 index 0000000..3fd238a --- /dev/null +++ b/clojurians_log/playbooks/roles/librato.librato/test/integration/default/serverspec/zookeeper_spec.rb @@ -0,0 +1,9 @@ +require 'serverspec' +set :backend, :exec + +describe file('/opt/collectd/etc/collectd.conf.d/zookeeper.conf') do + it { should exist } + it { should contain 'LoadPlugin zookeeper' } + it { should contain 'Host "localhost"' } + it { should contain 'Port "2181"' } +end diff --git a/clojurians_log/playbooks/roles/plexus.clojurians-log/tasks/main.yml b/clojurians_log/playbooks/roles/plexus.clojurians-log/tasks/main.yml index af97f5c..9bc8eac 100644 --- a/clojurians_log/playbooks/roles/plexus.clojurians-log/tasks/main.yml +++ b/clojurians_log/playbooks/roles/plexus.clojurians-log/tasks/main.yml @@ -37,6 +37,21 @@ group: "{{ clojure_app_user }}" mode: 0600 +- name: Clone the Clojurians log app + become: yes + become_user: "{{ clojure_app_user }}" + git: + repo: "https://github.com/clojureverse/clojurians-log-app.git" + dest: "/tmp/clojurians-log-tmp" + +- name: Push master to the bare repo + command: "git push {{ clojure_app_repo_dir }} master:master" + become: yes + become_user: "{{ clojure_app_user }}" + args: + chdir: "/tmp/clojurians-log-tmp" + creates: "{{ clojure_app_repo_dir }}/refs/heads/master" + - name: Check out existing logs become: yes become_user: "{{ clojure_app_user }}" @@ -47,3 +62,8 @@ accept_hostkey: yes key_file: "{{ clojure_app_home_dir }}/.ssh/id_rsa" tags: git-checkout + +- name: Import logs + command: "nc -N localhost {{ clojure_socket_repl_port }} < {{ clojure_app_app_dir }}/repl/production_import.clj > {{ clojure_app_app_dir }}/initial_import.txt" + args: + creates: "{{ clojure_app_app_dir }}/initial_import.txt" diff --git a/clojurians_log/playbooks/roles/plexus.clojurians-log/templates/config.edn b/clojurians_log/playbooks/roles/plexus.clojurians-log/templates/config.edn index b4d6664..9836b6c 100644 --- a/clojurians_log/playbooks/roles/plexus.clojurians-log/templates/config.edn +++ b/clojurians_log/playbooks/roles/plexus.clojurians-log/templates/config.edn @@ -1,6 +1,6 @@ {:datomic {:uri "datomic:sql://{{ clojure_app_name }}?jdbc:postgresql://localhost:5432/{{ database_name }}?user={{ database_user }}&password={{ database_password }}"} :http {:port {{ clojurians_app_http_port }} - :origin "https://clojurians-log.clojureverse.org"} + :origin "https://{{ clojurians_app_fqdn }}"} :slack {:api-token "{{ slack_api_token }}" :log-dir "{{ clojure_app_home_dir }}/logs"} :cache-time 2678400 ;; ~one month diff --git a/clojurians_log/playbooks/roles/plexus.rtmbot/tasks/main.yml b/clojurians_log/playbooks/roles/plexus.rtmbot/tasks/main.yml index 8b169de..89ac163 100644 --- a/clojurians_log/playbooks/roles/plexus.rtmbot/tasks/main.yml +++ b/clojurians_log/playbooks/roles/plexus.rtmbot/tasks/main.yml @@ -1,38 +1,13 @@ --- -- name: Install rtmbot dependency - pip: - name: 'slackclient' - tags: rtmbot - -- name: Create rtmbot directory - file: - path: "{{ clojure_app_home_dir }}/rtmbot" - state: directory - owner: "{{ clojure_app_user }}" - group: "{{ clojure_app_user }}" - tags: rtmbot - -- name: Create rtmbot logs directory - file: - path: "{{ clojure_app_home_dir }}/rtmbot/logs" - state: directory - owner: "{{ clojure_app_user }}" - group: "{{ clojure_app_user }}" - tags: rtmbot - -- name: Install run-rtmbot.sh - template: - src: templates/run-rtmbot.sh.j2 - dest: "{{ clojure_app_home_dir }}/rtmbot/run-rtmbot.sh" - owner: "{{ clojure_app_user }}" - group: "{{ clojure_app_user }}" - mode: 0755 - tags: rtmbot +- name: Install rtmbot + git: + repo: "https://github.com/clojureverse/rtmbot.git" + dest: "/var/rtmbot" - name: Install rtmbot.conf template: src: templates/rtmbot.conf.j2 - dest: "{{ clojure_app_home_dir }}/rtmbot/rtmbot.conf" + dest: "/var/rtmbot/rtmbot.conf" owner: "{{ clojure_app_user }}" group: "{{ clojure_app_user }}" mode: 0644 diff --git a/clojurians_log/playbooks/roles/plexus.rtmbot/templates/rtmbot.conf.j2 b/clojurians_log/playbooks/roles/plexus.rtmbot/templates/rtmbot.conf.j2 index 977fda7..15d33e6 100644 --- a/clojurians_log/playbooks/roles/plexus.rtmbot/templates/rtmbot.conf.j2 +++ b/clojurians_log/playbooks/roles/plexus.rtmbot/templates/rtmbot.conf.j2 @@ -1,2 +1,3 @@ SLACK_TOKEN: "{{ slack_api_token }}" -LOGFILE: "{{ clojure_app_home_dir }}/rtmbot/rtmbot.log" +LOGFILE: "{{ clojure_app_home_dir }}/rtmbot.log" +DESTINATION: "{{ clojure_app_home_dir }}/logs/" \ No newline at end of file diff --git a/clojurians_log/playbooks/roles/plexus.rtmbot/templates/rtmbot.service.j2 b/clojurians_log/playbooks/roles/plexus.rtmbot/templates/rtmbot.service.j2 index f474847..1358fe9 100644 --- a/clojurians_log/playbooks/roles/plexus.rtmbot/templates/rtmbot.service.j2 +++ b/clojurians_log/playbooks/roles/plexus.rtmbot/templates/rtmbot.service.j2 @@ -3,8 +3,8 @@ Description=Slack RTM logging bot After=network.target [Service] -WorkingDirectory={{ clojure_app_home_dir }}/rtmbot -ExecStart={{ clojure_app_home_dir}}/rtmbot/run-rtmbot.sh +WorkingDirectory=/var/rtmbot +ExecStart=/var/rtmbot/rtmbot.sh User={{ clojure_app_user }} Restart=always RestartSec=5 diff --git a/clojurians_log/playbooks/vars/clojurians_log_secrets.yml.gpg b/clojurians_log/playbooks/vars/clojurians_log_secrets.yml.gpg index 7f2f64c..973da12 100644 Binary files a/clojurians_log/playbooks/vars/clojurians_log_secrets.yml.gpg and b/clojurians_log/playbooks/vars/clojurians_log_secrets.yml.gpg differ diff --git a/clojurians_log/security_groups.tf b/clojurians_log/security_groups.tf index 2fbf114..9762ce1 100644 --- a/clojurians_log/security_groups.tf +++ b/clojurians_log/security_groups.tf @@ -1,5 +1,5 @@ resource "exoscale_security_group" "clojurians_log" { - name = "clojurians_log" + name = "${local.instance_name}" description = "Security Group for Clojurians Log" } diff --git a/clojurians_log/userdata.sh.tmpl b/clojurians_log/userdata.sh.tmpl index ae74f9e..894597d 100755 --- a/clojurians_log/userdata.sh.tmpl +++ b/clojurians_log/userdata.sh.tmpl @@ -14,6 +14,7 @@ sudo DEBIAN_FRONTEND=noninteractive apt install -y ansible cd ~ +# TODO: copy from the local filesystem, instead of going over github # Clone repo and switch to it git clone ${git_clone_params} https://github.com/clojureverse/nebula cd nebula/clojurians_log @@ -33,5 +34,10 @@ base64 --decode < vars/clojurians_log_secrets.yml.b64 > vars/clojurians_log_secr # Bootstrap the instance ansible-playbook bootstrap.yml -i hosts >> ~/ansible.log +# Create a script for ansible, so it's easy to re-run at a later date +echo '#/bin/sh' >> ~/run_ansible +echo 'HOME=/root ansible-playbook clojurians-log.yml -i hosts ${ansible_playbook_params} "$@"' >> ~/run_ansible +chmod +x ~/run_ansible + # Setup clojurians_log -HOME=/root ansible-playbook clojurians-log.yml -i hosts ${ansible_playbook_params} >> ~/ansible.log +~/run_ansible >> ~/ansible.log diff --git a/clojurians_log/vars.tf b/clojurians_log/vars.tf index 1e33cfd..d0215e0 100644 --- a/clojurians_log/vars.tf +++ b/clojurians_log/vars.tf @@ -15,3 +15,15 @@ variable "exoscale_ssh_keypair_name" { type = string description = "The SSH keypair to be allowed on the instance" } + +variable "cloudflare_email" { + default = "" + type = string + description = "The email to authenticate with Cloudflare" +} + +variable "cloudflare_api_key" { + default = "" + type = string + description = "The email to authenticate with Cloudflare" +}