In [1]:
import yaml
with open('ansible-playbook-install-td-agent2.yml') as f:
y = yaml.load(f)
y
Out[1]:
[{'hosts': '192.168.33.10',
'sudo': True,
'tasks': [{'apt': 'name=curl state=present update_cache=yes',
'name': 'install curl'},
{'name': 'install td-agent2',
'shell': 'curl -L http://toolbelt.treasuredata.com/sh/install-ubuntu-trusty-td-agent2.sh | sh'},
{'name': 'start td-agent', 'service': 'name=td-agent state=started'}],
'user': 'vagrant'}]
In [2]:
!ansible-playbook -v -i inventory ansible-playbook-install-td-agent2.yml
PLAY [192.168.33.10] **********************************************************
GATHERING FACTS ***************************************************************
ok: [192.168.33.10]
TASK: [install curl] **********************************************************
ok: [192.168.33.10] => {"changed": false}
TASK: [install td-agent2] *****************************************************
changed: [192.168.33.10] => {"changed": true, "cmd": "curl -L http://toolbelt.treasuredata.com/sh/install-ubuntu-trusty-td-agent2.sh | sh", "delta": "0:01:49.939264", "end": "2015-04-01 10:31:25.709139", "rc": 0, "start": "2015-04-01 10:29:35.769875", "stderr": " % Total % Received % Xferd Average Speed Time Time Time Current\n Dload Upload Total Spent Left Speed\n\r 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0\r 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0\r 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0\r 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0\r 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0\r 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0\r100 575 100 575 0 0 97 0 0:00:05 0:00:05 --:--:-- 131\n % Total % Received % Xferd Average Speed Time Time Time Current\n Dload Upload Total Spent Left Speed\n\r 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0\r 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0\r 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0\r 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0\r 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0\r 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0\r 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0\r100 934 100 934 0 0 141 0 0:00:06 0:00:06 --:--:-- 228", "stdout": "This script requires superuser access to install apt packages.\nYou will be prompted for your password by sudo.\nOK\nGet:1 http://packages.treasuredata.com trusty InRelease [1,204 B]\nGet:2 http://packages.treasuredata.com trusty/contrib amd64 Packages [386 B]\nIgn http://packages.treasuredata.com trusty/contrib Translation-en_US\nIgn http://packages.treasuredata.com trusty/contrib Translation-en\nIgn http://archive.ubuntu.com trusty InRelease\nIgn http://security.ubuntu.com trusty-security InRelease\nIgn http://archive.ubuntu.com trusty-updates InRelease\nGet:3 http://security.ubuntu.com trusty-security Release.gpg [933 B]\nIgn http://archive.ubuntu.com trusty-backports InRelease\nHit http://archive.ubuntu.com trusty Release.gpg\nGet:4 http://security.ubuntu.com trusty-security Release [63.5 kB]\nHit http://archive.ubuntu.com trusty-updates Release.gpg\nHit http://archive.ubuntu.com trusty-backports Release.gpg\nGet:5 https://get.docker.com docker InRelease\nHit http://archive.ubuntu.com trusty Release\nHit http://archive.ubuntu.com trusty-updates Release\nHit http://archive.ubuntu.com trusty-backports Release\nGet:6 http://security.ubuntu.com trusty-security/main Sources [76.1 kB]\nHit http://archive.ubuntu.com trusty/main Sources\nIgn https://get.docker.com docker InRelease\nHit http://archive.ubuntu.com trusty/restricted Sources\nHit http://archive.ubuntu.com trusty/universe Sources\nHit http://archive.ubuntu.com trusty/multiverse Sources\nGet:7 http://security.ubuntu.com trusty-security/restricted Sources [2,061 B]\nHit http://archive.ubuntu.com trusty/main amd64 Packages\nHit https://get.docker.com docker Release.gpg\nHit http://archive.ubuntu.com trusty/restricted amd64 Packages\nGet:8 http://security.ubuntu.com trusty-security/universe Sources [18.0 kB]\nHit http://archive.ubuntu.com trusty/universe amd64 Packages\nHit http://archive.ubuntu.com trusty/multiverse amd64 Packages\nHit http://archive.ubuntu.com trusty/main i386 Packages\nGet:9 http://security.ubuntu.com trusty-security/multiverse Sources [1,905 B]\nHit http://archive.ubuntu.com trusty/restricted i386 Packages\nHit https://get.docker.com docker Release\nHit http://archive.ubuntu.com trusty/universe i386 Packages\nGet:10 http://security.ubuntu.com trusty-security/main amd64 Packages [251 kB]\nHit http://archive.ubuntu.com trusty/multiverse i386 Packages\nHit http://archive.ubuntu.com trusty/main Translation-en\nHit https://get.docker.com docker/main amd64 Packages\nHit http://archive.ubuntu.com trusty/multiverse Translation-en\nHit http://archive.ubuntu.com trusty/restricted Translation-en\nHit https://get.docker.com docker/main i386 Packages\nHit http://archive.ubuntu.com trusty/universe Translation-en\nHit http://archive.ubuntu.com trusty-updates/main Sources\nHit http://archive.ubuntu.com trusty-updates/restricted Sources\nGet:11 https://get.docker.com docker/main Translation-en_US\nHit http://archive.ubuntu.com trusty-updates/universe Sources\nHit http://archive.ubuntu.com trusty-updates/multiverse Sources\nGet:12 http://security.ubuntu.com trusty-security/restricted amd64 Packages [8,875 B]\nHit http://archive.ubuntu.com trusty-updates/main amd64 Packages\nHit http://archive.ubuntu.com trusty-updates/restricted amd64 Packages\nGet:13 http://security.ubuntu.com trusty-security/universe amd64 Packages [89.7 kB]\nHit http://archive.ubuntu.com trusty-updates/universe amd64 Packages\nHit http://archive.ubuntu.com trusty-updates/multiverse amd64 Packages\nGet:14 http://security.ubuntu.com trusty-security/multiverse amd64 Packages [3,459 B]\nHit http://archive.ubuntu.com trusty-updates/main i386 Packages\nHit http://archive.ubuntu.com trusty-updates/restricted i386 Packages\nGet:15 http://security.ubuntu.com trusty-security/main i386 Packages [242 kB]\nHit http://archive.ubuntu.com trusty-updates/universe i386 Packages\nHit http://archive.ubuntu.com trusty-updates/multiverse i386 Packages\nHit http://archive.ubuntu.com trusty-updates/main Translation-en\nHit http://archive.ubuntu.com trusty-updates/multiverse Translation-en\nHit http://archive.ubuntu.com trusty-updates/restricted Translation-en\nHit http://archive.ubuntu.com trusty-updates/universe Translation-en\nHit http://archive.ubuntu.com trusty-backports/main Sources\nHit http://archive.ubuntu.com trusty-backports/restricted Sources\nHit http://archive.ubuntu.com trusty-backports/universe Sources\nHit http://archive.ubuntu.com trusty-backports/multiverse Sources\nHit http://archive.ubuntu.com trusty-backports/main amd64 Packages\nGet:16 http://security.ubuntu.com trusty-security/restricted i386 Packages [8,846 B]\nHit http://archive.ubuntu.com trusty-backports/restricted amd64 Packages\nGet:17 http://security.ubuntu.com trusty-security/universe i386 Packages [89.7 kB]\nHit http://archive.ubuntu.com trusty-backports/universe amd64 Packages\nHit http://archive.ubuntu.com trusty-backports/multiverse amd64 Packages\nHit http://archive.ubuntu.com trusty-backports/main i386 Packages\nGet:18 http://security.ubuntu.com trusty-security/multiverse i386 Packages [3,628 B]\nHit http://archive.ubuntu.com trusty-backports/restricted i386 Packages\nHit http://archive.ubuntu.com trusty-backports/universe i386 Packages\nHit http://security.ubuntu.com trusty-security/main Translation-en\nHit http://archive.ubuntu.com trusty-backports/multiverse i386 Packages\nHit http://security.ubuntu.com trusty-security/multiverse Translation-en\nHit http://archive.ubuntu.com trusty-backports/main Translation-en\nHit http://security.ubuntu.com trusty-security/restricted Translation-en\nHit http://archive.ubuntu.com trusty-backports/multiverse Translation-en\nHit http://archive.ubuntu.com trusty-backports/restricted Translation-en\nHit http://security.ubuntu.com trusty-security/universe Translation-en\nHit http://archive.ubuntu.com trusty-backports/universe Translation-en\nIgn http://archive.ubuntu.com trusty/main Translation-en_US\nIgn http://archive.ubuntu.com trusty/multiverse Translation-en_US\nIgn http://archive.ubuntu.com trusty/restricted Translation-en_US\nIgn http://archive.ubuntu.com trusty/universe Translation-en_US\nIgn https://get.docker.com docker/main Translation-en_US\nIgn https://get.docker.com docker/main Translation-en\nFetched 861 kB in 33s (25.6 kB/s)\nReading package lists...\nReading package lists...\nBuilding dependency tree...\nReading state information...\nThe following NEW packages will be installed:\n td-agent\n0 upgraded, 1 newly installed, 0 to remove and 151 not upgraded.\nNeed to get 60.6 MB of archives.\nAfter this operation, 210 MB of additional disk space will be used.\nGet:1 http://packages.treasuredata.com/2/ubuntu/trusty/ trusty/contrib td-agent amd64 2.1.5-1 [60.6 MB]\nFetched 60.6 MB in 54s (1,112 kB/s)\nSelecting previously unselected package td-agent.\n(Reading database ... 60669 files and directories currently installed.)\nPreparing to unpack .../td-agent_2.1.5-1_amd64.deb ...\nUnpacking td-agent (2.1.5-1) ...\nSetting up td-agent (2.1.5-1) ...\nAdding system user `td-agent' (UID 105) ...\nAdding new group `td-agent' (GID 112) ...\nAdding new user `td-agent' (UID 105) with group `td-agent' ...\nNot creating home directory `/home/td-agent'.\nInstalling default conffile /etc/td-agent/td-agent.conf ...\n * Starting td-agent td-agent\n ...done.\nProcessing triggers for libc-bin (2.19-0ubuntu6) ...", "warnings": ["Consider using get_url module rather than running curl"]}
TASK: [start td-agent] ********************************************************
ok: [192.168.33.10] => {"changed": false, "name": "td-agent", "state": "started"}
PLAY RECAP ********************************************************************
192.168.33.10 : ok=4 changed=1 unreachable=0 failed=0
In [3]:
!ansible -i inventory 192.168.33.10 --sudo -a 'service td-agent status'
192.168.33.10 | success | rc=0 >>
* ruby is running
In [4]:
import yaml
with open('ansible-playbook-install-td-agent-plugin.yml') as f:
y = yaml.load(f)
y
Out[4]:
[{'hosts': '192.168.33.10',
'sudo': True,
'tasks': [{'name': 'install plugin',
'shell': '/usr/sbin/td-agent-gem install fluent-plugin-docker-metrics'},
{'copy': 'src=td-agent.conf dest=/etc/td-agent/td-agent.conf force=yes',
'name': 'copy td-agent.conf',
'notify': ['restart td-agent']},
{'name': 'td-agent user add to docker',
'shell': 'gpasswd -a td-agent docker'},
{'name': 'restart td-agent', 'service': 'name=td-agent state=restarted'}],
'user': 'vagrant'}]
In [6]:
!ansible-playbook -v -i inventory ansible-playbook-install-td-agent-plugin.yml
PLAY [192.168.33.10] **********************************************************
GATHERING FACTS ***************************************************************
ok: [192.168.33.10]
TASK: [install plugin] ********************************************************
changed: [192.168.33.10] => {"changed": true, "cmd": "/usr/sbin/td-agent-gem install fluent-plugin-docker-metrics", "delta": "0:00:27.067095", "end": "2015-04-01 10:33:22.432661", "rc": 0, "start": "2015-04-01 10:32:55.365566", "stderr": "", "stdout": "Successfully installed fluent-plugin-docker-metrics-0.0.2\nParsing documentation for fluent-plugin-docker-metrics-0.0.2\nDone installing documentation for fluent-plugin-docker-metrics after 0 seconds\n1 gem installed", "warnings": []}
TASK: [copy td-agent.conf] ****************************************************
ok: [192.168.33.10] => {"changed": false, "checksum": "286ef72deb93df352a506ea04484b2b95ff28b2c", "dest": "/etc/td-agent/td-agent.conf", "gid": 0, "group": "root", "mode": "0644", "owner": "root", "path": "/etc/td-agent/td-agent.conf", "size": 296, "state": "file", "uid": 0}
TASK: [td-agent user add to docker] *******************************************
changed: [192.168.33.10] => {"changed": true, "cmd": "gpasswd -a td-agent docker", "delta": "0:00:00.009066", "end": "2015-04-01 10:33:22.685368", "rc": 0, "start": "2015-04-01 10:33:22.676302", "stderr": "", "stdout": "Adding user td-agent to group docker", "warnings": []}
TASK: [restart td-agent] ******************************************************
changed: [192.168.33.10] => {"changed": true, "name": "td-agent", "state": "started"}
PLAY RECAP ********************************************************************
192.168.33.10 : ok=5 changed=3 unreachable=0 failed=0
In [7]:
!ansible -i inventory 192.168.33.10 --sudo -a 'cat /etc/td-agent/td-agent.conf'
192.168.33.10 | success | rc=0 >>
<source>
type docker_metrics
# stats_interval 1m
stats_interval 10s
# cgroup_path /sys/fs/cgroup
# tag_prefix docker
# docker_socket unix:///var/run/docker.sock
# docker_network_path /sys/class/net
# docker_infos_path /var/lib/docker/execdriver/native
</source>
<match **>
type stdout
</match>
In [8]:
!ansible -i inventory 192.168.33.10 --sudo -a 'service td-agent status'
192.168.33.10 | success | rc=0 >>
* ruby is running
In [10]:
!ansible -i inventory 192.168.33.10 --sudo -a 'tail -n 20 /var/log/td-agent/td-agent.log'
192.168.33.10 | success | rc=0 >>
2015-04-01 10:40:03 +0000 docker.memory.stat: {"key":"memory_stat_inactive_file","value":13852672,"type":"gauge","hostname":"vagrant-ubuntu-trusty","id":"e869612fda1b8195cc3b846b15132cb3b8221225add712d6b25c8741d4716618","name":"/cocky_kirch"}
2015-04-01 10:40:03 +0000 docker.memory.stat: {"key":"memory_stat_active_file","value":12288,"type":"gauge","hostname":"vagrant-ubuntu-trusty","id":"e869612fda1b8195cc3b846b15132cb3b8221225add712d6b25c8741d4716618","name":"/cocky_kirch"}
2015-04-01 10:40:03 +0000 docker.memory.stat: {"key":"memory_stat_unevictable","value":0,"type":"gauge","hostname":"vagrant-ubuntu-trusty","id":"e869612fda1b8195cc3b846b15132cb3b8221225add712d6b25c8741d4716618","name":"/cocky_kirch"}
2015-04-01 10:40:03 +0000 docker.memory.stat: {"key":"memory_stat_hierarchical_memory_limit","value":18446744073709551615,"type":"gauge","hostname":"vagrant-ubuntu-trusty","id":"e869612fda1b8195cc3b846b15132cb3b8221225add712d6b25c8741d4716618","name":"/cocky_kirch"}
2015-04-01 10:40:03 +0000 docker.memory.stat: {"key":"memory_stat_total_cache","value":13885440,"type":"gauge","hostname":"vagrant-ubuntu-trusty","id":"e869612fda1b8195cc3b846b15132cb3b8221225add712d6b25c8741d4716618","name":"/cocky_kirch"}
2015-04-01 10:40:03 +0000 docker.memory.stat: {"key":"memory_stat_total_rss","value":102400,"type":"gauge","hostname":"vagrant-ubuntu-trusty","id":"e869612fda1b8195cc3b846b15132cb3b8221225add712d6b25c8741d4716618","name":"/cocky_kirch"}
2015-04-01 10:40:03 +0000 docker.memory.stat: {"key":"memory_stat_total_rss_huge","value":0,"type":"gauge","hostname":"vagrant-ubuntu-trusty","id":"e869612fda1b8195cc3b846b15132cb3b8221225add712d6b25c8741d4716618","name":"/cocky_kirch"}
2015-04-01 10:40:03 +0000 docker.memory.stat: {"key":"memory_stat_total_mapped_file","value":471040,"type":"gauge","hostname":"vagrant-ubuntu-trusty","id":"e869612fda1b8195cc3b846b15132cb3b8221225add712d6b25c8741d4716618","name":"/cocky_kirch"}
2015-04-01 10:40:03 +0000 docker.memory.stat: {"key":"memory_stat_total_writeback","value":0,"type":"gauge","hostname":"vagrant-ubuntu-trusty","id":"e869612fda1b8195cc3b846b15132cb3b8221225add712d6b25c8741d4716618","name":"/cocky_kirch"}
2015-04-01 10:40:03 +0000 docker.memory.stat: {"key":"memory_stat_total_pgpgin","value":3773,"type":"gauge","hostname":"vagrant-ubuntu-trusty","id":"e869612fda1b8195cc3b846b15132cb3b8221225add712d6b25c8741d4716618","name":"/cocky_kirch"}
2015-04-01 10:40:03 +0000 docker.memory.stat: {"key":"memory_stat_total_pgpgout","value":358,"type":"gauge","hostname":"vagrant-ubuntu-trusty","id":"e869612fda1b8195cc3b846b15132cb3b8221225add712d6b25c8741d4716618","name":"/cocky_kirch"}
2015-04-01 10:40:03 +0000 docker.memory.stat: {"key":"memory_stat_total_pgfault","value":1787,"type":"gauge","hostname":"vagrant-ubuntu-trusty","id":"e869612fda1b8195cc3b846b15132cb3b8221225add712d6b25c8741d4716618","name":"/cocky_kirch"}
2015-04-01 10:40:03 +0000 docker.memory.stat: {"key":"memory_stat_total_pgmajfault","value":128,"type":"gauge","hostname":"vagrant-ubuntu-trusty","id":"e869612fda1b8195cc3b846b15132cb3b8221225add712d6b25c8741d4716618","name":"/cocky_kirch"}
2015-04-01 10:40:03 +0000 docker.memory.stat: {"key":"memory_stat_total_inactive_anon","value":4096,"type":"gauge","hostname":"vagrant-ubuntu-trusty","id":"e869612fda1b8195cc3b846b15132cb3b8221225add712d6b25c8741d4716618","name":"/cocky_kirch"}
2015-04-01 10:40:03 +0000 docker.memory.stat: {"key":"memory_stat_total_active_anon","value":118784,"type":"gauge","hostname":"vagrant-ubuntu-trusty","id":"e869612fda1b8195cc3b846b15132cb3b8221225add712d6b25c8741d4716618","name":"/cocky_kirch"}
2015-04-01 10:40:03 +0000 docker.memory.stat: {"key":"memory_stat_total_inactive_file","value":13852672,"type":"gauge","hostname":"vagrant-ubuntu-trusty","id":"e869612fda1b8195cc3b846b15132cb3b8221225add712d6b25c8741d4716618","name":"/cocky_kirch"}
2015-04-01 10:40:03 +0000 docker.memory.stat: {"key":"memory_stat_total_active_file","value":12288,"type":"gauge","hostname":"vagrant-ubuntu-trusty","id":"e869612fda1b8195cc3b846b15132cb3b8221225add712d6b25c8741d4716618","name":"/cocky_kirch"}
2015-04-01 10:40:03 +0000 docker.memory.stat: {"key":"memory_stat_total_unevictable","value":0,"type":"gauge","hostname":"vagrant-ubuntu-trusty","id":"e869612fda1b8195cc3b846b15132cb3b8221225add712d6b25c8741d4716618","name":"/cocky_kirch"}
2015-04-01 10:40:03 +0000 docker.cpuacct.stat: {"key":"cpuacct_stat_user","value":0,"type":"counter","hostname":"vagrant-ubuntu-trusty","id":"e869612fda1b8195cc3b846b15132cb3b8221225add712d6b25c8741d4716618","name":"/cocky_kirch"}
2015-04-01 10:40:03 +0000 docker.cpuacct.stat: {"key":"cpuacct_stat_system","value":1,"type":"counter","hostname":"vagrant-ubuntu-trusty","id":"e869612fda1b8195cc3b846b15132cb3b8221225add712d6b25c8741d4716618","name":"/cocky_kirch"}
In [34]:
!ansible -i inventory 192.168.33.10 --sudo -a 'tail -n 20 /var/log/td-agent/td-agent.log'
192.168.33.10 | success | rc=0 >>
2015-03-31 09:17:33 +0000 [info]: gem 'fluent-plugin-docker-metrics' version '0.0.2'
2015-03-31 09:17:33 +0000 [info]: gem 'fluent-plugin-mongo' version '0.7.6'
2015-03-31 09:17:33 +0000 [info]: gem 'fluent-plugin-rewrite-tag-filter' version '1.4.1'
2015-03-31 09:17:33 +0000 [info]: gem 'fluent-plugin-s3' version '0.5.6'
2015-03-31 09:17:33 +0000 [info]: gem 'fluent-plugin-scribe' version '0.10.14'
2015-03-31 09:17:33 +0000 [info]: gem 'fluent-plugin-td' version '0.10.26'
2015-03-31 09:17:33 +0000 [info]: gem 'fluent-plugin-td-monitoring' version '0.2.0'
2015-03-31 09:17:33 +0000 [info]: gem 'fluent-plugin-webhdfs' version '0.4.1'
2015-03-31 09:17:33 +0000 [info]: gem 'fluentd' version '0.10.61'
2015-03-31 09:17:33 +0000 [info]: using configuration file: <ROOT>
<source>
type docker_metrics
stats_interval 10s
</source>
<match **>
type stdout
</match>
</ROOT>
2015-03-31 09:17:33 +0000 [info]: adding source type="docker_metrics"
2015-03-31 09:17:33 +0000 [info]: adding match pattern="**" type="stdout"
In [33]:
!ansible -i inventory 192.168.33.10 --sudo -a 'ls -la /var/log/td-agent/'
192.168.33.10 | success | rc=0 >>
total 220
drwxr-xr-x 3 td-agent td-agent 4096 Mar 31 08:36 .
drwxrwxr-x 8 root syslog 4096 Mar 31 08:36 ..
drwxr-xr-x 2 td-agent td-agent 4096 Mar 31 08:36 buffer
-rw-r--r-- 1 td-agent td-agent 212543 Mar 31 09:17 td-agent.log
In [ ]:
Content source: doragon/presentation
Similar notebooks: