Merged in DCD-436-cloudwatch-integration (pull request #23)

DCD-436 CloudWatch agent integration

Approved-by: Ben Partridge <bpartridge@atlassian.com>
Approved-by: Adam Brokes <abrokes@atlassian.com>
Approved-by: Varun Arbatti <varbatti@atlassian.com>

bitbucket-pipelines.yml

@@ -14,7 +14,7 @@ pipelines:
     - step:
         name: Pre Parallelization stage
         script:
-          - echo "Running tests in 26 batches"
+          - echo "Running tests in 28 batches"
     - step:
         name: Check if number of batches match actual number of scenarios
         script:
@@ -261,4 +261,22 @@ pipelines:
            - ./bin/install-ansible
            - ./bin/run-tests-in-batches --batch 26
            
+       - step:
+           name: Molecule Test Batch - 27
+           services:
+             - docker
+           script:
+           - apt-get update && apt-get install -y virtualenv python-dev
+           - ./bin/install-ansible
+           - ./bin/run-tests-in-batches --batch 27
+           
+       - step:
+           name: Molecule Test Batch - 28
+           services:
+             - docker
+           script:
+           - apt-get update && apt-get install -y virtualenv python-dev
+           - ./bin/install-ansible
+           - ./bin/run-tests-in-batches --batch 28
+           
        

group_vars/aws_node_local.yml

@@ -34,6 +34,16 @@ atl_product_installation_versioned: "{{ atl_product_installation_base }}/{{ atl_
 atl_product_installation_current: "{{ atl_product_installation_base }}/current"
 atl_installer_temp: "{{ atl_installation_base }}/tmp"
 
+atl_product_log_locations:
+  confluence:
+    - "{{ atl_product_installation_current }}/logs"
+    - "{{ atl_product_home }}/logs"
+  jira:
+    - "{{ atl_product_installation_current }}/logs"
+    - "{{ atl_product_home }}/log"
+  stash:
+    - "{{ atl_product_home }}/log"
+  crowd: []
 
 #  The following are imports from the environment. These are generally
 #  set in /etc/atl by the CloudFormation template and sourced before
@@ -46,6 +56,9 @@ atl_aws_region: "{{ lookup('env', 'ATL_AWS_REGION') }}"
 atl_aws_iam_role: "{{ lookup('env', 'ATL_AWS_IAM_ROLE') }}"
 atl_aws_iam_role_arn: "{{ lookup('env', 'ATL_AWS_IAM_ROLE_ARN') }}"
 
+atl_aws_enable_cloudwatch: "{{ lookup('env', 'ATL_AWS_ENABLE_CLOUDWATCH')|bool or false }}"
+atl_aws_enable_cloudwatch_logs: "{{ lookup('env', 'ATL_AWS_ENABLE_CLOUDWATCH_LOGS')|bool or false }}"
+
 atl_db_engine: "{{ lookup('env', 'ATL_DB_ENGINE') }}"
 atl_db_host: "{{ lookup('env', 'ATL_DB_HOST') }}"
 atl_db_port: "{{ lookup('env', 'ATL_DB_PORT') or '5432' }}"

roles/aws_common/defaults/main.yml

@@ -0,0 +1,13 @@
+---
+
+# See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/download-cloudwatch-agent-commandline.html
+aws_download_region: "{{ ansible_ec2_placement_region | default('us-west-2') }}"
+aws_cloudwatch_agent_rpm: "https://s3.{{ aws_download_region }}.amazonaws.com/amazoncloudwatch-agent-{{ aws_download_region }}/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm"
+
+atl_aws_enable_cloudwatch: false
+atl_aws_enable_cloudwatch_logs: false
+
+# Mostly for molecule testing, as skip-tags doesn't work with handlers.
+atl_aws_agent_restart: true
+
+atl_aws_log_group: "{{ atl_product_edition }}-{{ atl_aws_stack_name }}"

roles/aws_common/handlers/main.yml

@@ -0,0 +1,15 @@
+---
+
+- name: Enable CloudWatch Agent
+  systemd:
+    name: "amazon-cloudwatch-agent.service"
+    daemon_reload: true
+    enabled: true
+  when: atl_aws_agent_restart
+
+- name: Restart CloudWatch Agent
+  systemd:
+    name: "amazon-cloudwatch-agent.service"
+    enabled: true
+    state: restarted
+  when: atl_aws_agent_restart

roles/aws_common/molecule/cw-disabled/Dockerfile.j2

@@ -0,0 +1,14 @@
+# Molecule managed
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi

roles/aws_common/molecule/cw-disabled/molecule.yml

@@ -0,0 +1,26 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+lint:
+  name: yamllint
+platforms:
+  - name: amazon_linux2
+    image: amazonlinux:2
+    groups:
+      - aws_node_local
+# - name: ubuntu_lts
+#   image: ubuntu:bionic
+provisioner:
+  name: ansible
+  lint:
+    name: ansible-lint
+  inventory:
+    links:
+      group_vars: ../../../../group_vars/
+verifier:
+  name: testinfra
+  lint:
+    name: flake8
+    enabled: false

roles/aws_common/molecule/cw-disabled/playbook.yml

@@ -0,0 +1,17 @@
+---
+- name: Converge
+  hosts: all
+  vars:
+    ansible_ec2_local_ipv4: "1.1.1.1"
+    ansible_default_ipv4:
+      address: "9.9.9.9"
+    ansible_ec2_instance_id: "NONE"
+
+    atl_product_family: "jira"
+    atl_product_edition: "jira-software"
+    atl_aws_stack_name: "MY_STACK"
+
+    atl_aws_enable_cloudwatch: "{{ 'false'|bool }}"
+
+  roles:
+    - role: aws_common

roles/aws_common/molecule/cw-disabled/tests/test_default.py

@@ -0,0 +1,23 @@
+import os
+import pytest
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+@pytest.mark.parametrize('exe', [
+    '/usr/bin/ec2-metadata',
+    '/usr/bin/amazon-ssm-agent',
+    '/sbin/mount.efs'
+])
+def test_package_exes(host, exe):
+    assert host.file(exe).exists
+
+@pytest.mark.parametrize('path', [
+    '/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent',
+    '/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json'
+])
+def test_package_not_installed(host, path):
+    assert not host.file(path).exists

roles/aws_common/molecule/default/molecule.yml

@@ -8,13 +8,19 @@ lint:
 platforms:
   - name: amazon_linux2
     image: amazonlinux:2
+    groups:
+      - aws_node_local
 # - name: ubuntu_lts
 #   image: ubuntu:bionic
 provisioner:
   name: ansible
   lint:
     name: ansible-lint
+  inventory:
+    links:
+      group_vars: ../../../../group_vars/
 verifier:
   name: testinfra
   lint:
     name: flake8
+    enabled: false

roles/aws_common/molecule/default/playbook.yml

@@ -6,5 +6,15 @@
     ansible_default_ipv4:
       address: "9.9.9.9"
     ansible_ec2_instance_id: "NONE"
+
+    atl_product_family: "jira"
+    atl_product_edition: "jira-software"
+    atl_aws_stack_name: "MY_STACK"
+
+    # The `bool` pipe is a sanity check for group file.
+    atl_aws_enable_cloudwatch: "{{ 'true'|bool }}"
+    atl_aws_enable_cloudwatch_logs: true
+    atl_aws_agent_restart: false
+
   roles:
     - role: aws_common

roles/aws_common/molecule/default/tests/test_default.py

@@ -10,7 +10,16 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
 @pytest.mark.parametrize('exe', [
     '/usr/bin/ec2-metadata',
     '/usr/bin/amazon-ssm-agent',
-    '/sbin/mount.efs'
+    '/sbin/mount.efs',
+    '/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent'
 ])
 def test_package_exes(host, exe):
     assert host.file(exe).exists
+
+
+def test_service_file(host):
+    f = host.file('/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json')
+    assert f.contains('"log_group_name": "jira-software-MY_STACK"')
+    assert f.user == 'root'
+    assert f.group == 'root'
+    assert f.mode == 0o0644

roles/aws_common/molecule/logs-disabled/Dockerfile.j2

@@ -0,0 +1,14 @@
+# Molecule managed
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi

roles/aws_common/molecule/logs-disabled/molecule.yml

@@ -0,0 +1,26 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+lint:
+  name: yamllint
+platforms:
+  - name: amazon_linux2
+    image: amazonlinux:2
+    groups:
+      - aws_node_local
+# - name: ubuntu_lts
+#   image: ubuntu:bionic
+provisioner:
+  name: ansible
+  lint:
+    name: ansible-lint
+  inventory:
+    links:
+      group_vars: ../../../../group_vars/
+verifier:
+  name: testinfra
+  lint:
+    name: flake8
+    enabled: false

roles/aws_common/molecule/logs-disabled/playbook.yml

@@ -0,0 +1,20 @@
+---
+- name: Converge
+  hosts: all
+  vars:
+    ansible_ec2_local_ipv4: "1.1.1.1"
+    ansible_default_ipv4:
+      address: "9.9.9.9"
+    ansible_ec2_instance_id: "NONE"
+
+    atl_product_family: "jira"
+    atl_product_edition: "jira-software"
+    atl_aws_stack_name: "MY_STACK"
+
+    atl_aws_enable_cloudwatch: true
+    atl_aws_enable_cloudwatch_logs: false
+
+    atl_aws_agent_restart: false
+
+  roles:
+    - role: aws_common

roles/aws_common/molecule/logs-disabled/tests/test_default.py

@@ -0,0 +1,25 @@
+import os
+import pytest
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+@pytest.mark.parametrize('exe', [
+    '/usr/bin/ec2-metadata',
+    '/usr/bin/amazon-ssm-agent',
+    '/sbin/mount.efs',
+    '/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent'
+])
+def test_package_exes(host, exe):
+    assert host.file(exe).exists
+
+
+def test_service_file(host):
+    f = host.file('/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json')
+    assert not f.contains('"log_group_name": "jira-software-MY_STACK"')
+    assert f.user == 'root'
+    assert f.group == 'root'
+    assert f.mode == 0o0644

roles/aws_common/tasks/amazon.yml

@@ -6,3 +6,11 @@
       - ec2-utils
       - amazon-ssm-agent
       - amazon-efs-utils
+
+- name: Install CloudWatch Agent
+  yum:
+    name:
+      - "{{ aws_cloudwatch_agent_rpm }}"
+  when: atl_aws_enable_cloudwatch is defined and atl_aws_enable_cloudwatch
+  notify:
+    - Enable CloudWatch Agent

roles/aws_common/tasks/main.yml

@@ -1,14 +1,25 @@
 ---
 
-- name: Install distro-specific prerequisites
-  include_tasks: "{{ ansible_distribution|lower }}.yml"
-
 - name: Fetch local EC2 metadata
   ec2_metadata_facts:
   tags:
     - notest
 
+- name: Install distro-specific prerequisites
+  include_tasks: "{{ ansible_distribution|lower }}.yml"
+
 - name: Use EC2 instance ID for cluster node ID
   set_fact:
     atl_cluster_node_id: "{{ ansible_ec2_instance_id }}"
     atl_local_ipv4: "{{ ansible_ec2_local_ipv4 | default(ansible_default_ipv4.address) }}"
+
+- name: Generate CloudWatch config
+  template:
+    src: "amazon-cloudwatch-agent.json.j2"
+    dest: "/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json"
+    owner: root
+    group: root
+    mode: 0644
+  when: atl_aws_enable_cloudwatch is defined and atl_aws_enable_cloudwatch
+  notify:
+    - Restart CloudWatch Agent

roles/aws_common/templates/amazon-cloudwatch-agent.json.j2

@@ -0,0 +1,92 @@
+{
+  "agent": {
+    "metrics_collection_interval": 10,
+    "run_as_user": "root"
+  },
+
+  {% if atl_aws_enable_cloudwatch_logs is defined and atl_aws_enable_cloudwatch_logs %}
+    "logs": {
+      "logs_collected": {
+        "files": {
+          "collect_list": [
+
+            {% set comma = joiner(',') %}
+            {% for path in atl_product_log_locations[atl_product_family] %}
+              {{ comma() }}
+              {
+                "file_path": "{{ path }}/*",
+                "log_group_name": "{{ atl_aws_log_group }}",
+                "log_stream_name": "{instance_id}"
+              }
+            {% endfor %}
+
+          ]
+        }
+      }
+    },
+  {% endif %}
+
+  "metrics": {
+    "append_dimensions": {
+      "AutoScalingGroupName": "${aws:AutoScalingGroupName}",
+      "ImageId": "${aws:ImageId}",
+      "InstanceId": "${aws:InstanceId}",
+      "InstanceType": "${aws:InstanceType}"
+    },
+    "metrics_collected": {
+      "cpu": {
+        "measurement": [
+          "cpu_usage_idle",
+          "cpu_usage_iowait",
+          "cpu_usage_user",
+          "cpu_usage_system"
+        ],
+        "metrics_collection_interval": 10,
+        "totalcpu": false
+      },
+      "disk": {
+        "measurement": [
+          "used_percent",
+          "inodes_free"
+        ],
+        "metrics_collection_interval": 10,
+        "resources": [
+          "*"
+        ]
+      },
+      "diskio": {
+        "measurement": [
+          "io_time",
+          "write_bytes",
+          "read_bytes",
+          "writes",
+          "reads"
+        ],
+        "metrics_collection_interval": 10,
+        "resources": [
+          "*"
+        ]
+      },
+      "mem": {
+        "measurement": [
+          "mem_used_percent"
+        ],
+        "metrics_collection_interval": 10
+      },
+      "netstat": {
+        "measurement": [
+          "tcp_established",
+          "tcp_time_wait"
+        ],
+        "metrics_collection_interval": 10
+      },
+      "swap": {
+        "measurement": [
+          "swap_used_percent"
+        ],
+        "metrics_collection_interval": 10
+      }
+    }
+  }
+
+}