From cca50a4d9d8a7099e94dffe1af7f6c8629a774e0 Mon Sep 17 00:00:00 2001
From: Steve Smith <ssmith@atlassian.com>
Date: Tue, 10 Sep 2019 12:31:17 +1000
Subject: [PATCH 01/13] DCD-436: Install CloudWatch agent on AWS instances.

---
 roles/aws_common/defaults/main.yml                      | 5 +++++
 roles/aws_common/molecule/default/tests/test_default.py | 3 ++-
 roles/aws_common/tasks/amazon.yml                       | 1 +
 roles/aws_common/tasks/main.yml                         | 6 +++---
 4 files changed, 11 insertions(+), 4 deletions(-)
 create mode 100644 roles/aws_common/defaults/main.yml

diff --git a/roles/aws_common/defaults/main.yml b/roles/aws_common/defaults/main.yml
new file mode 100644
index 0000000..08808bc
--- /dev/null
+++ b/roles/aws_common/defaults/main.yml
@@ -0,0 +1,5 @@
+---
+
+# See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/download-cloudwatch-agent-commandline.html
+aws_download_region: "{{ ansible_ec2_placement_region | default('us-west-2') }}"
+aws_cloudwatch_agent_rpm: "https://s3.{{ aws_download_region }}.amazonaws.com/amazoncloudwatch-agent-{{ aws_download_region }}/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm"
diff --git a/roles/aws_common/molecule/default/tests/test_default.py b/roles/aws_common/molecule/default/tests/test_default.py
index 9f4b6c8..cf68ac4 100644
--- a/roles/aws_common/molecule/default/tests/test_default.py
+++ b/roles/aws_common/molecule/default/tests/test_default.py
@@ -10,7 +10,8 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
 @pytest.mark.parametrize('exe', [
     '/usr/bin/ec2-metadata',
     '/usr/bin/amazon-ssm-agent',
-    '/sbin/mount.efs'
+    '/sbin/mount.efs',
+    '/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent'
 ])
 def test_package_exes(host, exe):
     assert host.file(exe).exists
diff --git a/roles/aws_common/tasks/amazon.yml b/roles/aws_common/tasks/amazon.yml
index 4c2aebb..2f0ba7f 100644
--- a/roles/aws_common/tasks/amazon.yml
+++ b/roles/aws_common/tasks/amazon.yml
@@ -6,3 +6,4 @@
       - ec2-utils
       - amazon-ssm-agent
       - amazon-efs-utils
+      - "{{ aws_cloudwatch_agent_rpm }}"
diff --git a/roles/aws_common/tasks/main.yml b/roles/aws_common/tasks/main.yml
index 087b681..0f4b0cd 100644
--- a/roles/aws_common/tasks/main.yml
+++ b/roles/aws_common/tasks/main.yml
@@ -1,13 +1,13 @@
 ---
 
-- name: Install distro-specific prerequisites
-  include_tasks: "{{ ansible_distribution|lower }}.yml"
-
 - name: Fetch local EC2 metadata
   ec2_metadata_facts:
   tags:
     - notest
 
+- name: Install distro-specific prerequisites
+  include_tasks: "{{ ansible_distribution|lower }}.yml"
+
 - name: Use EC2 instance ID for cluster node ID
   set_fact:
     atl_cluster_node_id: "{{ ansible_ec2_instance_id }}"

From 5a98350702ea9f665c3ae6bc9c82950aebcb9302 Mon Sep 17 00:00:00 2001
From: Steve Smith <ssmith@atlassian.com>
Date: Wed, 11 Sep 2019 11:24:50 +1000
Subject: [PATCH 02/13] DCD-436: Enable/start CW agent.

---
 roles/aws_common/defaults/main.yml             |  3 +++
 roles/aws_common/handlers/main.yml             | 15 +++++++++++++++
 roles/aws_common/molecule/default/playbook.yml |  1 +
 roles/aws_common/tasks/amazon.yml              |  6 ++++++
 4 files changed, 25 insertions(+)
 create mode 100644 roles/aws_common/handlers/main.yml

diff --git a/roles/aws_common/defaults/main.yml b/roles/aws_common/defaults/main.yml
index 08808bc..8d5cef2 100644
--- a/roles/aws_common/defaults/main.yml
+++ b/roles/aws_common/defaults/main.yml
@@ -3,3 +3,6 @@
 # See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/download-cloudwatch-agent-commandline.html
 aws_download_region: "{{ ansible_ec2_placement_region | default('us-west-2') }}"
 aws_cloudwatch_agent_rpm: "https://s3.{{ aws_download_region }}.amazonaws.com/amazoncloudwatch-agent-{{ aws_download_region }}/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm"
+
+# Mostly for molecule testing, as skip-tags doesn't work with handlers.
+atl_aws_agent_restart: true
diff --git a/roles/aws_common/handlers/main.yml b/roles/aws_common/handlers/main.yml
new file mode 100644
index 0000000..768f8cf
--- /dev/null
+++ b/roles/aws_common/handlers/main.yml
@@ -0,0 +1,15 @@
+---
+
+- name: Enable CloudWatch Agent
+  systemd:
+    name: "amazon-cloudwatch-agent.service"
+    daemon_reload: true
+    enabled: true
+  when: atl_aws_agent_restart
+
+- name: Restart CloudWatch Agent
+  systemd:
+    name: "amazon-cloudwatch-agent.service"
+    enabled: true
+    status: restarted
+  when: atl_aws_agent_restart
diff --git a/roles/aws_common/molecule/default/playbook.yml b/roles/aws_common/molecule/default/playbook.yml
index 6ccfba2..e44015d 100644
--- a/roles/aws_common/molecule/default/playbook.yml
+++ b/roles/aws_common/molecule/default/playbook.yml
@@ -6,5 +6,6 @@
     ansible_default_ipv4:
       address: "9.9.9.9"
     ansible_ec2_instance_id: "NONE"
+    atl_aws_agent_restart: false
   roles:
     - role: aws_common
diff --git a/roles/aws_common/tasks/amazon.yml b/roles/aws_common/tasks/amazon.yml
index 2f0ba7f..dce19fe 100644
--- a/roles/aws_common/tasks/amazon.yml
+++ b/roles/aws_common/tasks/amazon.yml
@@ -6,4 +6,10 @@
       - ec2-utils
       - amazon-ssm-agent
       - amazon-efs-utils
+
+- name: Install CloudWatch Agent
+  yum:
+    name:
       - "{{ aws_cloudwatch_agent_rpm }}"
+  notify:
+    - Enable CloudWatch Agent

From e114fca7324aeadb3aa81274e13e84ba5d9e42f3 Mon Sep 17 00:00:00 2001
From: Steve Smith <ssmith@atlassian.com>
Date: Wed, 11 Sep 2019 13:05:07 +1000
Subject: [PATCH 03/13] DCD-436: Make log group unique.

---
 group_vars/aws_node_local.yml                 |  8 ++
 roles/aws_common/defaults/main.yml            |  2 +
 .../aws_common/molecule/default/molecule.yml  |  6 ++
 .../aws_common/molecule/default/playbook.yml  |  5 ++
 .../molecule/default/tests/test_default.py    |  8 ++
 roles/aws_common/tasks/main.yml               | 10 +++
 .../templates/amazon-cloudwatch-agent.json.j2 | 88 +++++++++++++++++++
 7 files changed, 127 insertions(+)
 create mode 100644 roles/aws_common/templates/amazon-cloudwatch-agent.json.j2

diff --git a/group_vars/aws_node_local.yml b/group_vars/aws_node_local.yml
index 6cb28f4..d6ff087 100644
--- a/group_vars/aws_node_local.yml
+++ b/group_vars/aws_node_local.yml
@@ -34,6 +34,14 @@ atl_product_installation_versioned: "{{ atl_product_installation_base }}/{{ atl_
 atl_product_installation_current: "{{ atl_product_installation_base }}/current"
 atl_installer_temp: "{{ atl_installation_base }}/tmp"
 
+atl_product_log_locations:
+  confluence:
+    - "{{ atl_product_installation_current }}/logs"
+  jira:
+    - "{{ atl_product_installation_current }}/logs"
+    - "{{ atl_product_home }}/logs"
+  stash: []
+  crowd: []
 
 #  The following are imports from the environment. These are generally
 #  set in /etc/atl by the CloudFormation template and sourced before
diff --git a/roles/aws_common/defaults/main.yml b/roles/aws_common/defaults/main.yml
index 8d5cef2..fbd36c7 100644
--- a/roles/aws_common/defaults/main.yml
+++ b/roles/aws_common/defaults/main.yml
@@ -6,3 +6,5 @@ aws_cloudwatch_agent_rpm: "https://s3.{{ aws_download_region }}.amazonaws.com/am
 
 # Mostly for molecule testing, as skip-tags doesn't work with handlers.
 atl_aws_agent_restart: true
+
+atl_aws_log_group: "{{ atl_product_edition }}-{{ atl_aws_stack_name }}"
diff --git a/roles/aws_common/molecule/default/molecule.yml b/roles/aws_common/molecule/default/molecule.yml
index e8b8bcf..9db2aa4 100644
--- a/roles/aws_common/molecule/default/molecule.yml
+++ b/roles/aws_common/molecule/default/molecule.yml
@@ -8,13 +8,19 @@ lint:
 platforms:
   - name: amazon_linux2
     image: amazonlinux:2
+    groups:
+      - aws_node_local
 # - name: ubuntu_lts
 #   image: ubuntu:bionic
 provisioner:
   name: ansible
   lint:
     name: ansible-lint
+  inventory:
+    links:
+      group_vars: ../../../../group_vars/
 verifier:
   name: testinfra
   lint:
     name: flake8
+    enabled: false
diff --git a/roles/aws_common/molecule/default/playbook.yml b/roles/aws_common/molecule/default/playbook.yml
index e44015d..0eb5b02 100644
--- a/roles/aws_common/molecule/default/playbook.yml
+++ b/roles/aws_common/molecule/default/playbook.yml
@@ -6,6 +6,11 @@
     ansible_default_ipv4:
       address: "9.9.9.9"
     ansible_ec2_instance_id: "NONE"
+
+    atl_product_family: "jira"
+    atl_product_edition: "jira-software"
+    atl_aws_stack_name: "MY_STACK"
+
     atl_aws_agent_restart: false
   roles:
     - role: aws_common
diff --git a/roles/aws_common/molecule/default/tests/test_default.py b/roles/aws_common/molecule/default/tests/test_default.py
index cf68ac4..53261bb 100644
--- a/roles/aws_common/molecule/default/tests/test_default.py
+++ b/roles/aws_common/molecule/default/tests/test_default.py
@@ -15,3 +15,11 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
 ])
 def test_package_exes(host, exe):
     assert host.file(exe).exists
+
+
+def test_service_file(host):
+    f = host.file('/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json')
+    assert f.contains('"log_group_name": "jira-software-MY_STACK"')
+    assert f.user == 'root'
+    assert f.group == 'root'
+    assert f.mode == 0o0644
diff --git a/roles/aws_common/tasks/main.yml b/roles/aws_common/tasks/main.yml
index 0f4b0cd..bb578c0 100644
--- a/roles/aws_common/tasks/main.yml
+++ b/roles/aws_common/tasks/main.yml
@@ -12,3 +12,13 @@
   set_fact:
     atl_cluster_node_id: "{{ ansible_ec2_instance_id }}"
     atl_local_ipv4: "{{ ansible_ec2_local_ipv4 | default(ansible_default_ipv4.address) }}"
+
+- name: Generate CloudWatch config
+  template:
+    src: "amazon-cloudwatch-agent.json.j2"
+    dest: "/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json"
+    owner: root
+    group: root
+    mode: 0644
+  notify:
+    - Restart CloudWatch Agent
diff --git a/roles/aws_common/templates/amazon-cloudwatch-agent.json.j2 b/roles/aws_common/templates/amazon-cloudwatch-agent.json.j2
new file mode 100644
index 0000000..b33ff45
--- /dev/null
+++ b/roles/aws_common/templates/amazon-cloudwatch-agent.json.j2
@@ -0,0 +1,88 @@
+{
+  "agent": {
+    "metrics_collection_interval": 10,
+    "run_as_user": "root"
+  },
+
+  "logs": {
+    "logs_collected": {
+      "files": {
+        "collect_list": [
+          {% set comma = joiner(',') %}
+          {% for path in atl_product_log_locations[atl_product_family] %}
+            {{ comma() }}
+            {
+              "file_path": "{{ path }}/*",
+              "log_group_name": "{{ atl_aws_log_group }}",
+              "log_stream_name": "{instance_id}"
+            }
+          {% endfor %}
+        ]
+      }
+    }
+  },
+
+  "metrics": {
+    "append_dimensions": {
+      "AutoScalingGroupName": "${aws:AutoScalingGroupName}",
+      "ImageId": "${aws:ImageId}",
+      "InstanceId": "${aws:InstanceId}",
+      "InstanceType": "${aws:InstanceType}"
+    },
+    "metrics_collected": {
+      "cpu": {
+        "measurement": [
+          "cpu_usage_idle",
+          "cpu_usage_iowait",
+          "cpu_usage_user",
+          "cpu_usage_system"
+        ],
+        "metrics_collection_interval": 10,
+        "totalcpu": false
+      },
+      "disk": {
+        "measurement": [
+          "used_percent",
+          "inodes_free"
+        ],
+        "metrics_collection_interval": 10,
+        "resources": [
+          "*"
+        ]
+      },
+      "diskio": {
+        "measurement": [
+          "io_time",
+          "write_bytes",
+          "read_bytes",
+          "writes",
+          "reads"
+        ],
+        "metrics_collection_interval": 10,
+        "resources": [
+          "*"
+        ]
+      },
+      "mem": {
+        "measurement": [
+          "mem_used_percent"
+        ],
+        "metrics_collection_interval": 10
+      },
+      "netstat": {
+        "measurement": [
+          "tcp_established",
+          "tcp_time_wait"
+        ],
+        "metrics_collection_interval": 10
+      },
+      "swap": {
+        "measurement": [
+          "swap_used_percent"
+        ],
+        "metrics_collection_interval": 10
+      }
+    }
+  }
+
+}

From f8358623d87833d8f4230b2569ce064b9e2c532f Mon Sep 17 00:00:00 2001
From: Steve Smith <ssmith@atlassian.com>
Date: Wed, 11 Sep 2019 13:22:52 +1000
Subject: [PATCH 04/13] DCD-436: Fix systemd restart.

---
 roles/aws_common/handlers/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/aws_common/handlers/main.yml b/roles/aws_common/handlers/main.yml
index 768f8cf..0e43a13 100644
--- a/roles/aws_common/handlers/main.yml
+++ b/roles/aws_common/handlers/main.yml
@@ -11,5 +11,5 @@
   systemd:
     name: "amazon-cloudwatch-agent.service"
     enabled: true
-    status: restarted
+    state: restarted
   when: atl_aws_agent_restart

From 9697229abb42f22687ee8d4ac0923dd89b1d9a0d Mon Sep 17 00:00:00 2001
From: Steve Smith <ssmith@atlassian.com>
Date: Wed, 11 Sep 2019 13:39:00 +1000
Subject: [PATCH 05/13] DCD-436: Fix Jira log directory.

---
 group_vars/aws_node_local.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/group_vars/aws_node_local.yml b/group_vars/aws_node_local.yml
index d6ff087..02a0742 100644
--- a/group_vars/aws_node_local.yml
+++ b/group_vars/aws_node_local.yml
@@ -39,7 +39,7 @@ atl_product_log_locations:
     - "{{ atl_product_installation_current }}/logs"
   jira:
     - "{{ atl_product_installation_current }}/logs"
-    - "{{ atl_product_home }}/logs"
+    - "{{ atl_product_home }}/log"
   stash: []
   crowd: []
 

From 28d35abd535861996333902ecd35fecc26a782f5 Mon Sep 17 00:00:00 2001
From: Steve Smith <ssmith@atlassian.com>
Date: Wed, 11 Sep 2019 14:24:57 +1000
Subject: [PATCH 06/13] DCD-436: Add Confluence home log dir.

---
 group_vars/aws_node_local.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/group_vars/aws_node_local.yml b/group_vars/aws_node_local.yml
index 02a0742..7504812 100644
--- a/group_vars/aws_node_local.yml
+++ b/group_vars/aws_node_local.yml
@@ -37,6 +37,7 @@ atl_installer_temp: "{{ atl_installation_base }}/tmp"
 atl_product_log_locations:
   confluence:
     - "{{ atl_product_installation_current }}/logs"
+    - "{{ atl_product_home }}/logs"
   jira:
     - "{{ atl_product_installation_current }}/logs"
     - "{{ atl_product_home }}/log"

From 35e37afd24bc10ce89d10fafc0e0e667ec584835 Mon Sep 17 00:00:00 2001
From: Steve Smith <ssmith@atlassian.com>
Date: Wed, 11 Sep 2019 14:33:48 +1000
Subject: [PATCH 07/13] DCD-436: Add Bitbucket log dir.

---
 group_vars/aws_node_local.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/group_vars/aws_node_local.yml b/group_vars/aws_node_local.yml
index 7504812..65505fc 100644
--- a/group_vars/aws_node_local.yml
+++ b/group_vars/aws_node_local.yml
@@ -41,7 +41,8 @@ atl_product_log_locations:
   jira:
     - "{{ atl_product_installation_current }}/logs"
     - "{{ atl_product_home }}/log"
-  stash: []
+  stash:
+    - "{{ atl_product_home }}/log"
   crowd: []
 
 #  The following are imports from the environment. These are generally

From 114c1d2e8e86df589bc2b5b54d87d5e152ecaa4b Mon Sep 17 00:00:00 2001
From: Steve Smith <ssmith@atlassian.com>
Date: Wed, 11 Sep 2019 14:59:00 +1000
Subject: [PATCH 08/13] DCD-436: Add ability to disable log aggregation.

---
 group_vars/aws_node_local.yml                 |  2 ++
 .../aws_common/molecule/default/playbook.yml  |  2 ++
 .../molecule/logs-disabled/Dockerfile.j2      | 14 ++++++++
 .../molecule/logs-disabled/molecule.yml       | 26 ++++++++++++++
 .../molecule/logs-disabled/playbook.yml       | 19 ++++++++++
 .../logs-disabled/tests/test_default.py       | 25 +++++++++++++
 .../templates/amazon-cloudwatch-agent.json.j2 | 36 ++++++++++---------
 7 files changed, 108 insertions(+), 16 deletions(-)
 create mode 100644 roles/aws_common/molecule/logs-disabled/Dockerfile.j2
 create mode 100644 roles/aws_common/molecule/logs-disabled/molecule.yml
 create mode 100644 roles/aws_common/molecule/logs-disabled/playbook.yml
 create mode 100644 roles/aws_common/molecule/logs-disabled/tests/test_default.py

diff --git a/group_vars/aws_node_local.yml b/group_vars/aws_node_local.yml
index 65505fc..1bb262b 100644
--- a/group_vars/aws_node_local.yml
+++ b/group_vars/aws_node_local.yml
@@ -56,6 +56,8 @@ atl_aws_region: "{{ lookup('env', 'ATL_AWS_REGION') }}"
 atl_aws_iam_role: "{{ lookup('env', 'ATL_AWS_IAM_ROLE') }}"
 atl_aws_iam_role_arn: "{{ lookup('env', 'ATL_AWS_IAM_ROLE_ARN') }}"
 
+atl_aws_enable_cw_logs: "{{ lookup('env', 'ATL_AWS_ENABLE_CW_LOGS') or false }}"
+
 atl_db_engine: "{{ lookup('env', 'ATL_DB_ENGINE') }}"
 atl_db_host: "{{ lookup('env', 'ATL_DB_HOST') }}"
 atl_db_port: "{{ lookup('env', 'ATL_DB_PORT') or '5432' }}"
diff --git a/roles/aws_common/molecule/default/playbook.yml b/roles/aws_common/molecule/default/playbook.yml
index 0eb5b02..6af238a 100644
--- a/roles/aws_common/molecule/default/playbook.yml
+++ b/roles/aws_common/molecule/default/playbook.yml
@@ -11,6 +11,8 @@
     atl_product_edition: "jira-software"
     atl_aws_stack_name: "MY_STACK"
 
+    atl_aws_enable_cw_logs: true
     atl_aws_agent_restart: false
+
   roles:
     - role: aws_common
diff --git a/roles/aws_common/molecule/logs-disabled/Dockerfile.j2 b/roles/aws_common/molecule/logs-disabled/Dockerfile.j2
new file mode 100644
index 0000000..e6aa95d
--- /dev/null
+++ b/roles/aws_common/molecule/logs-disabled/Dockerfile.j2
@@ -0,0 +1,14 @@
+# Molecule managed
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi
diff --git a/roles/aws_common/molecule/logs-disabled/molecule.yml b/roles/aws_common/molecule/logs-disabled/molecule.yml
new file mode 100644
index 0000000..9db2aa4
--- /dev/null
+++ b/roles/aws_common/molecule/logs-disabled/molecule.yml
@@ -0,0 +1,26 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+lint:
+  name: yamllint
+platforms:
+  - name: amazon_linux2
+    image: amazonlinux:2
+    groups:
+      - aws_node_local
+# - name: ubuntu_lts
+#   image: ubuntu:bionic
+provisioner:
+  name: ansible
+  lint:
+    name: ansible-lint
+  inventory:
+    links:
+      group_vars: ../../../../group_vars/
+verifier:
+  name: testinfra
+  lint:
+    name: flake8
+    enabled: false
diff --git a/roles/aws_common/molecule/logs-disabled/playbook.yml b/roles/aws_common/molecule/logs-disabled/playbook.yml
new file mode 100644
index 0000000..3431e5c
--- /dev/null
+++ b/roles/aws_common/molecule/logs-disabled/playbook.yml
@@ -0,0 +1,19 @@
+---
+- name: Converge
+  hosts: all
+  vars:
+    ansible_ec2_local_ipv4: "1.1.1.1"
+    ansible_default_ipv4:
+      address: "9.9.9.9"
+    ansible_ec2_instance_id: "NONE"
+
+    atl_product_family: "jira"
+    atl_product_edition: "jira-software"
+    atl_aws_stack_name: "MY_STACK"
+
+    atl_aws_enable_cw_logs: "false"
+
+    atl_aws_agent_restart: false
+
+  roles:
+    - role: aws_common
diff --git a/roles/aws_common/molecule/logs-disabled/tests/test_default.py b/roles/aws_common/molecule/logs-disabled/tests/test_default.py
new file mode 100644
index 0000000..cc94261
--- /dev/null
+++ b/roles/aws_common/molecule/logs-disabled/tests/test_default.py
@@ -0,0 +1,25 @@
+import os
+import pytest
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+@pytest.mark.parametrize('exe', [
+    '/usr/bin/ec2-metadata',
+    '/usr/bin/amazon-ssm-agent',
+    '/sbin/mount.efs',
+    '/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent'
+])
+def test_package_exes(host, exe):
+    assert host.file(exe).exists
+
+
+def test_service_file(host):
+    f = host.file('/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json')
+    assert not f.contains('"log_group_name": "jira-software-MY_STACK"')
+    assert f.user == 'root'
+    assert f.group == 'root'
+    assert f.mode == 0o0644
diff --git a/roles/aws_common/templates/amazon-cloudwatch-agent.json.j2 b/roles/aws_common/templates/amazon-cloudwatch-agent.json.j2
index b33ff45..3c23e84 100644
--- a/roles/aws_common/templates/amazon-cloudwatch-agent.json.j2
+++ b/roles/aws_common/templates/amazon-cloudwatch-agent.json.j2
@@ -4,23 +4,27 @@
     "run_as_user": "root"
   },
 
-  "logs": {
-    "logs_collected": {
-      "files": {
-        "collect_list": [
-          {% set comma = joiner(',') %}
-          {% for path in atl_product_log_locations[atl_product_family] %}
-            {{ comma() }}
-            {
-              "file_path": "{{ path }}/*",
-              "log_group_name": "{{ atl_aws_log_group }}",
-              "log_stream_name": "{instance_id}"
-            }
-          {% endfor %}
-        ]
+  {% if atl_aws_enable_cw_logs == true or atl_aws_enable_cw_logs == 'true' %}
+    "logs": {
+      "logs_collected": {
+        "files": {
+          "collect_list": [
+
+            {% set comma = joiner(',') %}
+            {% for path in atl_product_log_locations[atl_product_family] %}
+              {{ comma() }}
+              {
+                "file_path": "{{ path }}/*",
+                "log_group_name": "{{ atl_aws_log_group }}",
+                "log_stream_name": "{instance_id}"
+              }
+            {% endfor %}
+
+          ]
+        }
       }
-    }
-  },
+    },
+  {% endif %}
 
   "metrics": {
     "append_dimensions": {

From e1090ba4feb86cf7629c17c3d342d3a0f03fafad Mon Sep 17 00:00:00 2001
From: Steve Smith <ssmith@atlassian.com>
Date: Wed, 11 Sep 2019 15:04:24 +1000
Subject: [PATCH 09/13] DCD-436: Add new test scenario to the pipelines config.

---
 bitbucket-pipelines.yml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml
index ec5ec9f..6de5040 100644
--- a/bitbucket-pipelines.yml
+++ b/bitbucket-pipelines.yml
@@ -14,7 +14,7 @@ pipelines:
     - step:
         name: Pre Parallelization stage
         script:
-          - echo "Running tests in 26 batches"
+          - echo "Running tests in 27 batches"
     - step:
         name: Check if number of batches match actual number of scenarios
         script:
@@ -261,4 +261,13 @@ pipelines:
            - ./bin/install-ansible
            - ./bin/run-tests-in-batches --batch 26
            
+       - step:
+           name: Molecule Test Batch - 27
+           services:
+             - docker
+           script:
+           - apt-get update && apt-get install -y virtualenv python-dev
+           - ./bin/install-ansible
+           - ./bin/run-tests-in-batches --batch 27
+           
        

From ee6df8f8a871464c7127208c946bee1734da283e Mon Sep 17 00:00:00 2001
From: Steve Smith <ssmith@atlassian.com>
Date: Thu, 12 Sep 2019 10:57:45 +1000
Subject: [PATCH 10/13] DCD-436: Add flag to disable CloudWatch completely.

---
 group_vars/aws_node_local.yml                 |  3 ++-
 roles/aws_common/defaults/main.yml            |  3 +++
 .../molecule/cw-disabled/Dockerfile.j2        | 14 ++++++++++
 .../molecule/cw-disabled/molecule.yml         | 26 +++++++++++++++++++
 .../molecule/cw-disabled/playbook.yml         | 17 ++++++++++++
 .../cw-disabled/tests/test_default.py         | 23 ++++++++++++++++
 .../aws_common/molecule/default/playbook.yml  |  2 ++
 .../molecule/logs-disabled/playbook.yml       |  3 ++-
 roles/aws_common/tasks/amazon.yml             |  1 +
 roles/aws_common/tasks/main.yml               |  1 +
 .../templates/amazon-cloudwatch-agent.json.j2 |  2 +-
 11 files changed, 92 insertions(+), 3 deletions(-)
 create mode 100644 roles/aws_common/molecule/cw-disabled/Dockerfile.j2
 create mode 100644 roles/aws_common/molecule/cw-disabled/molecule.yml
 create mode 100644 roles/aws_common/molecule/cw-disabled/playbook.yml
 create mode 100644 roles/aws_common/molecule/cw-disabled/tests/test_default.py

diff --git a/group_vars/aws_node_local.yml b/group_vars/aws_node_local.yml
index 1bb262b..d6c2c9a 100644
--- a/group_vars/aws_node_local.yml
+++ b/group_vars/aws_node_local.yml
@@ -56,7 +56,8 @@ atl_aws_region: "{{ lookup('env', 'ATL_AWS_REGION') }}"
 atl_aws_iam_role: "{{ lookup('env', 'ATL_AWS_IAM_ROLE') }}"
 atl_aws_iam_role_arn: "{{ lookup('env', 'ATL_AWS_IAM_ROLE_ARN') }}"
 
-atl_aws_enable_cw_logs: "{{ lookup('env', 'ATL_AWS_ENABLE_CW_LOGS') or false }}"
+atl_aws_enable_cw: "{{ lookup('env', 'ATL_AWS_ENABLE_CW')|bool or false }}"
+atl_aws_enable_cw_logs: "{{ lookup('env', 'ATL_AWS_ENABLE_CW_LOGS')|bool or false }}"
 
 atl_db_engine: "{{ lookup('env', 'ATL_DB_ENGINE') }}"
 atl_db_host: "{{ lookup('env', 'ATL_DB_HOST') }}"
diff --git a/roles/aws_common/defaults/main.yml b/roles/aws_common/defaults/main.yml
index fbd36c7..f6966f6 100644
--- a/roles/aws_common/defaults/main.yml
+++ b/roles/aws_common/defaults/main.yml
@@ -4,6 +4,9 @@
 aws_download_region: "{{ ansible_ec2_placement_region | default('us-west-2') }}"
 aws_cloudwatch_agent_rpm: "https://s3.{{ aws_download_region }}.amazonaws.com/amazoncloudwatch-agent-{{ aws_download_region }}/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm"
 
+atl_aws_enable_cw: true
+atl_aws_enable_cw_logs: false
+
 # Mostly for molecule testing, as skip-tags doesn't work with handlers.
 atl_aws_agent_restart: true
 
diff --git a/roles/aws_common/molecule/cw-disabled/Dockerfile.j2 b/roles/aws_common/molecule/cw-disabled/Dockerfile.j2
new file mode 100644
index 0000000..e6aa95d
--- /dev/null
+++ b/roles/aws_common/molecule/cw-disabled/Dockerfile.j2
@@ -0,0 +1,14 @@
+# Molecule managed
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi
diff --git a/roles/aws_common/molecule/cw-disabled/molecule.yml b/roles/aws_common/molecule/cw-disabled/molecule.yml
new file mode 100644
index 0000000..9db2aa4
--- /dev/null
+++ b/roles/aws_common/molecule/cw-disabled/molecule.yml
@@ -0,0 +1,26 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+lint:
+  name: yamllint
+platforms:
+  - name: amazon_linux2
+    image: amazonlinux:2
+    groups:
+      - aws_node_local
+# - name: ubuntu_lts
+#   image: ubuntu:bionic
+provisioner:
+  name: ansible
+  lint:
+    name: ansible-lint
+  inventory:
+    links:
+      group_vars: ../../../../group_vars/
+verifier:
+  name: testinfra
+  lint:
+    name: flake8
+    enabled: false
diff --git a/roles/aws_common/molecule/cw-disabled/playbook.yml b/roles/aws_common/molecule/cw-disabled/playbook.yml
new file mode 100644
index 0000000..b235a9e
--- /dev/null
+++ b/roles/aws_common/molecule/cw-disabled/playbook.yml
@@ -0,0 +1,17 @@
+---
+- name: Converge
+  hosts: all
+  vars:
+    ansible_ec2_local_ipv4: "1.1.1.1"
+    ansible_default_ipv4:
+      address: "9.9.9.9"
+    ansible_ec2_instance_id: "NONE"
+
+    atl_product_family: "jira"
+    atl_product_edition: "jira-software"
+    atl_aws_stack_name: "MY_STACK"
+
+    atl_aws_enable_cw: "{{ 'false'|bool }}"
+
+  roles:
+    - role: aws_common
diff --git a/roles/aws_common/molecule/cw-disabled/tests/test_default.py b/roles/aws_common/molecule/cw-disabled/tests/test_default.py
new file mode 100644
index 0000000..eae1cd7
--- /dev/null
+++ b/roles/aws_common/molecule/cw-disabled/tests/test_default.py
@@ -0,0 +1,23 @@
+import os
+import pytest
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+@pytest.mark.parametrize('exe', [
+    '/usr/bin/ec2-metadata',
+    '/usr/bin/amazon-ssm-agent',
+    '/sbin/mount.efs'
+])
+def test_package_exes(host, exe):
+    assert host.file(exe).exists
+
+@pytest.mark.parametrize('path', [
+    '/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent',
+    '/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json'
+])
+def test_package_not_installed(host, path):
+    assert not host.file(path).exists
diff --git a/roles/aws_common/molecule/default/playbook.yml b/roles/aws_common/molecule/default/playbook.yml
index 6af238a..4c5ea24 100644
--- a/roles/aws_common/molecule/default/playbook.yml
+++ b/roles/aws_common/molecule/default/playbook.yml
@@ -11,6 +11,8 @@
     atl_product_edition: "jira-software"
     atl_aws_stack_name: "MY_STACK"
 
+    # The `bool` pipe is a sanity check for group file.
+    atl_aws_enable_cw: "{{ 'true'|bool }}"
     atl_aws_enable_cw_logs: true
     atl_aws_agent_restart: false
 
diff --git a/roles/aws_common/molecule/logs-disabled/playbook.yml b/roles/aws_common/molecule/logs-disabled/playbook.yml
index 3431e5c..902f70f 100644
--- a/roles/aws_common/molecule/logs-disabled/playbook.yml
+++ b/roles/aws_common/molecule/logs-disabled/playbook.yml
@@ -11,7 +11,8 @@
     atl_product_edition: "jira-software"
     atl_aws_stack_name: "MY_STACK"
 
-    atl_aws_enable_cw_logs: "false"
+    atl_aws_enable_cw: true
+    atl_aws_enable_cw_logs: false
 
     atl_aws_agent_restart: false
 
diff --git a/roles/aws_common/tasks/amazon.yml b/roles/aws_common/tasks/amazon.yml
index dce19fe..b471439 100644
--- a/roles/aws_common/tasks/amazon.yml
+++ b/roles/aws_common/tasks/amazon.yml
@@ -11,5 +11,6 @@
   yum:
     name:
       - "{{ aws_cloudwatch_agent_rpm }}"
+  when: atl_aws_enable_cw is defined and atl_aws_enable_cw
   notify:
     - Enable CloudWatch Agent
diff --git a/roles/aws_common/tasks/main.yml b/roles/aws_common/tasks/main.yml
index bb578c0..16219ae 100644
--- a/roles/aws_common/tasks/main.yml
+++ b/roles/aws_common/tasks/main.yml
@@ -20,5 +20,6 @@
     owner: root
     group: root
     mode: 0644
+  when: atl_aws_enable_cw is defined and atl_aws_enable_cw
   notify:
     - Restart CloudWatch Agent
diff --git a/roles/aws_common/templates/amazon-cloudwatch-agent.json.j2 b/roles/aws_common/templates/amazon-cloudwatch-agent.json.j2
index 3c23e84..d3a63da 100644
--- a/roles/aws_common/templates/amazon-cloudwatch-agent.json.j2
+++ b/roles/aws_common/templates/amazon-cloudwatch-agent.json.j2
@@ -4,7 +4,7 @@
     "run_as_user": "root"
   },
 
-  {% if atl_aws_enable_cw_logs == true or atl_aws_enable_cw_logs == 'true' %}
+  {% if atl_aws_enable_cw_logs is defined and atl_aws_enable_cw_logs %}
     "logs": {
       "logs_collected": {
         "files": {

From 011e7187649a37c2a626d5ac4174f3ed114344cb Mon Sep 17 00:00:00 2001
From: Steve Smith <ssmith@atlassian.com>
Date: Thu, 12 Sep 2019 11:21:28 +1000
Subject: [PATCH 11/13] DCD-436: CloudWatch disabled by default for backwards
 compatibility.

---
 roles/aws_common/defaults/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/aws_common/defaults/main.yml b/roles/aws_common/defaults/main.yml
index f6966f6..11fca42 100644
--- a/roles/aws_common/defaults/main.yml
+++ b/roles/aws_common/defaults/main.yml
@@ -4,7 +4,7 @@
 aws_download_region: "{{ ansible_ec2_placement_region | default('us-west-2') }}"
 aws_cloudwatch_agent_rpm: "https://s3.{{ aws_download_region }}.amazonaws.com/amazoncloudwatch-agent-{{ aws_download_region }}/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm"
 
-atl_aws_enable_cw: true
+atl_aws_enable_cw: false
 atl_aws_enable_cw_logs: false
 
 # Mostly for molecule testing, as skip-tags doesn't work with handlers.

From 83aed04c761db3ee5674c1c1daacec7c2dfd9b82 Mon Sep 17 00:00:00 2001
From: Steve Smith <ssmith@atlassian.com>
Date: Thu, 12 Sep 2019 12:30:29 +1000
Subject: [PATCH 12/13] DCD-436: Enable new molecule scenario

---
 bitbucket-pipelines.yml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml
index 6de5040..a63c197 100644
--- a/bitbucket-pipelines.yml
+++ b/bitbucket-pipelines.yml
@@ -14,7 +14,7 @@ pipelines:
     - step:
         name: Pre Parallelization stage
         script:
-          - echo "Running tests in 27 batches"
+          - echo "Running tests in 28 batches"
     - step:
         name: Check if number of batches match actual number of scenarios
         script:
@@ -270,4 +270,13 @@ pipelines:
            - ./bin/install-ansible
            - ./bin/run-tests-in-batches --batch 27
            
+       - step:
+           name: Molecule Test Batch - 28
+           services:
+             - docker
+           script:
+           - apt-get update && apt-get install -y virtualenv python-dev
+           - ./bin/install-ansible
+           - ./bin/run-tests-in-batches --batch 28
+           
        

From edae21135648bfefabb67acef6c37c0617bfc816 Mon Sep 17 00:00:00 2001
From: Steve Smith <ssmith@atlassian.com>
Date: Fri, 13 Sep 2019 12:01:47 +1000
Subject: [PATCH 13/13] DCD-436: Use full cloudwatch name for clarity.

---
 group_vars/aws_node_local.yml                              | 4 ++--
 roles/aws_common/defaults/main.yml                         | 4 ++--
 roles/aws_common/molecule/cw-disabled/playbook.yml         | 2 +-
 roles/aws_common/molecule/default/playbook.yml             | 4 ++--
 roles/aws_common/molecule/logs-disabled/playbook.yml       | 4 ++--
 roles/aws_common/tasks/amazon.yml                          | 2 +-
 roles/aws_common/tasks/main.yml                            | 2 +-
 roles/aws_common/templates/amazon-cloudwatch-agent.json.j2 | 2 +-
 8 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/group_vars/aws_node_local.yml b/group_vars/aws_node_local.yml
index d6c2c9a..cd6ea3b 100644
--- a/group_vars/aws_node_local.yml
+++ b/group_vars/aws_node_local.yml
@@ -56,8 +56,8 @@ atl_aws_region: "{{ lookup('env', 'ATL_AWS_REGION') }}"
 atl_aws_iam_role: "{{ lookup('env', 'ATL_AWS_IAM_ROLE') }}"
 atl_aws_iam_role_arn: "{{ lookup('env', 'ATL_AWS_IAM_ROLE_ARN') }}"
 
-atl_aws_enable_cw: "{{ lookup('env', 'ATL_AWS_ENABLE_CW')|bool or false }}"
-atl_aws_enable_cw_logs: "{{ lookup('env', 'ATL_AWS_ENABLE_CW_LOGS')|bool or false }}"
+atl_aws_enable_cloudwatch: "{{ lookup('env', 'ATL_AWS_ENABLE_CLOUDWATCH')|bool or false }}"
+atl_aws_enable_cloudwatch_logs: "{{ lookup('env', 'ATL_AWS_ENABLE_CLOUDWATCH_LOGS')|bool or false }}"
 
 atl_db_engine: "{{ lookup('env', 'ATL_DB_ENGINE') }}"
 atl_db_host: "{{ lookup('env', 'ATL_DB_HOST') }}"
diff --git a/roles/aws_common/defaults/main.yml b/roles/aws_common/defaults/main.yml
index 11fca42..3cb3b65 100644
--- a/roles/aws_common/defaults/main.yml
+++ b/roles/aws_common/defaults/main.yml
@@ -4,8 +4,8 @@
 aws_download_region: "{{ ansible_ec2_placement_region | default('us-west-2') }}"
 aws_cloudwatch_agent_rpm: "https://s3.{{ aws_download_region }}.amazonaws.com/amazoncloudwatch-agent-{{ aws_download_region }}/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm"
 
-atl_aws_enable_cw: false
-atl_aws_enable_cw_logs: false
+atl_aws_enable_cloudwatch: false
+atl_aws_enable_cloudwatch_logs: false
 
 # Mostly for molecule testing, as skip-tags doesn't work with handlers.
 atl_aws_agent_restart: true
diff --git a/roles/aws_common/molecule/cw-disabled/playbook.yml b/roles/aws_common/molecule/cw-disabled/playbook.yml
index b235a9e..ffe0919 100644
--- a/roles/aws_common/molecule/cw-disabled/playbook.yml
+++ b/roles/aws_common/molecule/cw-disabled/playbook.yml
@@ -11,7 +11,7 @@
     atl_product_edition: "jira-software"
     atl_aws_stack_name: "MY_STACK"
 
-    atl_aws_enable_cw: "{{ 'false'|bool }}"
+    atl_aws_enable_cloudwatch: "{{ 'false'|bool }}"
 
   roles:
     - role: aws_common
diff --git a/roles/aws_common/molecule/default/playbook.yml b/roles/aws_common/molecule/default/playbook.yml
index 4c5ea24..80c540a 100644
--- a/roles/aws_common/molecule/default/playbook.yml
+++ b/roles/aws_common/molecule/default/playbook.yml
@@ -12,8 +12,8 @@
     atl_aws_stack_name: "MY_STACK"
 
     # The `bool` pipe is a sanity check for group file.
-    atl_aws_enable_cw: "{{ 'true'|bool }}"
-    atl_aws_enable_cw_logs: true
+    atl_aws_enable_cloudwatch: "{{ 'true'|bool }}"
+    atl_aws_enable_cloudwatch_logs: true
     atl_aws_agent_restart: false
 
   roles:
diff --git a/roles/aws_common/molecule/logs-disabled/playbook.yml b/roles/aws_common/molecule/logs-disabled/playbook.yml
index 902f70f..9b0f699 100644
--- a/roles/aws_common/molecule/logs-disabled/playbook.yml
+++ b/roles/aws_common/molecule/logs-disabled/playbook.yml
@@ -11,8 +11,8 @@
     atl_product_edition: "jira-software"
     atl_aws_stack_name: "MY_STACK"
 
-    atl_aws_enable_cw: true
-    atl_aws_enable_cw_logs: false
+    atl_aws_enable_cloudwatch: true
+    atl_aws_enable_cloudwatch_logs: false
 
     atl_aws_agent_restart: false
 
diff --git a/roles/aws_common/tasks/amazon.yml b/roles/aws_common/tasks/amazon.yml
index b471439..a6592bf 100644
--- a/roles/aws_common/tasks/amazon.yml
+++ b/roles/aws_common/tasks/amazon.yml
@@ -11,6 +11,6 @@
   yum:
     name:
       - "{{ aws_cloudwatch_agent_rpm }}"
-  when: atl_aws_enable_cw is defined and atl_aws_enable_cw
+  when: atl_aws_enable_cloudwatch is defined and atl_aws_enable_cloudwatch
   notify:
     - Enable CloudWatch Agent
diff --git a/roles/aws_common/tasks/main.yml b/roles/aws_common/tasks/main.yml
index 16219ae..5155e76 100644
--- a/roles/aws_common/tasks/main.yml
+++ b/roles/aws_common/tasks/main.yml
@@ -20,6 +20,6 @@
     owner: root
     group: root
     mode: 0644
-  when: atl_aws_enable_cw is defined and atl_aws_enable_cw
+  when: atl_aws_enable_cloudwatch is defined and atl_aws_enable_cloudwatch
   notify:
     - Restart CloudWatch Agent
diff --git a/roles/aws_common/templates/amazon-cloudwatch-agent.json.j2 b/roles/aws_common/templates/amazon-cloudwatch-agent.json.j2
index d3a63da..17b31ca 100644
--- a/roles/aws_common/templates/amazon-cloudwatch-agent.json.j2
+++ b/roles/aws_common/templates/amazon-cloudwatch-agent.json.j2
@@ -4,7 +4,7 @@
     "run_as_user": "root"
   },
 
-  {% if atl_aws_enable_cw_logs is defined and atl_aws_enable_cw_logs %}
+  {% if atl_aws_enable_cloudwatch_logs is defined and atl_aws_enable_cloudwatch_logs %}
     "logs": {
       "logs_collected": {
         "files": {