From 84e9377824d7a787ed4bdc63c3da900dc08c973b Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Mon, 8 Jul 2019 12:03:24 +1000 Subject: [PATCH 01/10] DCD-434: Add molecule test for bitbucket config. --- .../molecule/default/Dockerfile.j2 | 14 ++++++++ .../molecule/default/molecule.yml | 36 +++++++++++++++++++ .../molecule/default/playbook.yml | 19 ++++++++++ .../molecule/default/tests/test_default.py | 16 +++++++++ 4 files changed, 85 insertions(+) create mode 100644 roles/bitbucket_config/molecule/default/Dockerfile.j2 create mode 100644 roles/bitbucket_config/molecule/default/molecule.yml create mode 100644 roles/bitbucket_config/molecule/default/playbook.yml create mode 100644 roles/bitbucket_config/molecule/default/tests/test_default.py diff --git a/roles/bitbucket_config/molecule/default/Dockerfile.j2 b/roles/bitbucket_config/molecule/default/Dockerfile.j2 new file mode 100644 index 0000000..e6aa95d --- /dev/null +++ b/roles/bitbucket_config/molecule/default/Dockerfile.j2 @@ -0,0 +1,14 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi diff --git a/roles/bitbucket_config/molecule/default/molecule.yml b/roles/bitbucket_config/molecule/default/molecule.yml new file mode 100644 index 0000000..7f082f6 --- /dev/null +++ b/roles/bitbucket_config/molecule/default/molecule.yml @@ -0,0 +1,36 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: amazon_linux2 + image: amazonlinux:2 + groups: + - aws_node_local + ulimits: + - nofile:262144:262144 + - name: ubuntu_lts + image: ubuntu:bionic + groups: + - aws_node_local + ulimits: + - nofile:262144:262144 +provisioner: + name: ansible + options: + skip-tags: runtime_pkg + lint: + name: ansible-lint + options: + x: ["701"] + inventory: + links: + group_vars: ../../../../group_vars/ +verifier: + name: testinfra + lint: + name: flake8 + enabled: false diff --git a/roles/bitbucket_config/molecule/default/playbook.yml b/roles/bitbucket_config/molecule/default/playbook.yml new file mode 100644 index 0000000..f9c424b --- /dev/null +++ b/roles/bitbucket_config/molecule/default/playbook.yml @@ -0,0 +1,19 @@ +--- +- name: Converge + hosts: all + vars: + atl_product_family: "stash" + atl_product_edition: "bitbucket" + atl_product_user: "bitbucket" + atl_product_version: "6.3.1" + + atl_product_home: "{{ atl_shared_mountpoint }}/{{ atl_product_edition }}" + + atl_jdbc_user: 'bb_db_user' + atl_jdbc_password: 'molecule_password' + + roles: + - role: linux_common + - role: product_common + - role: product_install + - role: bitbucket_config diff --git a/roles/bitbucket_config/molecule/default/tests/test_default.py b/roles/bitbucket_config/molecule/default/tests/test_default.py new file mode 100644 index 0000000..00593ec --- /dev/null +++ b/roles/bitbucket_config/molecule/default/tests/test_default.py @@ -0,0 +1,16 @@ +import os + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +def test_config_file(host): + f = host.file('/media/atl/bitbucket/shared/bitbucket.properties') + assert f.exists + assert f.user == 'bitbucket' + + assert f.contains("jdbc.driver=org.postgresql.Driver") + assert f.contains("jdbc.user=bb_db_user") + assert f.contains("jdbc.password=molecule_password") From 81dbd67c118e9415a4436ac436972a9c991dfcc6 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Mon, 8 Jul 2019 12:19:49 +1000 Subject: [PATCH 02/10] DCD-434: Add injection of heap and additional JVM opts into BB environment. --- group_vars/aws_node_local.yml | 1 + roles/product_startup/molecule/bitbucket/playbook.yml | 5 +++++ .../product_startup/molecule/bitbucket/tests/test_default.py | 3 +++ 3 files changed, 9 insertions(+) diff --git a/group_vars/aws_node_local.yml b/group_vars/aws_node_local.yml index 699745f..b4ed155 100644 --- a/group_vars/aws_node_local.yml +++ b/group_vars/aws_node_local.yml @@ -77,6 +77,7 @@ atl_jdbc_query_params_for_engine: atl_jdbc_url: "jdbc:postgresql://{{ atl_db_host }}:{{ atl_db_port }}/{{ atl_jdbc_db_name }}{{ atl_jdbc_query_params_for_engine[atl_db_engine]| default('') }}" atl_jvm_heap: "{{ lookup('env', 'ATL_JVM_HEAP') or '2048m' }}" +atl_jvm_opts: "{{ lookup('env', 'ATL_JVM_OPTS') or '' }}" atl_catalina_opts: "{{ lookup('env', 'ATL_CATALINA_OPTS') or '' }}" atl_proxy_name: "{{ lookup('env', 'ATL_PROXY_NAME') | lower }}" atl_proxy_port: "{{ lookup('env', 'ATL_TOMCAT_PROXYPORT') }}" diff --git a/roles/product_startup/molecule/bitbucket/playbook.yml b/roles/product_startup/molecule/bitbucket/playbook.yml index 08520cb..0542b3a 100644 --- a/roles/product_startup/molecule/bitbucket/playbook.yml +++ b/roles/product_startup/molecule/bitbucket/playbook.yml @@ -8,11 +8,16 @@ atl_product_home: "{{ atl_shared_mountpoint }}/{{ atl_product_edition }}" atl_product_home_shared: "{{ atl_product_home }}/shared" + atl_jvm_heap: "dummy_heap" + atl_jvm_opts: "dummy_opts" atl_startup_systemd_params: - "UMask=0027" - "LimitNOFILE=4096" - "Environment=BITBUCKET_HOME={{ atl_product_home }}" + - "Environment=JVM_MAXIMUM_MEMORY={{ atl_jvm_heap }}" + - "Environment=JVM_MINIMUM_MEMORY={{ atl_jvm_heap }}" + - "Environment=JVM_SUPPORT_RECOMMENDED_ARGS={{ atl_jvm_opts }}" atl_startup_exec_options: - "-fg" diff --git a/roles/product_startup/molecule/bitbucket/tests/test_default.py b/roles/product_startup/molecule/bitbucket/tests/test_default.py index d12ab59..14feb7c 100644 --- a/roles/product_startup/molecule/bitbucket/tests/test_default.py +++ b/roles/product_startup/molecule/bitbucket/tests/test_default.py @@ -12,3 +12,6 @@ def test_service_file(host): assert f.contains("^UMask=0027$") assert f.contains("^LimitNOFILE=4096$") assert f.contains("^Environment=BITBUCKET_HOME=/media/atl/bitbucket$") + assert f.contains("^Environment=JVM_MINIMUM_MEMORY=dummy_heap$") + assert f.contains("^Environment=JVM_MAXIMUM_MEMORY=dummy_heap$") + assert f.contains("^Environment=JVM_SUPPORT_RECOMMENDED_ARGS=dummy_opts$") From 6c3e2d7d8362af6067fc3f36c0aa67365f929ad1 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Mon, 8 Jul 2019 12:38:32 +1000 Subject: [PATCH 03/10] DCD-434: Add injection of heap and additional JVM opts into BB environment for node playbook. --- aws_bitbucket_dc_node.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/aws_bitbucket_dc_node.yml b/aws_bitbucket_dc_node.yml index d473ea5..7834e12 100644 --- a/aws_bitbucket_dc_node.yml +++ b/aws_bitbucket_dc_node.yml @@ -17,6 +17,9 @@ - "UMask=0027" - "LimitNOFILE=4096" - "Environment=BITBUCKET_HOME={{ atl_product_home }}" + - "Environment=JVM_MAXIMUM_MEMORY={{ atl_jvm_heap }}" + - "Environment=JVM_MINIMUM_MEMORY={{ atl_jvm_heap }}" + - "Environment=JVM_SUPPORT_RECOMMENDED_ARGS={{ atl_jvm_opts }}" atl_startup_exec_options: - "-fg" - "--no-search" From 59b8903294a8d5057eb9950dbbaac6c4b58e53b1 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Mon, 8 Jul 2019 14:07:18 +1000 Subject: [PATCH 04/10] DCD-434: Ignore changes to installation permissions as we may override the defaults in product_install. --- roles/bitbucket_config/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/bitbucket_config/tasks/main.yml b/roles/bitbucket_config/tasks/main.yml index cf6bcce..4857761 100644 --- a/roles/bitbucket_config/tasks/main.yml +++ b/roles/bitbucket_config/tasks/main.yml @@ -22,3 +22,4 @@ group: "root" mode: "u=rwX,g=rX,o=rX" recurse: true + changed_when: false # For Molecule idempotence check From df5f139086c8b68c29d93d5c10311831ee2488aa Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Mon, 8 Jul 2019 14:41:43 +1000 Subject: [PATCH 05/10] DCD-434: Remove test for psql as we no longer install it by default. --- roles/product_common/molecule/default/tests/test_default.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/product_common/molecule/default/tests/test_default.py b/roles/product_common/molecule/default/tests/test_default.py index 26bdc54..b29ef83 100644 --- a/roles/product_common/molecule/default/tests/test_default.py +++ b/roles/product_common/molecule/default/tests/test_default.py @@ -13,8 +13,7 @@ def test_user_created(host): @pytest.mark.parametrize('exe', [ - '/usr/bin/git', - '/usr/bin/psql' + '/usr/bin/git' ]) def test_package_exes(host, exe): assert host.file(exe).exists From 4b2dfd8f90304f318dfb418333fd38d653391537 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 9 Jul 2019 16:15:45 +1000 Subject: [PATCH 06/10] DCD-418: Limit permissions on the systemd unit and move DB params to the environment. --- roles/product_startup/molecule/default/tests/test_default.py | 3 +++ roles/product_startup/tasks/main.yml | 3 +++ roles/synchrony_config/tasks/main.yml | 2 +- roles/synchrony_config/templates/atl.synchrony.j2 | 5 +++-- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/roles/product_startup/molecule/default/tests/test_default.py b/roles/product_startup/molecule/default/tests/test_default.py index f01d546..3beccfe 100644 --- a/roles/product_startup/molecule/default/tests/test_default.py +++ b/roles/product_startup/molecule/default/tests/test_default.py @@ -9,3 +9,6 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( def test_service_file(host): f = host.file('/etc/systemd/system/jira-software.service') assert f.contains("^ExecStart=/opt/atlassian/jira-software/current/bin/start-jira.sh -fg$") + assert f.user == 'root' + assert f.user == 'root' + assert f.mode == 0o0640 diff --git a/roles/product_startup/tasks/main.yml b/roles/product_startup/tasks/main.yml index 4922627..706bee3 100644 --- a/roles/product_startup/tasks/main.yml +++ b/roles/product_startup/tasks/main.yml @@ -4,6 +4,9 @@ template: src: "product.service.j2" dest: "/etc/systemd/system/{{ atl_systemd_service_name }}" + owner: root + group: root + mode: 0640 notify: - Enable Product - Restart Product diff --git a/roles/synchrony_config/tasks/main.yml b/roles/synchrony_config/tasks/main.yml index 330fe6d..966e84f 100644 --- a/roles/synchrony_config/tasks/main.yml +++ b/roles/synchrony_config/tasks/main.yml @@ -12,4 +12,4 @@ src: "atl.synchrony.j2" dest: "/etc/atl.synchrony" group: "{{ atl_product_user }}" - mode: "0640" + mode: "0640" diff --git a/roles/synchrony_config/templates/atl.synchrony.j2 b/roles/synchrony_config/templates/atl.synchrony.j2 index 96c7ae2..e340fa3 100644 --- a/roles/synchrony_config/templates/atl.synchrony.j2 +++ b/roles/synchrony_config/templates/atl.synchrony.j2 @@ -9,11 +9,12 @@ ATL_SYNCHRONY_JAR_PATH="{{ atl_product_installation_current }}/confluence/WEB-IN AWS_EC2_PRIVATE_IP="{{ atl_local_ipv4 }}" _RUNJAVA="{{ atl_product_installation_current }}/jre/bin/java" +SYNCHRONY_DATABASE_USERNAME="{{ atl_jdbc_user }}" +SYNCHRONY_DATABASE_PASSWORD="{{ atl_jdbc_password }}" + ATL_SYNCHRONY_JVM_PROPERTIES="{{ atl_synchrony_stack_space }} {{ atl_synchrony_memory }} \ -Dsynchrony.cluster.impl=hazelcast-btf \ -Dsynchrony.database.url={{ atl_jdbc_url }} \ - -Dsynchrony.database.username={{ atl_jdbc_user }} \ - -Dsynchrony.database.password={{ atl_jdbc_password }} \ -Dsynchrony.bind={{ atl_local_ipv4 }} \ -Dsynchrony.cluster.bind={{ atl_local_ipv4 }} \ -Dcluster.interfaces={{ atl_local_ipv4 }} \ From 82d94c3a5bdda9eaf9b956135b445b22f49f3586 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 10 Jul 2019 09:21:28 +1000 Subject: [PATCH 07/10] DCD-418: Move JWT keys to environment vars. --- roles/synchrony_config/files/start-synchrony | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/roles/synchrony_config/files/start-synchrony b/roles/synchrony_config/files/start-synchrony index 5029e66..e52555d 100755 --- a/roles/synchrony_config/files/start-synchrony +++ b/roles/synchrony_config/files/start-synchrony @@ -55,9 +55,11 @@ function waitForConfluenceConfigInSharedHome() { waitForConfluenceConfigInSharedHome +# Additional settings are in /etc/atl.synchrony +export SYNCHRONY_JWT_PRIVATE_KEY +export SYNCHRONY_JWT_PUBLIC_KEY + exec ${_RUNJAVA} \ -classpath ${SYNCHRONY_CLASSPATH} \ ${ATL_SYNCHRONY_JVM_PROPERTIES} \ - -Djwt.private.key=${SYNCHRONY_JWT_PRIVATE_KEY} \ - -Djwt.public.key=${SYNCHRONY_JWT_PUBLIC_KEY} \ synchrony.core sql From 5dc978afb0698f5934d51189582a860784e92247 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 10 Jul 2019 16:52:18 +1000 Subject: [PATCH 08/10] DCD-418: Fix test. --- roles/product_startup/molecule/default/tests/test_default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/product_startup/molecule/default/tests/test_default.py b/roles/product_startup/molecule/default/tests/test_default.py index 3beccfe..8210664 100644 --- a/roles/product_startup/molecule/default/tests/test_default.py +++ b/roles/product_startup/molecule/default/tests/test_default.py @@ -10,5 +10,5 @@ def test_service_file(host): f = host.file('/etc/systemd/system/jira-software.service') assert f.contains("^ExecStart=/opt/atlassian/jira-software/current/bin/start-jira.sh -fg$") assert f.user == 'root' - assert f.user == 'root' + assert f.group == 'root' assert f.mode == 0o0640 From 5872eba6c81e91f4f162d865d381050c7316d3d9 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 11 Jul 2019 10:17:07 +1000 Subject: [PATCH 09/10] The S3 Jira latest file does not appear to be updating, so use marketplace. --- .../molecule/default/tests/test_default.py | 13 ++++++++----- .../jira_software_latest/tests/test_default.py | 12 +++++++----- .../jira_version_latest/tests/test_default.py | 12 +++++++----- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/roles/product_install/molecule/default/tests/test_default.py b/roles/product_install/molecule/default/tests/test_default.py index 82f1205..18c41a2 100644 --- a/roles/product_install/molecule/default/tests/test_default.py +++ b/roles/product_install/molecule/default/tests/test_default.py @@ -2,6 +2,7 @@ import os from six.moves import urllib import testinfra.utils.ansible_runner +import json testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') @@ -11,14 +12,16 @@ def test_version_file_is_latest(host): verfile = host.file('/media/atl/jira/shared/jira-core.version') assert verfile.exists - upstream_fd = urllib.request.urlopen("https://s3.amazonaws.com/atlassian-software/releases/jira-core/latest") - upstream = upstream_fd.read() + upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/jira/versions/latest") + upstream_json = json.load(upstream_fd) + upstream = upstream_json['version'] - assert verfile.content.decode("UTF-8").strip() == upstream.decode("UTF-8").strip() + assert verfile.content.decode("UTF-8").strip() == upstream.strip() def test_latest_is_downloaded(host): - upstream_fd = urllib.request.urlopen("https://s3.amazonaws.com/atlassian-software/releases/jira-core/latest") - upstream = upstream_fd.read().decode("UTF-8").strip() + upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/jira/versions/latest") + upstream_json = json.load(upstream_fd) + upstream = upstream_json['version'] installer = host.file('/opt/atlassian/tmp/jira-core.'+upstream+'.bin') assert installer.exists diff --git a/roles/product_install/molecule/jira_software_latest/tests/test_default.py b/roles/product_install/molecule/jira_software_latest/tests/test_default.py index 05e48cb..df69114 100644 --- a/roles/product_install/molecule/jira_software_latest/tests/test_default.py +++ b/roles/product_install/molecule/jira_software_latest/tests/test_default.py @@ -23,14 +23,16 @@ def test_version_file_is_latest(host): verfile = host.file('/media/atl/jira/shared/jira-software.version') assert verfile.exists - upstream_fd = urllib.request.urlopen("https://s3.amazonaws.com/atlassian-software/releases/jira-software/latest") - upstream = upstream_fd.read() + upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/jira/versions/latest") + upstream_json = json.load(upstream_fd) + upstream = upstream_json['version'] - assert verfile.content.decode("UTF-8").strip() == upstream.decode("UTF-8").strip() + assert verfile.content.decode("UTF-8").strip() == upstream.strip() def test_latest_is_downloaded(host): - upstream_fd = urllib.request.urlopen("https://s3.amazonaws.com/atlassian-software/releases/jira-software/latest") - upstream = upstream_fd.read().decode("UTF-8").strip() + upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/jira/versions/latest") + upstream_json = json.load(upstream_fd) + upstream = upstream_json['version'] installer = host.file('/opt/atlassian/tmp/jira-software.'+upstream+'.bin') assert installer.exists diff --git a/roles/product_install/molecule/jira_version_latest/tests/test_default.py b/roles/product_install/molecule/jira_version_latest/tests/test_default.py index 82f1205..e66a346 100644 --- a/roles/product_install/molecule/jira_version_latest/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_latest/tests/test_default.py @@ -11,14 +11,16 @@ def test_version_file_is_latest(host): verfile = host.file('/media/atl/jira/shared/jira-core.version') assert verfile.exists - upstream_fd = urllib.request.urlopen("https://s3.amazonaws.com/atlassian-software/releases/jira-core/latest") - upstream = upstream_fd.read() + upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/jira/versions/latest") + upstream_json = json.load(upstream_fd) + upstream = upstream_json['version'] - assert verfile.content.decode("UTF-8").strip() == upstream.decode("UTF-8").strip() + assert verfile.content.decode("UTF-8").strip() == upstream.strip() def test_latest_is_downloaded(host): - upstream_fd = urllib.request.urlopen("https://s3.amazonaws.com/atlassian-software/releases/jira-core/latest") - upstream = upstream_fd.read().decode("UTF-8").strip() + upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/jira/versions/latest") + upstream_json = json.load(upstream_fd) + upstream = upstream_json['version'] installer = host.file('/opt/atlassian/tmp/jira-core.'+upstream+'.bin') assert installer.exists From 05a8b1b3fa95bf6fc4f72de5fa801bc787f7f024 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 11 Jul 2019 11:27:23 +1000 Subject: [PATCH 10/10] Add some missing imports. --- .../molecule/jira_software_latest/tests/test_default.py | 1 + .../molecule/jira_version_latest/tests/test_default.py | 1 + 2 files changed, 2 insertions(+) diff --git a/roles/product_install/molecule/jira_software_latest/tests/test_default.py b/roles/product_install/molecule/jira_software_latest/tests/test_default.py index df69114..ff6fa8b 100644 --- a/roles/product_install/molecule/jira_software_latest/tests/test_default.py +++ b/roles/product_install/molecule/jira_software_latest/tests/test_default.py @@ -2,6 +2,7 @@ import os from six.moves import urllib import testinfra.utils.ansible_runner +import json testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') diff --git a/roles/product_install/molecule/jira_version_latest/tests/test_default.py b/roles/product_install/molecule/jira_version_latest/tests/test_default.py index e66a346..18c41a2 100644 --- a/roles/product_install/molecule/jira_version_latest/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_latest/tests/test_default.py @@ -2,6 +2,7 @@ import os from six.moves import urllib import testinfra.utils.ansible_runner +import json testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')