From 2a67eae9c87b7120fe65e896a432d8f33a41dea9 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Thu, 17 Oct 2019 14:51:32 +1100 Subject: [PATCH 01/49] ITOPSENG-101 jsd_as_obr support --- aws_jira_dc_node.yml | 1 + .../tasks/jira-servicedesk_as_obr.yml | 30 +++++++++++++++++++ roles/product_install/tasks/main.yml | 4 +++ 3 files changed, 35 insertions(+) create mode 100644 roles/product_install/tasks/jira-servicedesk_as_obr.yml diff --git a/aws_jira_dc_node.yml b/aws_jira_dc_node.yml index f0af8a4..08d9657 100644 --- a/aws_jira_dc_node.yml +++ b/aws_jira_dc_node.yml @@ -7,6 +7,7 @@ atl_product_family: "jira" atl_product_user: "jira" atl_product_edition: "jira-{{ lookup('env', 'ATL_PRODUCT_EDITION') | lower }}" + atl_install_jsd_as_obr: "{{ lookup('env', 'ATL_JSD_ASOBR')" atl_systemd_service_name: "jira.service" atl_startup_systemd_params: diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml new file mode 100644 index 0000000..012c6d9 --- /dev/null +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -0,0 +1,30 @@ +--- + +- name: Get the installer product version info + uri: + url="https://marketplace.atlassian.com/rest/2/products/key/jira-software/versions/name/{{ atl_product_version }}" + return_content=yes + register: atl_product_version_info + +- name: lets grab the build number + debug: + msg="buildNumber={{ atl_product_version_info.json.buildNumber }}" + +- name: Get the JSD build version info + uri: + url="https://marketplace.atlassian.com/rest/2/products/key/jira-servicedesk/versions/latest?application=jira&applicationBuild={{ atl_product_version_info.json.buildNumber }}" + return_content=yes + register: atl_jsd_build_info + +- name: lets grab the obr binary href + debug: + msg="obr_ref={{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}" + +- name: grab the jsd obr + get_url: + url: "{{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}" + dest: "{{ atl_installer_temp }}" + +- name: override the atl_product_edition to jira-software + set_fact: + atl_product_edition: "jira-software" diff --git a/roles/product_install/tasks/main.yml b/roles/product_install/tasks/main.yml index 77371fa..b685a28 100644 --- a/roles/product_install/tasks/main.yml +++ b/roles/product_install/tasks/main.yml @@ -146,3 +146,7 @@ dest: "{{ atl_product_installation_current }}" state: link force: true + +- name: Include if jsd is requested to be installed from OBR + include_tasks: "jira-servicedesk_as_obr.yml" + when: atl_install_jsd_as_obr From 19329447a800d424ec7c6ec0dcd325023799b881 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 18 Oct 2019 13:57:13 +1100 Subject: [PATCH 02/49] ITOPSENG-101 logic to fetch and unpack jsd as an obr --- .../tasks/jira-servicedesk_as_obr.yml | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index 012c6d9..efb1711 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -6,7 +6,7 @@ return_content=yes register: atl_product_version_info -- name: lets grab the build number +- name: Show the returned build number debug: msg="buildNumber={{ atl_product_version_info.json.buildNumber }}" @@ -16,15 +16,33 @@ return_content=yes register: atl_jsd_build_info -- name: lets grab the obr binary href +- name: Show the returned obr binary href debug: msg="obr_ref={{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}" -- name: grab the jsd obr +- name: Fetch the jsd obr get_url: url: "{{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}" dest: "{{ atl_installer_temp }}" + register: jsdobr -- name: override the atl_product_edition to jira-software - set_fact: - atl_product_edition: "jira-software" +- name: check the name of the downloaded file + debug: + msg="{{ jsdobr.dest }}" + +- name: Copy the obr to shared_home + copy: + src: "{{ jsdobr.dest }}" + dest: "{{ atl_product_version_cache_dir }}" + remote_src: true + +- name: Unpack the obr into the installed-plugins dir + unarchive: + remote_src: true + src: "{{ jsdobr.dest }}" + dest: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins" + owner: "{{ atl_product_user }}" + group: "{{ atl_product_user }}" + mode: 0750 + extra_opts: + - "-j" From 9c032469f5a90283eb19bb2c046eba6b649cb346 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 18 Oct 2019 14:24:58 +1100 Subject: [PATCH 03/49] ITOPSENG-101 split some strings across lines to pass line length linting --- roles/product_install/tasks/jira-servicedesk_as_obr.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index efb1711..cbbb753 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -2,8 +2,8 @@ - name: Get the installer product version info uri: - url="https://marketplace.atlassian.com/rest/2/products/key/jira-software/versions/name/{{ atl_product_version }}" - return_content=yes + url: "https://marketplace.atlassian.com/rest/2/products/key/jira-software/versions/name/{{ atl_product_version }}" + return_content: yes register: atl_product_version_info - name: Show the returned build number @@ -12,8 +12,9 @@ - name: Get the JSD build version info uri: - url="https://marketplace.atlassian.com/rest/2/products/key/jira-servicedesk/versions/latest?application=jira&applicationBuild={{ atl_product_version_info.json.buildNumber }}" - return_content=yes + url: "https://marketplace.atlassian.com/rest/2/products/key/jira-servicedesk/versions/latest?application=\ + jira&applicationBuild={{ atl_product_version_info.json.buildNumber }}" + return_content: yes register: atl_jsd_build_info - name: Show the returned obr binary href From eae65657cebb2bae0b52b9191cc4d94dafaaab88 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 18 Oct 2019 14:44:43 +1100 Subject: [PATCH 04/49] ITOPSENG-101 fixed missing braces --- aws_jira_dc_node.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_jira_dc_node.yml b/aws_jira_dc_node.yml index 08d9657..c349bb8 100644 --- a/aws_jira_dc_node.yml +++ b/aws_jira_dc_node.yml @@ -7,7 +7,7 @@ atl_product_family: "jira" atl_product_user: "jira" atl_product_edition: "jira-{{ lookup('env', 'ATL_PRODUCT_EDITION') | lower }}" - atl_install_jsd_as_obr: "{{ lookup('env', 'ATL_JSD_ASOBR')" + atl_install_jsd_as_obr: "{{ lookup('env', 'ATL_JSD_ASOBR') }}" atl_systemd_service_name: "jira.service" atl_startup_systemd_params: From 0245f9059d8f53340618bcd88e48cfe14e79f844 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 18 Oct 2019 14:59:55 +1100 Subject: [PATCH 05/49] ITOPSENG-101 ensure installed-plugins exists --- roles/product_install/tasks/jira-servicedesk_as_obr.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index cbbb753..7f9702d 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -37,6 +37,14 @@ dest: "{{ atl_product_version_cache_dir }}" remote_src: true +- name: Ensure instaled-plugins dir exists + file: + path: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins" + state: directory + mode: 0750 + owner: "{{ atl_product_user }}" + group: "{{ atl_product_user }}" + - name: Unpack the obr into the installed-plugins dir unarchive: remote_src: true From 3ae840cf13ee4109462adba3ee566b11ec45fb75 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 18 Oct 2019 18:11:00 +1100 Subject: [PATCH 06/49] ITOPSENG-101 work around ansible not handling unzip -j --- .../tasks/jira-servicedesk_as_obr.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index 7f9702d..b8c0369 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -53,5 +53,13 @@ owner: "{{ atl_product_user }}" group: "{{ atl_product_user }}" mode: 0750 - extra_opts: - - "-j" + # Ansible unarchive task validation does not work with the -j flag so we need to achieve the same result in 2 steps + # extra_opts: + # - "-j" + +- name: Copy JSD dependency jars into the installed-plugins dir + copy: + src: "{{ item }}" + dest: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins" + remote_src: true + with_fileglob: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins/dependencies/*.jar" From d6a9a2c8415a6678709cfb83b21b8e63f6c5c643 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Mon, 21 Oct 2019 12:54:39 +1100 Subject: [PATCH 07/49] ITOPSENG-101 ensure the obr var exists and is true before including the associated tasks --- roles/product_install/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/product_install/tasks/main.yml b/roles/product_install/tasks/main.yml index b685a28..d84895d 100644 --- a/roles/product_install/tasks/main.yml +++ b/roles/product_install/tasks/main.yml @@ -149,4 +149,4 @@ - name: Include if jsd is requested to be installed from OBR include_tasks: "jira-servicedesk_as_obr.yml" - when: atl_install_jsd_as_obr + when: atl_install_jsd_as_obr.exists and atl_install_jsd_as_obr From eec68f6357cda6031216a0965d830e75f57617ef Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Mon, 21 Oct 2019 13:06:46 +1100 Subject: [PATCH 08/49] ITOPSENG-101 ensure the obr var exists and is true before including the associated tasks --- roles/product_install/tasks/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/roles/product_install/tasks/main.yml b/roles/product_install/tasks/main.yml index d84895d..6cea478 100644 --- a/roles/product_install/tasks/main.yml +++ b/roles/product_install/tasks/main.yml @@ -149,4 +149,6 @@ - name: Include if jsd is requested to be installed from OBR include_tasks: "jira-servicedesk_as_obr.yml" - when: atl_install_jsd_as_obr.exists and atl_install_jsd_as_obr + when: + - atl_install_jsd_as_obr is defined + - atl_install_jsd_as_obr From f8e8f94946a811fa4a9d7b592bfc8ca638efb8e8 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Mon, 21 Oct 2019 14:22:52 +1100 Subject: [PATCH 09/49] ITOPSENG-101 cleanup for PR comments --- roles/product_install/defaults/main.yml | 1 + roles/product_install/tasks/jira-servicedesk_as_obr.yml | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/roles/product_install/defaults/main.yml b/roles/product_install/defaults/main.yml index 211e76e..4b62635 100644 --- a/roles/product_install/defaults/main.yml +++ b/roles/product_install/defaults/main.yml @@ -23,6 +23,7 @@ atl_product_download: "{{ atl_installer_temp }}/{{ atl_product_download_filename atl_product_varfile: "{{ atl_installer_temp }}/{{ atl_product_family }}.varfile" atl_marketplace_base: "https://marketplace.atlassian.com" +atl_mpac_products: "https://marketplace.atlassian.com/rest/2/products" atl_servicedesk_latest_url: "https://marketplace.atlassian.com/rest/2/products/key/jira-servicedesk/versions/latest" atl_servicedesk_versioned_url: "https://marketplace.atlassian.com/rest/2/products/key/jira-servicedesk/versions/name/{{ atl_product_version }}" atl_servicedesk_url_map: diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index b8c0369..ca013e6 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -2,7 +2,7 @@ - name: Get the installer product version info uri: - url: "https://marketplace.atlassian.com/rest/2/products/key/jira-software/versions/name/{{ atl_product_version }}" + url: "{{ atl_mpac_products }}/key/jira-software/versions/name/{{ atl_product_version }}" return_content: yes register: atl_product_version_info @@ -12,7 +12,7 @@ - name: Get the JSD build version info uri: - url: "https://marketplace.atlassian.com/rest/2/products/key/jira-servicedesk/versions/latest?application=\ + url: "{{ atl_mpac_products }}/key/jira-servicedesk/versions/latest?application=\ jira&applicationBuild={{ atl_product_version_info.json.buildNumber }}" return_content: yes register: atl_jsd_build_info @@ -53,9 +53,6 @@ owner: "{{ atl_product_user }}" group: "{{ atl_product_user }}" mode: 0750 - # Ansible unarchive task validation does not work with the -j flag so we need to achieve the same result in 2 steps - # extra_opts: - # - "-j" - name: Copy JSD dependency jars into the installed-plugins dir copy: From 353480916bcc821371f9d4071ac764dd2f82ff91 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 8 Nov 2019 11:28:45 +1100 Subject: [PATCH 10/49] ITOPSENG-101 add the locking logic for storing in shared_home --- .../tests/test_default.py | 6 +- .../tasks/jira-servicedesk_as_obr.yml | 139 ++++++++++++++++-- 2 files changed, 128 insertions(+), 17 deletions(-) diff --git a/roles/product_install/molecule/jira_version_from_file/tests/test_default.py b/roles/product_install/molecule/jira_version_from_file/tests/test_default.py index b8a1966..8b75761 100644 --- a/roles/product_install/molecule/jira_version_from_file/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_from_file/tests/test_default.py @@ -11,15 +11,15 @@ def test_version_is_correct(host): verfile = host.file('/media/atl/jira/shared/jira-core.version') assert verfile.exists - assert verfile.content.decode("UTF-8").strip() == "7.9.0" + assert verfile.content.decode("UTF-8").strip() == "7.13.1" def test_is_downloaded(host): - installer = host.file('/media/atl/jira/shared/downloads/jira-core.7.9.0-x64.bin') + installer = host.file('/media/atl/jira/shared/downloads/jira-core.7.13.1-x64.bin') assert installer.exists assert installer.user == 'root' def test_completed_lockfile(host): - lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.7.9.0-x64.bin_completed') + lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.7.13.1-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index ca013e6..f6b62db 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -21,21 +21,132 @@ debug: msg="obr_ref={{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}" -- name: Fetch the jsd obr - get_url: - url: "{{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}" - dest: "{{ atl_installer_temp }}" - register: jsdobr - -- name: check the name of the downloaded file +- name: how about getting the obr filename debug: - msg="{{ jsdobr.dest }}" + msg="obr_name=jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr" -- name: Copy the obr to shared_home - copy: - src: "{{ jsdobr.dest }}" - dest: "{{ atl_product_version_cache_dir }}" - remote_src: true +- name: is shared_home set ? + debug: + msg="atl_product_home_shared_download_dir={{ atl_product_home_shared_download_dir }}" +# For the first run a temp obr should be downloaded but moved to +# shared home to ensure all subsequent nodes have access +# to the same specific version binary. +# To prevent a race condition with multiple downloads at the same time +# a directory is used as a lockfile (atomic operation) when moving obr. + +- name: Set assumptions to avoid race condition + set_fact: + download_obr: true + move_obr: false + atl_obr_download_href: "{{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}" + atl_obr_filename: "jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr" + atl_obr_temp_download: "{{ atl_installer_temp }}/{{ atl_obr_filename }}" + atl_obr_shared_download: "{{ atl_product_home_shared_download_dir }}/{{ atl_obr_filename }}" + atl_obr_moving_lock: "{{{ atl_product_home_shared_download_dir }}/{{ atl_obr_filename }}_moving" + atl_obr_completed_lock: "{{ atl_product_home_shared_download_dir }}/{{ atl_obr_filename }}_completed" + +# Check for pre-downloaded obr on shared_home and completed lock dir. +- name: Check for completed lock directory + stat: + path: "{{ atl_obr_completed_lock }}" + register: completed_lock + +- name: Check for obr in home_shared + stat: + path: "{{ atl_obr_shared_download }}" + register: home_shared_download + +# If obr exists and lockdir exists use this obr instead +- name: Check lock directory and obr exists on shared_home + set_fact: + download_obr: false + atl_obr_download: "{{ atl_obr_shared_download }}" + when: + - home_shared_download.stat.exists + - completed_lock.stat.isdir is defined + - completed_lock.stat.isdir + +# Fetch obr if required +- name: download_obr is true so fetch and do all the things + block: + + # Fetch obr and copy to temp + - name: Fetch obr + get_url: + url: "{{ atl_obr_download_href }}" + dest: "{{ atl_obr_temp_download }}" + mode: 0755 + force: false + register: atl_obr_completed + + # If obr was fetched make the lock directory + - name: Create moving_lock. + file: + path: "{{ atl_obr_moving_lock }}" + state: directory + when: + - atl_obr_completed is succeeded + register: moving_lock_created + + # Directory lock was created by this run? + # If so, then set a fact intending to move obr + - name: Move obr Scenario - lock created by this run + set_fact: + move_obr: true + when: + - moving_lock_created is succeeded + - moving_lock_created.changed + # Otherwise directory lock was either already created or + # could not be created. Fall back is to continue and install from temp + + when: download_obr + +# If the intention is to move obr to home_shared +- name: Move obr to home_shared + block: + + - name: Copy temp installer to home_shared + copy: + src: "{{ atl_obr_temp_download }}" + dest: "{{ atl_obr_shared_download }}" + remote_src: true + when: + - moving_lock_created is succeeded + - moving_lock_created.changed + register: copied + + - name: Create completed_lock once obr downloaded and copied + file: + path: "{{ atl_obr_shared_completed_lock }}" + state: directory + when: copied is succeeded + register: completed_lock_created + + - name: Remove moving_lock to show that obr is completed + file: + path: "{{ atl_obr_shared_moving_lock }}" + state: absent + when: + - completed_lock_created is succeeded + - copied is succeeded + register: moving_lock_removed + + - name: Delete old temp installer + file: + path: "{{ atl_obr_temp_download }}" + state: absent + when: moving_lock_removed is succeeded + register: temp_deleted + + - name: Set install to home_shared location + set_fact: + atl_obr_download: "{{ atl_obr_home_shared_download }}" + when: temp_deleted is succeeded + + when: move_obr + +# At this point the binary is in {{ atl_obr_download }} +# (which is either on home_shared or temp) - name: Ensure instaled-plugins dir exists file: @@ -48,7 +159,7 @@ - name: Unpack the obr into the installed-plugins dir unarchive: remote_src: true - src: "{{ jsdobr.dest }}" + src: "{{ atl_obr_download }}" dest: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins" owner: "{{ atl_product_user }}" group: "{{ atl_product_user }}" From 4686fd3b33908d3901aed543b103122ef1b00bb8 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 8 Nov 2019 11:32:56 +1100 Subject: [PATCH 11/49] ITOPSENG-101 move downloads dir directly under shared mountpoint so that it is not backed up by backmac --- roles/product_install/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/product_install/defaults/main.yml b/roles/product_install/defaults/main.yml index 67b01a6..6a9940e 100644 --- a/roles/product_install/defaults/main.yml +++ b/roles/product_install/defaults/main.yml @@ -22,7 +22,7 @@ atl_product_download_filename: "{{ atl_download_edition | default(atl_product_ed atl_product_temp_download: "{{ atl_installer_temp }}/{{ atl_product_download_filename }}" atl_product_varfile: "{{ atl_installer_temp }}/{{ atl_product_family }}.varfile" -atl_product_home_shared_download_dir: "{{ atl_product_home_shared }}/downloads" +atl_product_home_shared_download_dir: "{{ atl_shared_mountpoint }}/downloads" atl_product_home_shared_download: "{{ atl_product_home_shared_download_dir }}/{{ atl_product_download_filename }}" atl_product_home_shared_moving_lock: "{{ atl_product_home_shared_download }}_moving" atl_product_home_shared_completed_lock: "{{ atl_product_home_shared_download }}_completed" From 6f040584d7eebf4de1c2ed22897441b98cf8c1f5 Mon Sep 17 00:00:00 2001 From: bmeehan Date: Fri, 8 Nov 2019 01:18:26 +0000 Subject: [PATCH 12/49] ITOPSENG-101 working obr storage logic --- .../tasks/jira-servicedesk_as_obr.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index f6b62db..6e7d0f1 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -40,10 +40,10 @@ move_obr: false atl_obr_download_href: "{{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}" atl_obr_filename: "jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr" - atl_obr_temp_download: "{{ atl_installer_temp }}/{{ atl_obr_filename }}" - atl_obr_shared_download: "{{ atl_product_home_shared_download_dir }}/{{ atl_obr_filename }}" - atl_obr_moving_lock: "{{{ atl_product_home_shared_download_dir }}/{{ atl_obr_filename }}_moving" - atl_obr_completed_lock: "{{ atl_product_home_shared_download_dir }}/{{ atl_obr_filename }}_completed" + atl_obr_download: "{{ atl_installer_temp }}/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr" + atl_obr_shared_download: "{{ atl_product_home_shared_download_dir }}/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr" + atl_obr_moving_lock: "{{ atl_product_home_shared_download_dir }}/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr_moving" + atl_obr_completed_lock: "{{ atl_product_home_shared_download_dir }}/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr_completed" # Check for pre-downloaded obr on shared_home and completed lock dir. - name: Check for completed lock directory @@ -74,7 +74,7 @@ - name: Fetch obr get_url: url: "{{ atl_obr_download_href }}" - dest: "{{ atl_obr_temp_download }}" + dest: "{{ atl_obr_download }}" mode: 0755 force: false register: atl_obr_completed @@ -107,7 +107,7 @@ - name: Copy temp installer to home_shared copy: - src: "{{ atl_obr_temp_download }}" + src: "{{ atl_obr_download }}" dest: "{{ atl_obr_shared_download }}" remote_src: true when: @@ -117,14 +117,14 @@ - name: Create completed_lock once obr downloaded and copied file: - path: "{{ atl_obr_shared_completed_lock }}" + path: "{{ atl_obr_completed_lock }}" state: directory when: copied is succeeded register: completed_lock_created - name: Remove moving_lock to show that obr is completed file: - path: "{{ atl_obr_shared_moving_lock }}" + path: "{{ atl_obr_moving_lock }}" state: absent when: - completed_lock_created is succeeded @@ -133,7 +133,7 @@ - name: Delete old temp installer file: - path: "{{ atl_obr_temp_download }}" + path: "{{ atl_obr_download }}" state: absent when: moving_lock_removed is succeeded register: temp_deleted From ced8608b02341e389be88a4108d335548e5979de Mon Sep 17 00:00:00 2001 From: bmeehan Date: Fri, 8 Nov 2019 02:36:20 +0000 Subject: [PATCH 13/49] ITOPSENG-101 corrected obr shared_home location var --- roles/product_install/tasks/jira-servicedesk_as_obr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index 6e7d0f1..448bb3d 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -140,7 +140,7 @@ - name: Set install to home_shared location set_fact: - atl_obr_download: "{{ atl_obr_home_shared_download }}" + atl_obr_download: "{{ atl_obr_shared_download }}" when: temp_deleted is succeeded when: move_obr From 48904ca0845f8ebe607f841c20b0ca2ce5022c8a Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 8 Nov 2019 14:45:29 +1100 Subject: [PATCH 14/49] ITOPSENG-101 noop the extra_tasks for jira-all --- roles/product_install/tasks/jira-all_extra_tasks.yml | 1 + 1 file changed, 1 insertion(+) create mode 120000 roles/product_install/tasks/jira-all_extra_tasks.yml diff --git a/roles/product_install/tasks/jira-all_extra_tasks.yml b/roles/product_install/tasks/jira-all_extra_tasks.yml new file mode 120000 index 0000000..55832eb --- /dev/null +++ b/roles/product_install/tasks/jira-all_extra_tasks.yml @@ -0,0 +1 @@ +no_op.yml \ No newline at end of file From 9d7626a14a3695a84e0f66ef45cdbf6c17c69408 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 8 Nov 2019 15:13:34 +1100 Subject: [PATCH 15/49] ITOPSENG-101 initial molecule tests commit --- roles/product_install/defaults/main.yml | 2 +- .../molecule/jira_all/Dockerfile.j2 | 14 ++++++ .../molecule/jira_all/molecule.yml | 32 ++++++++++++ .../molecule/jira_all/playbook.yml | 28 +++++++++++ .../molecule/jira_all/tests/test_default.py | 49 +++++++++++++++++++ .../jira_version_latest/tests/test_default.py | 4 +- 6 files changed, 126 insertions(+), 3 deletions(-) create mode 100644 roles/product_install/molecule/jira_all/Dockerfile.j2 create mode 100644 roles/product_install/molecule/jira_all/molecule.yml create mode 100644 roles/product_install/molecule/jira_all/playbook.yml create mode 100644 roles/product_install/molecule/jira_all/tests/test_default.py diff --git a/roles/product_install/defaults/main.yml b/roles/product_install/defaults/main.yml index 6a9940e..7875e1c 100644 --- a/roles/product_install/defaults/main.yml +++ b/roles/product_install/defaults/main.yml @@ -15,7 +15,7 @@ atl_download_format_suffix_map: atl_download_suffix: "{{ atl_download_format_suffix_map[atl_download_format] }}" atl_release_base_url: "https://product-downloads.atlassian.com/software" -atl_product_base_url: "{{ atl_release_base_url }}/{{ atl_product_family }}/downloads" +atl_product_base_url: "{{ atl_release_base_url }}/{{ atl_product_family }}/downloads/binary" atl_product_download_url: "{{ atl_product_base_url }}/atlassian-{{ atl_download_edition | default(atl_product_edition) }}-{{ atl_product_version }}{{ atl_download_suffix }}" atl_product_download_filename: "{{ atl_download_edition | default(atl_product_edition) }}.{{ atl_product_version }}{{ atl_download_suffix }}" diff --git a/roles/product_install/molecule/jira_all/Dockerfile.j2 b/roles/product_install/molecule/jira_all/Dockerfile.j2 new file mode 100644 index 0000000..e6aa95d --- /dev/null +++ b/roles/product_install/molecule/jira_all/Dockerfile.j2 @@ -0,0 +1,14 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi diff --git a/roles/product_install/molecule/jira_all/molecule.yml b/roles/product_install/molecule/jira_all/molecule.yml new file mode 100644 index 0000000..46049f8 --- /dev/null +++ b/roles/product_install/molecule/jira_all/molecule.yml @@ -0,0 +1,32 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: amazon_linux2 + image: amazonlinux:2 + groups: + - aws_node_local + - name: ubuntu_lts + image: ubuntu:bionic + groups: + - aws_node_local +provisioner: + name: ansible + options: + skip-tags: runtime_pkg + lint: + name: ansible-lint + inventory: + links: + group_vars: ../../../../group_vars/ +verifier: + name: testinfra + additional_files_or_dirs: + - ../../resources/tests/test_*.py + lint: + name: flake8 + enabled: false diff --git a/roles/product_install/molecule/jira_all/playbook.yml b/roles/product_install/molecule/jira_all/playbook.yml new file mode 100644 index 0000000..4da7090 --- /dev/null +++ b/roles/product_install/molecule/jira_all/playbook.yml @@ -0,0 +1,28 @@ +--- +- name: Converge + hosts: all + vars: + atl_product_family: "jira" + atl_product_edition: "jira-{{ lookup('env', 'ATL_PRODUCT_EDITION') | lower }}" + atl_product_user: "jira" + atl_product_version: "7.13.1" + atl_install_jsd_as_obr: true + atl_systemd_service_name: "jira.service" + atl_jdbc_encoding: 'UNICODE' + atl_jdbc_collation: 'C' + atl_jdbc_ctype: 'C' + atl_jdbc_template: 'template0' + pre_tasks: + - name: Create cache dir + file: + path: '/media/atl/jira/shared/' + state: directory + - name: Seed version + copy: + dest: '/media/atl/jira/shared/jira-core.version' + content: "7.13.1" + force: false # For idempotency check + roles: + - role: linux_common + - role: product_common + - role: product_install diff --git a/roles/product_install/molecule/jira_all/tests/test_default.py b/roles/product_install/molecule/jira_all/tests/test_default.py new file mode 100644 index 0000000..dd315e4 --- /dev/null +++ b/roles/product_install/molecule/jira_all/tests/test_default.py @@ -0,0 +1,49 @@ +import os +from six.moves import urllib + +import testinfra.utils.ansible_runner +import json + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +def test_version_is_correct(host): + verfile = host.file('/media/atl/jira/shared/jira-core.version') + assert verfile.exists + + assert verfile.content.decode("UTF-8").strip() == "7.13.1" + +def test_is_downloaded(host): + installer = host.file('/media/atl/downloads/jira-core.7.13.1-x64.bin') + assert installer.exists + assert installer.user == 'root' + +def test_completed_lockfile(host): + lockfile = host.file('/media/atl/downloads/jira-core.7.13.1-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' + +def test_is_unpacked(host): + installer = host.file('/opt/atlassian/jira-core/7.13.1/atlassian-jira/') + assert installer.exists + assert installer.is_directory + assert installer.user == 'jira' + assert installer.mode == 0o0755 + +def test_obr_is_downloaded(host): + + installer = host.file('/media/atl/downloads/jira-servicedesk-application-3.16.1.obr') + assert installer.exists + assert installer.user == 'root' + +def test_obr_completed_lockfile(host): + lockfile = host.file('/media/atl/downloads/jira-servicedesk-application-3.16.1.obr_completed') + assert lockfile.exists + assert lockfile.user == 'root' + +def test_obr_is_unpacked(host): + jsdjar = host.file('/media/atl/jira/shared/plugins/installed-plugins/jira-servicedesk-application-3.16.1.jar') + assert jsdjar.exists + assert jsdjar.user == 'jira' + assert jsdjar.mode == 0o0750 \ No newline at end of file diff --git a/roles/product_install/molecule/jira_version_latest/tests/test_default.py b/roles/product_install/molecule/jira_version_latest/tests/test_default.py index 930ab59..8e20d76 100644 --- a/roles/product_install/molecule/jira_version_latest/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_latest/tests/test_default.py @@ -23,7 +23,7 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'-x64.bin') + installer = host.file('/media/atl/downloads/jira-core.'+upstream+'-x64.bin') assert installer.exists assert installer.user == 'root' @@ -32,6 +32,6 @@ def test_completed_lockfile(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/jira-core.'+upstream+'-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' \ No newline at end of file From 3b7623faa17e2f0713800316d07b1728c3f15b23 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 8 Nov 2019 15:20:34 +1100 Subject: [PATCH 16/49] ITOPSENG-101 corrected edition --- roles/product_install/molecule/jira_all/playbook.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/product_install/molecule/jira_all/playbook.yml b/roles/product_install/molecule/jira_all/playbook.yml index 4da7090..9bddc9a 100644 --- a/roles/product_install/molecule/jira_all/playbook.yml +++ b/roles/product_install/molecule/jira_all/playbook.yml @@ -3,7 +3,7 @@ hosts: all vars: atl_product_family: "jira" - atl_product_edition: "jira-{{ lookup('env', 'ATL_PRODUCT_EDITION') | lower }}" + atl_product_edition: "jira-software" atl_product_user: "jira" atl_product_version: "7.13.1" atl_install_jsd_as_obr: true From 51907590ced44c4098b7a613e0eef1cd8ec96a87 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 8 Nov 2019 15:34:45 +1100 Subject: [PATCH 17/49] ITOPSENG-101 reverting the binary suffix, as its only for wac content not product-downloads --- roles/product_install/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/product_install/defaults/main.yml b/roles/product_install/defaults/main.yml index 7875e1c..6a9940e 100644 --- a/roles/product_install/defaults/main.yml +++ b/roles/product_install/defaults/main.yml @@ -15,7 +15,7 @@ atl_download_format_suffix_map: atl_download_suffix: "{{ atl_download_format_suffix_map[atl_download_format] }}" atl_release_base_url: "https://product-downloads.atlassian.com/software" -atl_product_base_url: "{{ atl_release_base_url }}/{{ atl_product_family }}/downloads/binary" +atl_product_base_url: "{{ atl_release_base_url }}/{{ atl_product_family }}/downloads" atl_product_download_url: "{{ atl_product_base_url }}/atlassian-{{ atl_download_edition | default(atl_product_edition) }}-{{ atl_product_version }}{{ atl_download_suffix }}" atl_product_download_filename: "{{ atl_download_edition | default(atl_product_edition) }}.{{ atl_product_version }}{{ atl_download_suffix }}" From c25d8da1d587772a38ab388f085ba0d85619b369 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 8 Nov 2019 15:56:46 +1100 Subject: [PATCH 18/49] updated pipelines.yml for new test --- bitbucket-pipelines.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 408b078..c9f6057 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -14,7 +14,7 @@ pipelines: - step: name: Pre Parallelization stage script: - - echo "Running tests in 30 batches" + - echo "Running tests in 31 batches" - step: name: Check if number of batches match actual number of scenarios script: @@ -267,4 +267,12 @@ pipelines: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 30 + - step: + name: Molecule Test Batch - 31 + services: + - docker + script: + - apt-get update && ./bin/install-ansible --dev + - ./bin/run-tests-in-batches --batch 31 + From 1cf02e721fabfa5154b2dbff9582003236ff3941 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 8 Nov 2019 15:59:48 +1100 Subject: [PATCH 19/49] ITOPSENG-101 moved core tests to software --- .../molecule/jira_all/tests/test_default.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/product_install/molecule/jira_all/tests/test_default.py b/roles/product_install/molecule/jira_all/tests/test_default.py index dd315e4..d03878b 100644 --- a/roles/product_install/molecule/jira_all/tests/test_default.py +++ b/roles/product_install/molecule/jira_all/tests/test_default.py @@ -9,23 +9,23 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( def test_version_is_correct(host): - verfile = host.file('/media/atl/jira/shared/jira-core.version') + verfile = host.file('/media/atl/jira/shared/jira-software.version') assert verfile.exists assert verfile.content.decode("UTF-8").strip() == "7.13.1" def test_is_downloaded(host): - installer = host.file('/media/atl/downloads/jira-core.7.13.1-x64.bin') + installer = host.file('/media/atl/downloads/jira-software.7.13.1-x64.bin') assert installer.exists assert installer.user == 'root' def test_completed_lockfile(host): - lockfile = host.file('/media/atl/downloads/jira-core.7.13.1-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/jira-software.7.13.1-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' def test_is_unpacked(host): - installer = host.file('/opt/atlassian/jira-core/7.13.1/atlassian-jira/') + installer = host.file('/opt/atlassian/jira-software/7.13.1/atlassian-jira/') assert installer.exists assert installer.is_directory assert installer.user == 'jira' From 8a8b2a889ddc3f4b86f2848e499e7b004ffb51e1 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Mon, 11 Nov 2019 13:17:15 +1100 Subject: [PATCH 20/49] ITOPSENG-101 simpler work around junk paths in unarchive --- .../molecule/jira_all/tests/test_default.py | 1 - .../tasks/jira-servicedesk_as_obr.yml | 13 ++++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/roles/product_install/molecule/jira_all/tests/test_default.py b/roles/product_install/molecule/jira_all/tests/test_default.py index d03878b..b9522ec 100644 --- a/roles/product_install/molecule/jira_all/tests/test_default.py +++ b/roles/product_install/molecule/jira_all/tests/test_default.py @@ -32,7 +32,6 @@ def test_is_unpacked(host): assert installer.mode == 0o0755 def test_obr_is_downloaded(host): - installer = host.file('/media/atl/downloads/jira-servicedesk-application-3.16.1.obr') assert installer.exists assert installer.user == 'root' diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index 448bb3d..bf974ea 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -156,18 +156,17 @@ owner: "{{ atl_product_user }}" group: "{{ atl_product_user }}" +# Note as ansible unarchive cant handle "-j junk paths" we need to ignore errors to bypass the path verify - name: Unpack the obr into the installed-plugins dir unarchive: remote_src: true src: "{{ atl_obr_download }}" dest: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins" + creates: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.jar" + extra_opts: + - -j + - -n owner: "{{ atl_product_user }}" group: "{{ atl_product_user }}" mode: 0750 - -- name: Copy JSD dependency jars into the installed-plugins dir - copy: - src: "{{ item }}" - dest: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins" - remote_src: true - with_fileglob: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins/dependencies/*.jar" + ignore_errors: yes \ No newline at end of file From 49797ab64557cac17106496194208daa918b0dbf Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Mon, 11 Nov 2019 13:28:51 +1100 Subject: [PATCH 21/49] ITOPSENG-101 newline at end of file --- roles/product_install/tasks/jira-servicedesk_as_obr.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index bf974ea..4406f2c 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -169,4 +169,5 @@ owner: "{{ atl_product_user }}" group: "{{ atl_product_user }}" mode: 0750 - ignore_errors: yes \ No newline at end of file + ignore_errors: yes + \ No newline at end of file From c307d247274449f3d453cbe295ead1816033d64e Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Mon, 11 Nov 2019 13:39:09 +1100 Subject: [PATCH 22/49] ITOPSENG-101 fix trailing whitespace --- roles/product_install/tasks/jira-servicedesk_as_obr.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index 4406f2c..ce940cb 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -170,4 +170,3 @@ group: "{{ atl_product_user }}" mode: 0750 ignore_errors: yes - \ No newline at end of file From 96e006cbb31b798fa846c51c08ddf2d7844953dc Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Wed, 13 Nov 2019 11:55:56 +1100 Subject: [PATCH 23/49] Sets python environment for amazon-linux-extras to system python --- roles/restore_backups/tasks/amazon.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/restore_backups/tasks/amazon.yml b/roles/restore_backups/tasks/amazon.yml index bf32125..2444212 100644 --- a/roles/restore_backups/tasks/amazon.yml +++ b/roles/restore_backups/tasks/amazon.yml @@ -5,3 +5,5 @@ command: amazon-linux-extras install -y "postgresql{{ postgres_version }}" args: creates: /usr/bin/psql + environment: + PYTHON: /bin/python From f3ced1534a717debb25aeb482e43c2988188472d Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Wed, 13 Nov 2019 12:03:48 +1100 Subject: [PATCH 24/49] DCD-796: Adds test for amazon-linux-extras install postgres --- .../molecule/restore/Dockerfile.j2 | 14 +++++++++ .../molecule/restore/molecule.yml | 30 +++++++++++++++++++ .../molecule/restore/playbook.yml | 11 +++++++ .../molecule/restore/tests/test_default.py | 16 ++++++++++ 4 files changed, 71 insertions(+) create mode 100644 roles/restore_backups/molecule/restore/Dockerfile.j2 create mode 100644 roles/restore_backups/molecule/restore/molecule.yml create mode 100644 roles/restore_backups/molecule/restore/playbook.yml create mode 100644 roles/restore_backups/molecule/restore/tests/test_default.py diff --git a/roles/restore_backups/molecule/restore/Dockerfile.j2 b/roles/restore_backups/molecule/restore/Dockerfile.j2 new file mode 100644 index 0000000..e6aa95d --- /dev/null +++ b/roles/restore_backups/molecule/restore/Dockerfile.j2 @@ -0,0 +1,14 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi diff --git a/roles/restore_backups/molecule/restore/molecule.yml b/roles/restore_backups/molecule/restore/molecule.yml new file mode 100644 index 0000000..04c0973 --- /dev/null +++ b/roles/restore_backups/molecule/restore/molecule.yml @@ -0,0 +1,30 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: amazon_linux2 + image: amazonlinux:2 + groups: + - aws_node_local + ulimits: + - nofile:262144:262144 +provisioner: + name: ansible + options: + skip-tags: runtime_pkg + lint: + name: ansible-lint + options: + x: ["701"] + inventory: + links: + group_vars: ../../../../group_vars/ +verifier: + name: testinfra + lint: + name: flake8 + enabled: false diff --git a/roles/restore_backups/molecule/restore/playbook.yml b/roles/restore_backups/molecule/restore/playbook.yml new file mode 100644 index 0000000..b3c60b0 --- /dev/null +++ b/roles/restore_backups/molecule/restore/playbook.yml @@ -0,0 +1,11 @@ +--- +- name: Converge + hosts: all + vars: + atl_backup_manifest_url: 's3://dcd-slingshot-test/dummy_manifest.json' + atl_product_user: 'jira' + atl_backup_home_restore_canary_path: '/tmp/canary.tmp' + + tasks: + - name: Install distro-specific restore support packages + include_tasks: "../../tasks/{{ ansible_distribution|lower }}.yml" diff --git a/roles/restore_backups/molecule/restore/tests/test_default.py b/roles/restore_backups/molecule/restore/tests/test_default.py new file mode 100644 index 0000000..018ae24 --- /dev/null +++ b/roles/restore_backups/molecule/restore/tests/test_default.py @@ -0,0 +1,16 @@ +import os +import pytest + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +@pytest.mark.parametrize('exe', [ + '/usr/bin/pg_dump', + '/usr/bin/pg_restore', + '/usr/bin/psql' +]) +def test_postgresql_amazon_linux_extras_exes(host, exe): + assert host.file(exe).exists From b2b18e2cff13c4042f642a7222fe52b0ad869c80 Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Wed, 13 Nov 2019 13:10:43 +1100 Subject: [PATCH 25/49] DCD-796: Fix incorrect number of pipelines batches --- bitbucket-pipelines.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 408b078..0df8a14 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -14,7 +14,7 @@ pipelines: - step: name: Pre Parallelization stage script: - - echo "Running tests in 30 batches" + - echo "Running tests in 31 batches" - step: name: Check if number of batches match actual number of scenarios script: @@ -267,4 +267,11 @@ pipelines: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 30 - + - step: + name: Molecule Test Batch - 31 + services: + - docker + script: + - apt-get update && ./bin/install-ansible --dev + - ./bin/run-tests-in-batches --batch 31 + From a0496644d4c45b9c030e68a2bcb8a17826827f3f Mon Sep 17 00:00:00 2001 From: Adam Brokes Date: Wed, 13 Nov 2019 15:28:47 +1100 Subject: [PATCH 26/49] DCD-803: Quote the statement --- roles/restore_backups/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index 6b28bb1..5b752ff 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -100,7 +100,7 @@ register: result failed_when: - result.rc != 0 - - "COMMENT ON EXTENSION" not in result.msg + - '"COMMENT ON EXTENSION" not in result.msg' when: db_created.changed and atl_backup_db_dest is defined From 1e3208d49e854234261f6f11657c8ca8b0fecfbe Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Wed, 13 Nov 2019 15:42:51 +1100 Subject: [PATCH 27/49] DCD-796: Implement test that pg_dump is the correct version --- roles/restore_backups/molecule/restore/tests/test_default.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/restore_backups/molecule/restore/tests/test_default.py b/roles/restore_backups/molecule/restore/tests/test_default.py index 018ae24..6f88a4f 100644 --- a/roles/restore_backups/molecule/restore/tests/test_default.py +++ b/roles/restore_backups/molecule/restore/tests/test_default.py @@ -14,3 +14,7 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( ]) def test_postgresql_amazon_linux_extras_exes(host, exe): assert host.file(exe).exists + +def test_postgresql_version(host): + pg_dump_version_output = host.check_output('pg_dump --version') + assert '(PostgreSQL) 9.6' in pg_dump_version_output From c1486e14ecc9a13907a56c64f168027c159c04ed Mon Sep 17 00:00:00 2001 From: Adam Brokes Date: Wed, 13 Nov 2019 15:49:00 +1100 Subject: [PATCH 28/49] DCD-803: Add explaining comment --- roles/restore_backups/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index 5b752ff..1f9ebdb 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -98,6 +98,7 @@ state: restore target: "{{ atl_backup_db_dest }}" register: result + # managed DBs in cloud providers are not allowing full root access to the DB engine, we can safely ignore the COMMENT ON EXTENSION error failed_when: - result.rc != 0 - '"COMMENT ON EXTENSION" not in result.msg' From 6f596c47dcfde2701d0038826f39fadf1172ddc5 Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Wed, 13 Nov 2019 16:23:51 +1100 Subject: [PATCH 29/49] DCD-798: name dump as tar file and specify custom format in db_restore to workaround module not supporting custom pg_dump format --- roles/restore_backups/tasks/main.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index 1f9ebdb..57cd47f 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -76,7 +76,9 @@ overwrite: different bucket: "{{ atl_backup_manifest.artifacts.sharedHome.location.location | urlsplit('hostname') }}" object: "{{ atl_backup_manifest.artifacts.sharedHome.location.location | urlsplit('path') }}" - dest: "{{ atl_backup_home_dest }}" + # We save the backup as a .tar file so that the postgresql_db module uses pg_restore instead of psql to do restore + # This can be removed when ansible 2.10 is released + dest: "{{ atl_backup_home_dest }}.tar" - name: Install distro-specific restore support packages include_tasks: "{{ ansible_distribution|lower }}.yml" @@ -96,7 +98,8 @@ template: "{{ atl_jdbc_template }}" # Depends on fetch_backup roles state: restore - target: "{{ atl_backup_db_dest }}" + target: "{{ atl_backup_db_dest }}.tar" + target_opts: "-Fc" register: result # managed DBs in cloud providers are not allowing full root access to the DB engine, we can safely ignore the COMMENT ON EXTENSION error failed_when: From b1643eccee7ce138a0e2340b088702de926476da Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Wed, 13 Nov 2019 17:31:06 +1100 Subject: [PATCH 30/49] DCD-802: become product user when unarchiving shared home. Implement test that shared home is owned by correct user --- roles/restore_backups/molecule/restore/playbook.yml | 5 ++--- .../restore_backups/molecule/restore/tests/test_default.py | 6 ++++++ roles/restore_backups/tasks/main.yml | 3 +-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/roles/restore_backups/molecule/restore/playbook.yml b/roles/restore_backups/molecule/restore/playbook.yml index b3c60b0..e2f8157 100644 --- a/roles/restore_backups/molecule/restore/playbook.yml +++ b/roles/restore_backups/molecule/restore/playbook.yml @@ -6,6 +6,5 @@ atl_product_user: 'jira' atl_backup_home_restore_canary_path: '/tmp/canary.tmp' - tasks: - - name: Install distro-specific restore support packages - include_tasks: "../../tasks/{{ ansible_distribution|lower }}.yml" + roles: + - restore_backups diff --git a/roles/restore_backups/molecule/restore/tests/test_default.py b/roles/restore_backups/molecule/restore/tests/test_default.py index 6f88a4f..d8ec2b2 100644 --- a/roles/restore_backups/molecule/restore/tests/test_default.py +++ b/roles/restore_backups/molecule/restore/tests/test_default.py @@ -18,3 +18,9 @@ def test_postgresql_amazon_linux_extras_exes(host, exe): def test_postgresql_version(host): pg_dump_version_output = host.check_output('pg_dump --version') assert '(PostgreSQL) 9.6' in pg_dump_version_output + +def test_shared_home_owner(host): + for root, dirs, files in os.walk('/media/atl/jira/shared'): + for fileName in files + dirs: + assert host.file(fileName).user == 'jira' + assert host.file(fileName).group == 'jira' \ No newline at end of file diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index 1f9ebdb..825517b 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -124,8 +124,7 @@ unarchive: src: "{{ atl_backup_home_dest }}" dest: "{{ atl_product_home_shared }}" - owner: "{{ atl_product_user }}" - group: "{{ atl_product_user }}" + become: "{{ atl_product_user }}" - name: Create restore-canary if necessary copy: From 39326a207ba9ac2ddc8c8713c378e33c93ea8626 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Thu, 14 Nov 2019 14:19:51 +1100 Subject: [PATCH 31/49] ITOPSENG-101 fixed pipelines batch count --- bitbucket-pipelines.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index a58d52c..b51f0f9 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -14,7 +14,7 @@ pipelines: - step: name: Pre Parallelization stage script: - - echo "Running tests in 31 batches" + - echo "Running tests in 32 batches" - step: name: Check if number of batches match actual number of scenarios script: @@ -275,3 +275,12 @@ pipelines: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 31 + - step: + name: Molecule Test Batch - 32 + services: + - docker + script: + - apt-get update && ./bin/install-ansible --dev + - ./bin/run-tests-in-batches --batch 32 + + From aea07909b03de466e93a96b32d0f6a0ced8733f2 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Thu, 14 Nov 2019 14:34:11 +1100 Subject: [PATCH 32/49] ITOPSENG-101 add hardcoded download path changes for now till testaid is fixed --- .../molecule/bitbucket_latest/tests/test_default.py | 4 ++-- .../molecule/confluence_latest/tests/test_default.py | 4 ++-- .../molecule/crowd_latest/tests/test_default.py | 4 ++-- roles/product_install/molecule/default/tests/test_default.py | 4 ++-- .../molecule/jira_cached_with_downgrade/tests/test_default.py | 4 ++-- .../molecule/jira_cached_with_upgrade/tests/test_default.py | 4 ++-- .../molecule/jira_software_latest/tests/test_default.py | 4 ++-- .../molecule/jira_tarball/tests/test_default.py | 4 ++-- .../molecule/jira_version_from_file/tests/test_default.py | 4 ++-- .../molecule/jira_version_override/tests/test_default.py | 4 ++-- .../molecule/servicedesk3/tests/test_default.py | 4 ++-- .../molecule/servicedesk4/tests/test_default.py | 4 ++-- .../molecule/servicedesk_latest/tests/test_default.py | 4 ++-- 13 files changed, 26 insertions(+), 26 deletions(-) diff --git a/roles/product_install/molecule/bitbucket_latest/tests/test_default.py b/roles/product_install/molecule/bitbucket_latest/tests/test_default.py index bbd851d..0592cbf 100644 --- a/roles/product_install/molecule/bitbucket_latest/tests/test_default.py +++ b/roles/product_install/molecule/bitbucket_latest/tests/test_default.py @@ -37,7 +37,7 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/media/atl/bitbucket/shared/downloads/bitbucket.' + upstream + '-x64.bin') + installer = host.file('/media/atl/downloads/bitbucket.' + upstream + '-x64.bin') assert installer.exists assert installer.user == 'root' @@ -47,6 +47,6 @@ def test_completed_lockfile(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - lockfile = host.file('/media/atl/bitbucket/shared/downloads/bitbucket.' + upstream + '-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/bitbucket.' + upstream + '-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' diff --git a/roles/product_install/molecule/confluence_latest/tests/test_default.py b/roles/product_install/molecule/confluence_latest/tests/test_default.py index 7ec072b..01044a5 100644 --- a/roles/product_install/molecule/confluence_latest/tests/test_default.py +++ b/roles/product_install/molecule/confluence_latest/tests/test_default.py @@ -35,7 +35,7 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/media/atl/confluence/shared-home/downloads/confluence.'+upstream+'-x64.bin') + installer = host.file('/media/atl/downloads/confluence.'+upstream+'-x64.bin') assert installer.exists assert installer.user == 'root' @@ -44,6 +44,6 @@ def test_completed_lockfile(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - lockfile = host.file('/media/atl/confluence/shared-home/downloads/confluence.'+upstream+'-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/confluence.'+upstream+'-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' diff --git a/roles/product_install/molecule/crowd_latest/tests/test_default.py b/roles/product_install/molecule/crowd_latest/tests/test_default.py index b75a0b5..6e5dc7c 100644 --- a/roles/product_install/molecule/crowd_latest/tests/test_default.py +++ b/roles/product_install/molecule/crowd_latest/tests/test_default.py @@ -37,7 +37,7 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/media/atl/crowd/shared/downloads/crowd.' + upstream + '.tar.gz') + installer = host.file('/media/atl/downloads/crowd.' + upstream + '.tar.gz') assert installer.exists assert installer.user == 'root' @@ -47,6 +47,6 @@ def test_completed_lockfile(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - lockfile = host.file('/media/atl/crowd/shared/downloads/crowd.' + upstream + '.tar.gz_completed') + lockfile = host.file('/media/atl/downloads/crowd.' + upstream + '.tar.gz_completed') assert lockfile.exists assert lockfile.user == 'root' diff --git a/roles/product_install/molecule/default/tests/test_default.py b/roles/product_install/molecule/default/tests/test_default.py index 930ab59..8e20d76 100644 --- a/roles/product_install/molecule/default/tests/test_default.py +++ b/roles/product_install/molecule/default/tests/test_default.py @@ -23,7 +23,7 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'-x64.bin') + installer = host.file('/media/atl/downloads/jira-core.'+upstream+'-x64.bin') assert installer.exists assert installer.user == 'root' @@ -32,6 +32,6 @@ def test_completed_lockfile(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/jira-core.'+upstream+'-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' \ No newline at end of file diff --git a/roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py b/roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py index 788c3de..725ace4 100644 --- a/roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py +++ b/roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py @@ -14,12 +14,12 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "7.10.2" def test_is_downloaded(host): - installer = host.file('/media/atl/jira/shared/downloads/jira-core.7.10.2-x64.bin') + installer = host.file('/media/atl/downloads/jira-core.7.10.2-x64.bin') assert installer.exists assert installer.user == 'root' def test_completed_lockfile(host): - lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.7.10.2-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/jira-core.7.10.2-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' diff --git a/roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py b/roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py index 0818e1b..6b215f2 100644 --- a/roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py +++ b/roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py @@ -14,12 +14,12 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "7.10.1" def test_is_downloaded(host): - installer = host.file('/media/atl/jira/shared/downloads/jira-core.7.10.1-x64.bin') + installer = host.file('/media/atl/downloads/jira-core.7.10.1-x64.bin') assert installer.exists assert installer.user == 'root' def test_completed_lockfile(host): - lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.7.10.1-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/jira-core.7.10.1-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' diff --git a/roles/product_install/molecule/jira_software_latest/tests/test_default.py b/roles/product_install/molecule/jira_software_latest/tests/test_default.py index de1dca3..80d53b8 100644 --- a/roles/product_install/molecule/jira_software_latest/tests/test_default.py +++ b/roles/product_install/molecule/jira_software_latest/tests/test_default.py @@ -35,7 +35,7 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/media/atl/jira/shared/downloads/jira-software.'+upstream+'-x64.bin') + installer = host.file('/media/atl/downloads/jira-software.'+upstream+'-x64.bin') assert installer.exists assert installer.user == 'root' @@ -44,6 +44,6 @@ def test_completed_lockfile(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - lockfile = host.file('/media/atl/jira/shared/downloads/jira-software.'+upstream+'-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/jira-software.'+upstream+'-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' \ No newline at end of file diff --git a/roles/product_install/molecule/jira_tarball/tests/test_default.py b/roles/product_install/molecule/jira_tarball/tests/test_default.py index 11a7438..7155f41 100644 --- a/roles/product_install/molecule/jira_tarball/tests/test_default.py +++ b/roles/product_install/molecule/jira_tarball/tests/test_default.py @@ -23,7 +23,7 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'.tar.gz') + installer = host.file('/media/atl/downloads/jira-core.'+upstream+'.tar.gz') assert installer.exists assert installer.user == 'root' @@ -32,6 +32,6 @@ def test_completed_lockfile(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'.tar.gz_completed') + lockfile = host.file('/media/atl/downloads/jira-core.'+upstream+'.tar.gz_completed') assert lockfile.exists assert lockfile.user == 'root' \ No newline at end of file diff --git a/roles/product_install/molecule/jira_version_from_file/tests/test_default.py b/roles/product_install/molecule/jira_version_from_file/tests/test_default.py index 8b75761..07098b7 100644 --- a/roles/product_install/molecule/jira_version_from_file/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_from_file/tests/test_default.py @@ -14,12 +14,12 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "7.13.1" def test_is_downloaded(host): - installer = host.file('/media/atl/jira/shared/downloads/jira-core.7.13.1-x64.bin') + installer = host.file('/media/atl/downloads/jira-core.7.13.1-x64.bin') assert installer.exists assert installer.user == 'root' def test_completed_lockfile(host): - lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.7.13.1-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/jira-core.7.13.1-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' diff --git a/roles/product_install/molecule/jira_version_override/tests/test_default.py b/roles/product_install/molecule/jira_version_override/tests/test_default.py index 3f16801..e4fa67d 100644 --- a/roles/product_install/molecule/jira_version_override/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_override/tests/test_default.py @@ -14,12 +14,12 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "7.13.2" def test_is_downloaded(host): - installer = host.file('/media/atl/jira/shared/downloads/jira-core.7.13.2-x64.bin') + installer = host.file('/media/atl/downloads/jira-core.7.13.2-x64.bin') assert installer.exists assert installer.user == 'root' def test_completed_lockfile(host): - lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.7.13.2-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/jira-core.7.13.2-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' diff --git a/roles/product_install/molecule/servicedesk3/tests/test_default.py b/roles/product_install/molecule/servicedesk3/tests/test_default.py index 5f50b6e..ae78b3d 100644 --- a/roles/product_install/molecule/servicedesk3/tests/test_default.py +++ b/roles/product_install/molecule/servicedesk3/tests/test_default.py @@ -14,12 +14,12 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "3.9.0" def test_is_downloaded(host): - installer = host.file('/media/atl/jira/shared/downloads/servicedesk.3.9.0-x64.bin') + installer = host.file('/media/atl/downloads/servicedesk.3.9.0-x64.bin') assert installer.exists assert installer.user == 'root' def test_completed_lockfile(host): - lockfile = host.file('/media/atl/jira/shared/downloads/servicedesk.3.9.0-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/servicedesk.3.9.0-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' diff --git a/roles/product_install/molecule/servicedesk4/tests/test_default.py b/roles/product_install/molecule/servicedesk4/tests/test_default.py index 5a22e8c..bc4814e 100644 --- a/roles/product_install/molecule/servicedesk4/tests/test_default.py +++ b/roles/product_install/molecule/servicedesk4/tests/test_default.py @@ -14,12 +14,12 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "4.1.0" def test_is_downloaded(host): - installer = host.file('/media/atl/jira/shared/downloads/servicedesk.4.1.0-x64.bin') + installer = host.file('/media/atl/downloads/servicedesk.4.1.0-x64.bin') assert installer.exists assert installer.user == 'root' def test_completed_lockfile(host): - lockfile = host.file('/media/atl/jira/shared/downloads/servicedesk.4.1.0-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/servicedesk.4.1.0-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' diff --git a/roles/product_install/molecule/servicedesk_latest/tests/test_default.py b/roles/product_install/molecule/servicedesk_latest/tests/test_default.py index cd975f3..bd4eea8 100644 --- a/roles/product_install/molecule/servicedesk_latest/tests/test_default.py +++ b/roles/product_install/molecule/servicedesk_latest/tests/test_default.py @@ -23,12 +23,12 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == sd def test_is_downloaded(host): - installer = host.file('/media/atl/jira/shared/downloads/servicedesk.'+sd+'-x64.bin') + installer = host.file('/media/atl/downloads/servicedesk.'+sd+'-x64.bin') assert installer.exists assert installer.user == 'root' def test_completed_lockfile(host): - lockfile = host.file('/media/atl/jira/shared/downloads/servicedesk.'+sd+'-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/servicedesk.'+sd+'-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' From a0080b5b079c511f4fb2ac34c2900e2a5395ed87 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 15 Nov 2019 12:15:32 +1100 Subject: [PATCH 33/49] ITOPSENG-101 fixed version in version_from_file and added better obr unpacking --- .../jira_version_from_file/tests/test_default.py | 4 ++-- .../tasks/jira-servicedesk_as_obr.yml | 12 +++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/roles/product_install/molecule/jira_version_from_file/tests/test_default.py b/roles/product_install/molecule/jira_version_from_file/tests/test_default.py index 07098b7..1b725b8 100644 --- a/roles/product_install/molecule/jira_version_from_file/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_from_file/tests/test_default.py @@ -14,12 +14,12 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "7.13.1" def test_is_downloaded(host): - installer = host.file('/media/atl/downloads/jira-core.7.13.1-x64.bin') + installer = host.file('/media/atl/downloads/jira-core.7.9.0-x64.bin') assert installer.exists assert installer.user == 'root' def test_completed_lockfile(host): - lockfile = host.file('/media/atl/downloads/jira-core.7.13.1-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/jira-core.7.9.0-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index ce940cb..e236f44 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -163,10 +163,16 @@ src: "{{ atl_obr_download }}" dest: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins" creates: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.jar" - extra_opts: - - -j - - -n + list_files: no + exclude: + - M* owner: "{{ atl_product_user }}" group: "{{ atl_product_user }}" mode: 0750 ignore_errors: yes + +- name: Move JSD dependency jars into the installed-plugins dir + shell: + cmd: "mv {{ atl_product_version_cache_dir }}/plugins/installed-plugins/dependencies/*.jar {{ atl_product_version_cache_dir }}/plugins/installed-plugins" + creates: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins/servicedesk-core-ui-plugin-{{ atl_jsd_build_info.json.name }}-REL-0022.jar" + From 587b647cbaa5662786db09805ef9913143511753 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 15 Nov 2019 12:21:51 +1100 Subject: [PATCH 34/49] ITOPSENG-101 remove extra blank line at eof --- roles/product_install/tasks/jira-servicedesk_as_obr.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index e236f44..bce18c1 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -175,4 +175,3 @@ shell: cmd: "mv {{ atl_product_version_cache_dir }}/plugins/installed-plugins/dependencies/*.jar {{ atl_product_version_cache_dir }}/plugins/installed-plugins" creates: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins/servicedesk-core-ui-plugin-{{ atl_jsd_build_info.json.name }}-REL-0022.jar" - From 2f8790324a45caca62e03f79f88002f4e0300d22 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 15 Nov 2019 16:01:02 +1100 Subject: [PATCH 35/49] ITOPSENG-101 so over this --- roles/product_install/molecule/jira_all/molecule.yml | 2 -- .../molecule/jira_version_from_file/tests/test_default.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/roles/product_install/molecule/jira_all/molecule.yml b/roles/product_install/molecule/jira_all/molecule.yml index 46049f8..7fd3163 100644 --- a/roles/product_install/molecule/jira_all/molecule.yml +++ b/roles/product_install/molecule/jira_all/molecule.yml @@ -25,8 +25,6 @@ provisioner: group_vars: ../../../../group_vars/ verifier: name: testinfra - additional_files_or_dirs: - - ../../resources/tests/test_*.py lint: name: flake8 enabled: false diff --git a/roles/product_install/molecule/jira_version_from_file/tests/test_default.py b/roles/product_install/molecule/jira_version_from_file/tests/test_default.py index 1b725b8..589e7ce 100644 --- a/roles/product_install/molecule/jira_version_from_file/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_from_file/tests/test_default.py @@ -11,7 +11,7 @@ def test_version_is_correct(host): verfile = host.file('/media/atl/jira/shared/jira-core.version') assert verfile.exists - assert verfile.content.decode("UTF-8").strip() == "7.13.1" + assert verfile.content.decode("UTF-8").strip() == "7.9.0" def test_is_downloaded(host): installer = host.file('/media/atl/downloads/jira-core.7.9.0-x64.bin') From e648c9722131b1b4238296c524c5dd0b2bc39382 Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Mon, 18 Nov 2019 10:39:28 +1100 Subject: [PATCH 36/49] DCD-798: Fix saving shared home archive as tar instead of db dump when fetchin from S3 --- roles/restore_backups/tasks/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index 57cd47f..f743e6b 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -68,7 +68,9 @@ overwrite: different bucket: "{{ atl_backup_manifest.artifacts.db.location.location | urlsplit('hostname') }}" object: "{{ atl_backup_manifest.artifacts.db.location.location | urlsplit('path') }}" - dest: "{{ atl_backup_db_dest }}" + # We save the backup as a .tar file so that the postgresql_db module uses pg_restore instead of psql to do restore + # This can be removed when ansible 2.10 is released + dest: "{{ atl_backup_db_dest }}.tar" - name: Fetch Home backup from S3 aws_s3: @@ -76,9 +78,7 @@ overwrite: different bucket: "{{ atl_backup_manifest.artifacts.sharedHome.location.location | urlsplit('hostname') }}" object: "{{ atl_backup_manifest.artifacts.sharedHome.location.location | urlsplit('path') }}" - # We save the backup as a .tar file so that the postgresql_db module uses pg_restore instead of psql to do restore - # This can be removed when ansible 2.10 is released - dest: "{{ atl_backup_home_dest }}.tar" + dest: "{{ atl_backup_home_dest }}" - name: Install distro-specific restore support packages include_tasks: "{{ ansible_distribution|lower }}.yml" From 3fa354ff0e0591b0be929813201e1d75843232de Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Mon, 18 Nov 2019 12:17:43 +1100 Subject: [PATCH 37/49] Add note to README about raising issues. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 864b060..72136e1 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,10 @@ them in the `Custom command-line parameters for Ansible` field: -e atl_product_download_url=http://s3.amazon.com/atlassian/jira-9.0.0-PRE-TEST.tar.gz -e atl_use_system_jdk=true -e atl_download_format=tarball +## Reporting issues + +If you find any bugs in this repository, or have feature requests or use cases +for us, please raise them in our [public Jira project](https://jira.atlassian.com/projects/SCALE/summary). ## Development From 4827ae84236049d1edb491c5fd090cca1b89e058 Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Mon, 18 Nov 2019 16:37:26 +1100 Subject: [PATCH 38/49] DCD-802: Fix test and implementation for setting shared home owner and group to application user --- .../molecule/restore/playbook.yml | 55 ++++++++++++++++++- .../molecule/restore/tests/test_default.py | 14 +++-- roles/restore_backups/tasks/home_restore.yml | 37 +++++++++++++ roles/restore_backups/tasks/main.yml | 29 +--------- 4 files changed, 101 insertions(+), 34 deletions(-) create mode 100644 roles/restore_backups/tasks/home_restore.yml diff --git a/roles/restore_backups/molecule/restore/playbook.yml b/roles/restore_backups/molecule/restore/playbook.yml index e2f8157..d174e48 100644 --- a/roles/restore_backups/molecule/restore/playbook.yml +++ b/roles/restore_backups/molecule/restore/playbook.yml @@ -4,7 +4,58 @@ vars: atl_backup_manifest_url: 's3://dcd-slingshot-test/dummy_manifest.json' atl_product_user: 'jira' + atl_product_user_uid: '2001' atl_backup_home_restore_canary_path: '/tmp/canary.tmp' + atl_product_home_shared: '/media/atl/jira/shared' + atl_backup_id: 'test-backup' + atl_backup_home_dest: "{{ test_archive }}" - roles: - - restore_backups + test_archive_source: '/tmp/hello' + test_archive_file: 'hello.txt' + test_archive: '/tmp/hello.tar.gz' + test_pre_step_prefix: '[PRE-TEST]' + + pre_tasks: + - name: "{{ test_pre_step_prefix }} Install tar" + package: + state: present + name: tar + + - name: "{{ test_pre_step_prefix }} Install useradd and groupadd binaries" + package: + state: present + name: shadow-utils + + - name: "{{ test_pre_step_prefix }} Create application group" + group: + name: "{{ atl_product_user }}" + gid: "{{ atl_product_user_uid }}" + + - name: "{{ test_pre_step_prefix }} Create application user" + user: + name: "{{ atl_product_user }}" + uid: "{{ atl_product_user_uid }}" + group: "{{ atl_product_user }}" + + - block: + - name: "{{ test_pre_step_prefix }} Create a directory for the shared home archive" + file: + path: "{{ test_archive_source }}" + state: directory + - name: "{{ test_pre_step_prefix }} Create a file in the shared home" + lineinfile: + create: yes + line: 'Hello, world!' + path: "{{ test_archive_source }}/{{ test_archive_file }}" + - name: "{{ test_pre_step_prefix }} Archive the shared home" + archive: + path: "{{ test_archive_source }}" + dest: "{{ test_archive }}" + owner: "{{ atl_product_user }}" + + tasks: + - name: Install distro-specific restore support packages + include_tasks: "../../tasks/{{ ansible_distribution|lower }}.yml" + + - name: Restore shared home + include_tasks: "../../tasks/home_restore.yml" diff --git a/roles/restore_backups/molecule/restore/tests/test_default.py b/roles/restore_backups/molecule/restore/tests/test_default.py index d8ec2b2..f8ab676 100644 --- a/roles/restore_backups/molecule/restore/tests/test_default.py +++ b/roles/restore_backups/molecule/restore/tests/test_default.py @@ -19,8 +19,12 @@ def test_postgresql_version(host): pg_dump_version_output = host.check_output('pg_dump --version') assert '(PostgreSQL) 9.6' in pg_dump_version_output -def test_shared_home_owner(host): - for root, dirs, files in os.walk('/media/atl/jira/shared'): - for fileName in files + dirs: - assert host.file(fileName).user == 'jira' - assert host.file(fileName).group == 'jira' \ No newline at end of file +@pytest.mark.parametrize('file', [ + '/media/atl/jira/shared', + '/media/atl/jira/shared/hello', + '/media/atl/jira/shared/hello/hello.txt' +]) +def test_shared_home_owner(host, file): + assert host.file(file).exists + assert host.file(file).user == 'jira' + assert host.file(file).group == 'jira' \ No newline at end of file diff --git a/roles/restore_backups/tasks/home_restore.yml b/roles/restore_backups/tasks/home_restore.yml new file mode 100644 index 0000000..2a9fa1f --- /dev/null +++ b/roles/restore_backups/tasks/home_restore.yml @@ -0,0 +1,37 @@ +--- +- name: Check for the restore canary file + stat: + path: "{{ atl_backup_home_restore_canary_path }}" + register: restore_canary + +- block: + - name: Create shared home if necessary + file: + path: "{{ atl_product_home_shared }}" + state: directory + mode: 0750 + owner: "{{ atl_product_user }}" + group: "{{ atl_product_user }}" + + - name: Restore the shared-home backup + unarchive: + src: "{{ atl_backup_home_dest }}" + remote_src: yes + dest: "{{ atl_product_home_shared }}" + mode: 0640 + + - name: Set shared home file owner to application user + file: + path: "{{ atl_product_home_shared }}" + recurse: yes + group: "{{ atl_product_user }}" + state: directory + mode: 0640 + owner: "{{ atl_product_user }}" + + - name: Create restore-canary if necessary + copy: + dest: "{{ atl_backup_home_restore_canary_path }}" + content: "{{ atl_backup_id }}" + + when: not restore_canary.stat.exists diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index 3071044..c5b26ed 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -108,33 +108,8 @@ when: db_created.changed and atl_backup_db_dest is defined - - name: Check for the restore canary file - stat: - path: "{{ atl_backup_home_restore_canary_path }}" - register: restore_canary - - - block: - - - name: Create shared home if necessary - file: - path: "{{ atl_product_home_shared }}" - state: directory - mode: 0750 - owner: "{{ atl_product_user }}" - group: "{{ atl_product_user }}" - - - name: Restore the shared-home backup - unarchive: - src: "{{ atl_backup_home_dest }}" - dest: "{{ atl_product_home_shared }}" - become: "{{ atl_product_user }}" - - - name: Create restore-canary if necessary - copy: - dest: "{{ atl_backup_home_restore_canary_path }}" - content: "{{ atl_backup_id }}" - - when: not restore_canary.stat.exists + - name: Restore shared home + include_tasks: "{{ home_restore.yml }}" when: atl_restore_required From 688be1f5d79f2ff75b51d3aadbf71055f7b482fe Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Mon, 18 Nov 2019 16:59:56 +1100 Subject: [PATCH 39/49] DCD-802: Add test and logic for setting version file owner to root in shared home --- roles/restore_backups/molecule/restore/playbook.yml | 12 +++++++++++- .../molecule/restore/tests/test_default.py | 7 ++++++- roles/restore_backups/tasks/home_restore.yml | 13 +++++++++++-- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/roles/restore_backups/molecule/restore/playbook.yml b/roles/restore_backups/molecule/restore/playbook.yml index d174e48..3e4272c 100644 --- a/roles/restore_backups/molecule/restore/playbook.yml +++ b/roles/restore_backups/molecule/restore/playbook.yml @@ -4,6 +4,8 @@ vars: atl_backup_manifest_url: 's3://dcd-slingshot-test/dummy_manifest.json' atl_product_user: 'jira' + atl_product_edition: 'jira-software' + atl_product_version_cache: "{{ atl_product_home_shared }}/{{ atl_product_edition }}.version" atl_product_user_uid: '2001' atl_backup_home_restore_canary_path: '/tmp/canary.tmp' atl_product_home_shared: '/media/atl/jira/shared' @@ -12,6 +14,7 @@ test_archive_source: '/tmp/hello' test_archive_file: 'hello.txt' + test_product_version_file: "/tmp/{{ atl_product_edition }}.version" test_archive: '/tmp/hello.tar.gz' test_pre_step_prefix: '[PRE-TEST]' @@ -47,9 +50,16 @@ create: yes line: 'Hello, world!' path: "{{ test_archive_source }}/{{ test_archive_file }}" + - name: "{{ test_pre_step_prefix }} Create the version file in the shared home" + lineinfile: + create: yes + line: '8.5' + path: "{{ test_product_version_file }}" - name: "{{ test_pre_step_prefix }} Archive the shared home" archive: - path: "{{ test_archive_source }}" + path: + - "{{ test_archive_source }}" + - "{{ test_product_version_file }}" dest: "{{ test_archive }}" owner: "{{ atl_product_user }}" diff --git a/roles/restore_backups/molecule/restore/tests/test_default.py b/roles/restore_backups/molecule/restore/tests/test_default.py index f8ab676..98af566 100644 --- a/roles/restore_backups/molecule/restore/tests/test_default.py +++ b/roles/restore_backups/molecule/restore/tests/test_default.py @@ -27,4 +27,9 @@ def test_postgresql_version(host): def test_shared_home_owner(host, file): assert host.file(file).exists assert host.file(file).user == 'jira' - assert host.file(file).group == 'jira' \ No newline at end of file + assert host.file(file).group == 'jira' + +def test_version_file_owned_by_root(host): + assert host.file('/media/atl/jira/shared/jira-software.version').exists + assert host.file('/media/atl/jira/shared/jira-software.version').user == 'root' + assert host.file('/media/atl/jira/shared/jira-software.version').group == 'root' \ No newline at end of file diff --git a/roles/restore_backups/tasks/home_restore.yml b/roles/restore_backups/tasks/home_restore.yml index 2a9fa1f..c3cb081 100644 --- a/roles/restore_backups/tasks/home_restore.yml +++ b/roles/restore_backups/tasks/home_restore.yml @@ -20,14 +20,23 @@ dest: "{{ atl_product_home_shared }}" mode: 0640 - - name: Set shared home file owner to application user + - name: Set shared home owner and group to application user file: path: "{{ atl_product_home_shared }}" recurse: yes group: "{{ atl_product_user }}" + owner: "{{ atl_product_user }}" state: directory mode: 0640 - owner: "{{ atl_product_user }}" + + - name: Set version file owner and group to root + file: + path: "{{ atl_product_version_cache }}" + group: root + owner: root + state: file + # Ignore the error in case there is no product version file in the backup + ignore_errors: yes - name: Create restore-canary if necessary copy: From cbe62d02fd4c49a2bc795b5b886fd99f73bf0523 Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Mon, 18 Nov 2019 17:02:08 +1100 Subject: [PATCH 40/49] DCD-802: Organise vars in restore molecule playbook --- .../molecule/restore/playbook.yml | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/roles/restore_backups/molecule/restore/playbook.yml b/roles/restore_backups/molecule/restore/playbook.yml index 3e4272c..a2b36ea 100644 --- a/roles/restore_backups/molecule/restore/playbook.yml +++ b/roles/restore_backups/molecule/restore/playbook.yml @@ -2,21 +2,22 @@ - name: Converge hosts: all vars: - atl_backup_manifest_url: 's3://dcd-slingshot-test/dummy_manifest.json' - atl_product_user: 'jira' - atl_product_edition: 'jira-software' - atl_product_version_cache: "{{ atl_product_home_shared }}/{{ atl_product_edition }}.version" - atl_product_user_uid: '2001' - atl_backup_home_restore_canary_path: '/tmp/canary.tmp' - atl_product_home_shared: '/media/atl/jira/shared' - atl_backup_id: 'test-backup' atl_backup_home_dest: "{{ test_archive }}" + atl_backup_home_restore_canary_path: '/tmp/canary.tmp' + atl_backup_id: 'test-backup' + atl_backup_manifest_url: 'fake_manifest' + + atl_product_edition: 'jira-software' + atl_product_home_shared: '/media/atl/jira/shared' + atl_product_user: 'jira' + atl_product_user_uid: '2001' + atl_product_version_cache: "{{ atl_product_home_shared }}/{{ atl_product_edition }}.version" - test_archive_source: '/tmp/hello' - test_archive_file: 'hello.txt' - test_product_version_file: "/tmp/{{ atl_product_edition }}.version" test_archive: '/tmp/hello.tar.gz' + test_archive_file: 'hello.txt' + test_archive_source: '/tmp/hello' test_pre_step_prefix: '[PRE-TEST]' + test_product_version_file: "/tmp/{{ atl_product_edition }}.version" pre_tasks: - name: "{{ test_pre_step_prefix }} Install tar" From b12d26c282e77c2bc51bb4ac389a5fc2974909ac Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Tue, 19 Nov 2019 16:01:23 +1100 Subject: [PATCH 41/49] DCD-802: Fix reference to home_restore task in include task in restore_backups main.yml --- roles/restore_backups/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index c5b26ed..1c81430 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -109,7 +109,7 @@ - name: Restore shared home - include_tasks: "{{ home_restore.yml }}" + include_tasks: "home_restore.yml" when: atl_restore_required From 56d4be5c04b52bcf7168df3e20a6cb440e441969 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Wed, 20 Nov 2019 10:08:07 +1100 Subject: [PATCH 42/49] ITOPSENG-101 addresses PR comments --- .../tasks/jira-servicedesk_as_obr.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index bce18c1..e602765 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -110,9 +110,6 @@ src: "{{ atl_obr_download }}" dest: "{{ atl_obr_shared_download }}" remote_src: true - when: - - moving_lock_created is succeeded - - moving_lock_created.changed register: copied - name: Create completed_lock once obr downloaded and copied @@ -150,7 +147,7 @@ - name: Ensure instaled-plugins dir exists file: - path: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins" + path: "{{ atl_product_home_shared }}/plugins/installed-plugins" state: directory mode: 0750 owner: "{{ atl_product_user }}" @@ -161,8 +158,8 @@ unarchive: remote_src: true src: "{{ atl_obr_download }}" - dest: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins" - creates: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.jar" + dest: "{{ atl_product_home_shared }}/plugins/installed-plugins" + creates: "{{ atl_product_home_shared }}/plugins/installed-plugins/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.jar" list_files: no exclude: - M* @@ -173,5 +170,5 @@ - name: Move JSD dependency jars into the installed-plugins dir shell: - cmd: "mv {{ atl_product_version_cache_dir }}/plugins/installed-plugins/dependencies/*.jar {{ atl_product_version_cache_dir }}/plugins/installed-plugins" - creates: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins/servicedesk-core-ui-plugin-{{ atl_jsd_build_info.json.name }}-REL-0022.jar" + cmd: "mv {{ atl_product_home_shared }}/plugins/installed-plugins/dependencies/*.jar {{ atl_product_home_shared }}/plugins/installed-plugins" + creates: "{{ atl_product_home_shared }}/plugins/installed-plugins/servicedesk-core-ui-plugin-{{ atl_jsd_build_info.json.name }}-*.jar" From 096549edadb1006effb3989782249b3a87ae1f16 Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Wed, 20 Nov 2019 13:55:59 +1100 Subject: [PATCH 43/49] Preserve file permissions when restoring the shared home backup --- roles/restore_backups/tasks/home_restore.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/restore_backups/tasks/home_restore.yml b/roles/restore_backups/tasks/home_restore.yml index c3cb081..bdeb866 100644 --- a/roles/restore_backups/tasks/home_restore.yml +++ b/roles/restore_backups/tasks/home_restore.yml @@ -18,7 +18,7 @@ src: "{{ atl_backup_home_dest }}" remote_src: yes dest: "{{ atl_product_home_shared }}" - mode: 0640 + mode: preserve - name: Set shared home owner and group to application user file: @@ -27,7 +27,6 @@ group: "{{ atl_product_user }}" owner: "{{ atl_product_user }}" state: directory - mode: 0640 - name: Set version file owner and group to root file: From cb6647ba8e70108d5bf4400c617a13497b5aac7a Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Wed, 20 Nov 2019 15:53:16 +1100 Subject: [PATCH 44/49] DCD-802: Fix file mode preservation in restore and add test for file mode --- roles/restore_backups/molecule/restore/playbook.yml | 3 +++ roles/restore_backups/molecule/restore/tests/test_default.py | 5 +++++ roles/restore_backups/tasks/home_restore.yml | 1 - 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/roles/restore_backups/molecule/restore/playbook.yml b/roles/restore_backups/molecule/restore/playbook.yml index a2b36ea..073d24b 100644 --- a/roles/restore_backups/molecule/restore/playbook.yml +++ b/roles/restore_backups/molecule/restore/playbook.yml @@ -46,16 +46,19 @@ file: path: "{{ test_archive_source }}" state: directory + mode: 0755 - name: "{{ test_pre_step_prefix }} Create a file in the shared home" lineinfile: create: yes line: 'Hello, world!' path: "{{ test_archive_source }}/{{ test_archive_file }}" + mode: 0640 - name: "{{ test_pre_step_prefix }} Create the version file in the shared home" lineinfile: create: yes line: '8.5' path: "{{ test_product_version_file }}" + mode: 0640 - name: "{{ test_pre_step_prefix }} Archive the shared home" archive: path: diff --git a/roles/restore_backups/molecule/restore/tests/test_default.py b/roles/restore_backups/molecule/restore/tests/test_default.py index 98af566..c9882bc 100644 --- a/roles/restore_backups/molecule/restore/tests/test_default.py +++ b/roles/restore_backups/molecule/restore/tests/test_default.py @@ -1,5 +1,6 @@ import os import pytest +from stat import * import testinfra.utils.ansible_runner @@ -29,6 +30,10 @@ def test_shared_home_owner(host, file): assert host.file(file).user == 'jira' assert host.file(file).group == 'jira' +def test_file_modes(host): + assert host.file('/media/atl/jira/shared/hello').mode == 0o755 + assert host.file('/media/atl/jira/shared/hello/hello.txt').mode == 0o640 + def test_version_file_owned_by_root(host): assert host.file('/media/atl/jira/shared/jira-software.version').exists assert host.file('/media/atl/jira/shared/jira-software.version').user == 'root' diff --git a/roles/restore_backups/tasks/home_restore.yml b/roles/restore_backups/tasks/home_restore.yml index bdeb866..4c5f2ad 100644 --- a/roles/restore_backups/tasks/home_restore.yml +++ b/roles/restore_backups/tasks/home_restore.yml @@ -18,7 +18,6 @@ src: "{{ atl_backup_home_dest }}" remote_src: yes dest: "{{ atl_product_home_shared }}" - mode: preserve - name: Set shared home owner and group to application user file: From 0cbd11c7020144ec863533c8e84117c0c8407e1c Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Wed, 20 Nov 2019 15:54:08 +1100 Subject: [PATCH 45/49] DCD-802: Fix unused import in restore test --- roles/restore_backups/molecule/restore/tests/test_default.py | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/restore_backups/molecule/restore/tests/test_default.py b/roles/restore_backups/molecule/restore/tests/test_default.py index c9882bc..bcfa53d 100644 --- a/roles/restore_backups/molecule/restore/tests/test_default.py +++ b/roles/restore_backups/molecule/restore/tests/test_default.py @@ -1,6 +1,5 @@ import os import pytest -from stat import * import testinfra.utils.ansible_runner From d534a15c03e4674a7ba54d27eacfa156bcf5dcda Mon Sep 17 00:00:00 2001 From: Don Domingo Date: Thu, 21 Nov 2019 03:12:05 +0000 Subject: [PATCH 46/49] README.md edited online with Bitbucket --- README.md | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 72136e1..3485790 100644 --- a/README.md +++ b/README.md @@ -27,12 +27,11 @@ sourced as environment variables to be retrieved at runtime . See the helper-script `bin/ansible-with-atl-env` and the corresponding `groups_vars/aws_node_local.yml` var-file. -#### Overriding parameters +### Customizing your deployment -If you want to customise the playbook behaviour the simplest method is to fork -this repository and add your own. However, for some one-off tasks you can also -override the default and calculated settings with special values. To do this, provide -command-line overrides to +To customise playbook behaviour, you can fork this repository and edit it as +needed. However, for one-off tasks you can also override the default and +calculated settings with special values. To do this, provide command-line overrides to [ansible-playbook](https://docs.ansible.com/ansible/latest/cli/ansible-playbook.html). The most likely use-case for this is to download a custom product distribution @@ -61,6 +60,14 @@ them in the `Custom command-line parameters for Ansible` field: -e atl_product_download_url=http://s3.amazon.com/atlassian/jira-9.0.0-PRE-TEST.tar.gz -e atl_use_system_jdk=true -e atl_download_format=tarball +#### Parameters you can override + +The following files contain many of the most common system configuration parameters +you can override for your deployment: + +- `[/roles/product_install/defaults/main.yml](roles/product_install/defaults/main.yml)` +- `[/group_vars/aws_node_local.yml](group_vars/aws_node_local.yml)` + ## Reporting issues If you find any bugs in this repository, or have feature requests or use cases @@ -79,11 +86,14 @@ Where possible the roles are also product-agnostic (e.g. downloads), with more specific functionality added in later product-specific roles. Roles should be reasonably self-contained, with sensible defaults configured in -`/defaults/main.yml` and overridden by the playbook at runtime. Roles may +`/roles//defaults/main.yml`. Like all playbook parameters, you can override +them at runtime (see + +and overridden by the playbook at runtime. Roles may implicitly depend on variables being defined elsewhere where they cannot define -them natively (e.g. the `jira_config` role depends on the `atl_cluster_node_id` +them natively. For example, the `jira_config` role depends on the `atl_cluster_node_id` var being defined; on AWS this is provided by the `aws_common` role, which -should be run first). +should be run first. ### Development and testing From 2e3db07637879dad10ec9c58dc4149782ad8b37c Mon Sep 17 00:00:00 2001 From: Don Domingo Date: Thu, 21 Nov 2019 05:59:41 +0000 Subject: [PATCH 47/49] More edits --- README.md | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 3485790..ce36a5c 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ cloud environments. ## Usage -### Cloud DC-node deployment playbooks +### Configuring Data Center nodes on cloud deployments The usual scenario for usage as part of a cloud deployment is to invoke the script as part of post-creation actions invoked while a new product node is @@ -60,38 +60,34 @@ them in the `Custom command-line parameters for Ansible` field: -e atl_product_download_url=http://s3.amazon.com/atlassian/jira-9.0.0-PRE-TEST.tar.gz -e atl_use_system_jdk=true -e atl_download_format=tarball -#### Parameters you can override +#### Other customizable parameters -The following files contain many of the most common system configuration parameters -you can override for your deployment: +Consult the following files for clues on other parameters you can customize for your deployment: - `[/roles/product_install/defaults/main.yml](roles/product_install/defaults/main.yml)` - `[/group_vars/aws_node_local.yml](group_vars/aws_node_local.yml)` -## Reporting issues +More customizable parameters are defined in specific roles -- specifically, in the +role's `defaults/main.yml` file. Most of these parameters use the `atl_` prefix. You can +use the following [Bitbucket code search query](https://confluence.atlassian.com/bitbucket/search-873876782.html) to find them: -If you find any bugs in this repository, or have feature requests or use cases -for us, please raise them in our [public Jira project](https://jira.atlassian.com/projects/SCALE/summary). + repo:dc-deployments-automation repo:dc-deployments-automation path:*/defaults/main.yml atl ## Development ### Development philosophy The suite is intended to consist of a number of small, composable roles that can -be combined together into playbooks. Wherever possible the roles should be -platform-agnostic as possible, with platform-specific functionality broken out -into more specific roles. - -Where possible the roles are also product-agnostic (e.g. downloads), with more -specific functionality added in later product-specific roles. +be combined together into playbooks. Wherever possible, roles should be product-agnostic +(e.g. downloads) and platform-agnostic. Functions that are product-specific or +platform-specific are split off into separate roles. Roles should be reasonably self-contained, with sensible defaults configured in `/roles//defaults/main.yml`. Like all playbook parameters, you can override -them at runtime (see +them at runtime. -and overridden by the playbook at runtime. Roles may -implicitly depend on variables being defined elsewhere where they cannot define -them natively. For example, the `jira_config` role depends on the `atl_cluster_node_id` +Some roles implicitly depend on other variables beind defined elsewhere. +For example, the `jira_config` role depends on the `atl_cluster_node_id` var being defined; on AWS this is provided by the `aws_common` role, which should be run first. @@ -107,15 +103,18 @@ environment and running tests. information. * Inventory files are under `inv/`. For AWS `cfn-init` the inventory `inv/aws_node_local` inventory is probably what you want. - * Note that this expects the environment to be setup with infrastructure - information; see _Usage_ above. + * Note that this expects the environment to be setup with infrastructure information (refer to the _Usage_ section above). * Global group vars loaded automatically from `group_vars/.yml`. In particular note `group_vars/aws_node_local.yml` which loads infrastructure information from the environment. * Roles are under `roles/` - * Platform specific roles start with `_...`, - e.g. `roles/aws_common/`. - * Similarly, product-specific roles should start with `_...`. + * Platform specific roles start with `_...`, e.g. `roles/aws_common/`. + * Similarly, product-specific roles should start with `_...`. + +## Reporting issues + +If you find any bugs in this repository, or have feature requests or use cases +for us, please raise them in our [public Jira project](https://jira.atlassian.com/projects/SCALE/summary). ## License From deb3d2a6cf5e9a7a73817872c96007e70f311d6d Mon Sep 17 00:00:00 2001 From: Don Domingo Date: Thu, 21 Nov 2019 13:42:55 +0000 Subject: [PATCH 48/49] More edits added TOC testing anchor links --- README.md | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index ce36a5c..b313967 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # Atlassian Data-Center Installation Automation +[TOC] + ## Introduction This repository is a suite of Ansible roles, playbooks and support scripts to @@ -62,22 +64,27 @@ them in the `Custom command-line parameters for Ansible` field: #### Other customizable parameters -Consult the following files for clues on other parameters you can customize for your deployment: +For more deployment customization options, consult the following files for parameters you can +override: -- `[/roles/product_install/defaults/main.yml](roles/product_install/defaults/main.yml)` -- `[/group_vars/aws_node_local.yml](group_vars/aws_node_local.yml)` +- [`/roles/product_install/defaults/main.yml`](roles/product_install/defaults/main.yml) +- [`/group_vars/aws_node_local.yml`](group_vars/aws_node_local.yml) More customizable parameters are defined in specific roles -- specifically, in the role's `defaults/main.yml` file. Most of these parameters use the `atl_` prefix. You can -use the following [Bitbucket code search query](https://confluence.atlassian.com/bitbucket/search-873876782.html) to find them: +use the following [Bitbucket code search query](https://confluence.atlassian.com/bitbucket/search-873876782.html) +to find them: repo:dc-deployments-automation repo:dc-deployments-automation path:*/defaults/main.yml atl -## Development +### Development and testing -### Development philosophy +See [Development](DEVELOPMENT.md) for details on setting up a development +environment and running tests. -The suite is intended to consist of a number of small, composable roles that can +## Roles philosophy + +This suite is intended to consist of many small, composable roles that can be combined together into playbooks. Wherever possible, roles should be product-agnostic (e.g. downloads) and platform-agnostic. Functions that are product-specific or platform-specific are split off into separate roles. @@ -91,23 +98,19 @@ For example, the `jira_config` role depends on the `atl_cluster_node_id` var being defined; on AWS this is provided by the `aws_common` role, which should be run first. -### Development and testing - -See [Development](DEVELOPMENT.md) for details on setting up a development -environment and running tests. ## Ansible layout * Helper scripts are in `bin/`. In particular the `bin/ansible-with-atl-env` - wrapper is of use during AWS node initialisation. See _Usage_ above for more - information. + wrapper is of use during AWS node initialisation. Refer to the [Usage](#markdown-header-usage) section for + more information. * Inventory files are under `inv/`. For AWS `cfn-init` the inventory `inv/aws_node_local` inventory is probably what you want. * Note that this expects the environment to be setup with infrastructure information (refer to the _Usage_ section above). * Global group vars loaded automatically from `group_vars/.yml`. In particular note `group_vars/aws_node_local.yml` which loads infrastructure information from the environment. -* Roles are under `roles/` +* Roles are defined in `roles/` * Platform specific roles start with `_...`, e.g. `roles/aws_common/`. * Similarly, product-specific roles should start with `_...`. From 22f05459e58f05a90b324acc8167a555d8527ab7 Mon Sep 17 00:00:00 2001 From: Don Domingo Date: Thu, 21 Nov 2019 14:01:55 +0000 Subject: [PATCH 49/49] more edits, then added proper page anchors --- README.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index b313967..29605c6 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,16 @@ - -# Atlassian Data-Center Installation Automation - -[TOC] - -## Introduction +# Atlassian Data Center Installation Automation This repository is a suite of Ansible roles, playbooks and support scripts to automate the installation and maintenance of Atlassian Data Center products in cloud environments. -## Usage +On this page: -### Configuring Data Center nodes on cloud deployments +[TOC] + +# Usage + +## Configuring Data Center nodes on cloud deployments The usual scenario for usage as part of a cloud deployment is to invoke the script as part of post-creation actions invoked while a new product node is @@ -25,11 +24,11 @@ In practice, the Ansible roles require some information about the infrastructure that was deployed (e.g. RDS endpoint/password). The way this is currently achieved (on AWS) is that have the CloudFormation template dump this information into the file `/etc/atl` as `RESOURCE_VAR=` lines. This can be then -sourced as environment variables to be retrieved at runtime . See the +sourced as environment variables to be retrieved at runtime. See the helper-script `bin/ansible-with-atl-env` and the corresponding `groups_vars/aws_node_local.yml` var-file. -### Customizing your deployment +## Customizing your deployment To customise playbook behaviour, you can fork this repository and edit it as needed. However, for one-off tasks you can also override the default and @@ -62,7 +61,7 @@ them in the `Custom command-line parameters for Ansible` field: -e atl_product_download_url=http://s3.amazon.com/atlassian/jira-9.0.0-PRE-TEST.tar.gz -e atl_use_system_jdk=true -e atl_download_format=tarball -#### Other customizable parameters +### Other customizable parameters For more deployment customization options, consult the following files for parameters you can override: @@ -77,12 +76,12 @@ to find them: repo:dc-deployments-automation repo:dc-deployments-automation path:*/defaults/main.yml atl -### Development and testing +# Development and testing See [Development](DEVELOPMENT.md) for details on setting up a development environment and running tests. -## Roles philosophy +# Roles philosophy This suite is intended to consist of many small, composable roles that can be combined together into playbooks. Wherever possible, roles should be product-agnostic @@ -99,14 +98,15 @@ var being defined; on AWS this is provided by the `aws_common` role, which should be run first. -## Ansible layout +# Ansible layout * Helper scripts are in `bin/`. In particular the `bin/ansible-with-atl-env` wrapper is of use during AWS node initialisation. Refer to the [Usage](#markdown-header-usage) section for more information. * Inventory files are under `inv/`. For AWS `cfn-init` the inventory `inv/aws_node_local` inventory is probably what you want. - * Note that this expects the environment to be setup with infrastructure information (refer to the _Usage_ section above). + * Note that this expects the environment to be setup with infrastructure information. + Refer to the [Usage](#markdown-header-usage) section for more information. * Global group vars loaded automatically from `group_vars/.yml`. In particular note `group_vars/aws_node_local.yml` which loads infrastructure information from the environment. @@ -114,12 +114,12 @@ should be run first. * Platform specific roles start with `_...`, e.g. `roles/aws_common/`. * Similarly, product-specific roles should start with `_...`. -## Reporting issues +# Reporting issues If you find any bugs in this repository, or have feature requests or use cases for us, please raise them in our [public Jira project](https://jira.atlassian.com/projects/SCALE/summary). -## License +# License Copyright © 2019 Atlassian Corporation Pty Ltd. Licensed under the Apache License, Version 2.0.