From c724b0432af4cc37bff83f07cfee7f3ab712ea3a Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 8 Oct 2019 17:19:44 +1100 Subject: [PATCH 001/109] DCD-686: Initial implementation of DB restore from dump. --- roles/database_init/tasks/main.yml | 85 ++++++++++++++++++------------ 1 file changed, 51 insertions(+), 34 deletions(-) diff --git a/roles/database_init/tasks/main.yml b/roles/database_init/tasks/main.yml index 99638f7..003f7c8 100644 --- a/roles/database_init/tasks/main.yml +++ b/roles/database_init/tasks/main.yml @@ -1,40 +1,57 @@ --- -- name: Create application DB user - postgresql_user: - login_host: "{{ atl_db_host }}" - login_user: "{{ atl_db_root_user }}" - login_password: "{{ atl_db_root_password }}" - port: "{{ atl_db_port }}" - name: "{{ atl_jdbc_user }}" - password: "{{ atl_jdbc_password }}" - expires: 'infinity' - tags: - - new_only +- block: -- name: Update root privs for new user - postgresql_privs: - login_host: "{{ atl_db_host }}" - login_user: "{{ atl_db_root_user }}" - login_password: "{{ atl_db_root_password }}" - database: postgres - roles: "{{ atl_db_root_user }}" - objs: "{{ atl_jdbc_user }}" - type: group - tags: - - new_only + - name: Create application DB user + postgresql_user: + login_host: "{{ atl_db_host }}" + login_user: "{{ atl_db_root_user }}" + login_password: "{{ atl_db_root_password }}" + port: "{{ atl_db_port }}" + name: "{{ atl_jdbc_user }}" + password: "{{ atl_jdbc_password }}" + expires: 'infinity' + + - name: Update root privs for new user + postgresql_privs: + login_host: "{{ atl_db_host }}" + login_user: "{{ atl_db_root_user }}" + login_password: "{{ atl_db_root_password }}" + database: postgres + roles: "{{ atl_db_root_user }}" + objs: "{{ atl_jdbc_user }}" + type: group + + - name: Create new application database + postgresql_db: + login_host: "{{ atl_db_host }}" + login_user: "{{ atl_db_root_user }}" + login_password: "{{ atl_db_root_password }}" + port: "{{ atl_db_port }}" + name: "{{ atl_jdbc_db_name }}" + owner: "{{ atl_jdbc_user }}" + encoding: "{{ atl_jdbc_encoding }}" + lc_collate: "{{ atl_jdbc_collation }}" + lc_ctype: "{{ atl_jdbc_ctype }}" + template: "{{ atl_jdbc_template }}" + when: atl_backup_db_dest is not defined + + - name: Restore application database + postgresql_db: + login_host: "{{ atl_db_host }}" + login_user: "{{ atl_db_root_user }}" + login_password: "{{ atl_db_root_password }}" + port: "{{ atl_db_port }}" + name: "{{ atl_jdbc_db_name }}" + owner: "{{ atl_jdbc_user }}" + encoding: "{{ atl_jdbc_encoding }}" + lc_collate: "{{ atl_jdbc_collation }}" + lc_ctype: "{{ atl_jdbc_ctype }}" + template: "{{ atl_jdbc_template }}" + # Depends on fetch_backup roles + state: restore + target: "{{ atl_backup_db_dest }}" + when: atl_backup_db_dest is defined -- name: Create application database - postgresql_db: - login_host: "{{ atl_db_host }}" - login_user: "{{ atl_db_root_user }}" - login_password: "{{ atl_db_root_password }}" - port: "{{ atl_db_port }}" - name: "{{ atl_jdbc_db_name }}" - owner: "{{ atl_jdbc_user }}" - encoding: "{{ atl_jdbc_encoding }}" - lc_collate: "{{ atl_jdbc_collation }}" - lc_ctype: "{{ atl_jdbc_ctype }}" - template: "{{ atl_jdbc_template }}" tags: - new_only From 32d3640bbb90412df5eebda7aa00ac8045682709 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 9 Oct 2019 12:43:41 +1100 Subject: [PATCH 002/109] DCD-686: Add fetching of backups to Jira for testing. --- aws_jira_dc_node.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/aws_jira_dc_node.yml b/aws_jira_dc_node.yml index 41b7be6..6ef0765 100644 --- a/aws_jira_dc_node.yml +++ b/aws_jira_dc_node.yml @@ -21,6 +21,7 @@ - role: linux_common - role: aws_common - role: aws_shared_fs_config + - role: fetch_backups - role: product_common - role: product_install - role: database_init From cb691d25562630d088bd193edb6efee743ab1c76 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 9 Oct 2019 13:14:04 +1100 Subject: [PATCH 003/109] DCD-686: We still need some python2 packages. --- bin/install-ansible | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/install-ansible b/bin/install-ansible index b515cdf..196562e 100755 --- a/bin/install-ansible +++ b/bin/install-ansible @@ -6,8 +6,12 @@ source /etc/os-release if [[ $ID = "amzn" ]]; then yum install -y \ python3-devel \ - python3-pip + python3-pip \ + python2-boto3 \ + python2-botocore + else + # FIXME: Currently assumes Debian-based apt-get update && \ apt-get install -y \ python3-dev \ From 5996a176d5f54f9b221208582848b2331fd3729d Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 9 Oct 2019 13:25:32 +1100 Subject: [PATCH 004/109] DCD-686: We need postgres installed for client utils. --- bin/install-ansible | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/install-ansible b/bin/install-ansible index 196562e..98189a5 100755 --- a/bin/install-ansible +++ b/bin/install-ansible @@ -4,7 +4,9 @@ set -e source /etc/os-release if [[ $ID = "amzn" ]]; then + amazon-linux-extras enable postgresql9.6 yum install -y \ + postgresql \ python3-devel \ python3-pip \ python2-boto3 \ From adf9c270932fd6df095c6f3d762356cf901fe262 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 9 Oct 2019 13:44:02 +1100 Subject: [PATCH 005/109] DCD-686: Ignore any roles specified in the DB dump during restore. --- roles/database_init/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/database_init/tasks/main.yml b/roles/database_init/tasks/main.yml index 003f7c8..ec5ce38 100644 --- a/roles/database_init/tasks/main.yml +++ b/roles/database_init/tasks/main.yml @@ -51,6 +51,7 @@ # Depends on fetch_backup roles state: restore target: "{{ atl_backup_db_dest }}" + target_opts: "--no-owner --role={{ atl_jdbc_user }}" when: atl_backup_db_dest is defined tags: From 9d8b47ba9ea7f8c5f2aa82352b53138c528efda0 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 9 Oct 2019 13:48:50 +1100 Subject: [PATCH 006/109] DCD-686: Only download backups if changed. --- roles/fetch_backups/tasks/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/fetch_backups/tasks/main.yml b/roles/fetch_backups/tasks/main.yml index 45d7e1f..16b18b8 100644 --- a/roles/fetch_backups/tasks/main.yml +++ b/roles/fetch_backups/tasks/main.yml @@ -28,6 +28,7 @@ - name: Fetch the manifest from S3 aws_s3: mode: get + overwrite: different bucket: "{{ atl_backup_manifest_bucket }}" object: "{{ atl_backup_manifest_path }}" dest: "{{ atl_backup_manifest_dest }}" @@ -55,6 +56,7 @@ - name: Fetch DB backup from S3 aws_s3: mode: get + overwrite: different bucket: "{{ atl_backup_manifest.db_dump | urlsplit('hostname') }}" object: "{{ atl_backup_manifest.db_dump | urlsplit('path') }}" dest: "{{ atl_backup_db_dest }}" @@ -62,6 +64,7 @@ - name: Fetch Home backup from S3 aws_s3: mode: get + overwrite: different bucket: "{{ atl_backup_manifest.shared_home_dump | urlsplit('hostname') }}" object: "{{ atl_backup_manifest.shared_home_dump | urlsplit('path') }}" dest: "{{ atl_backup_home_dest }}" From 0400d8943d557f9f366dedde768d2e4afc47c275 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 9 Oct 2019 15:29:58 +1100 Subject: [PATCH 007/109] DCD-686: Only restore DB when it is a new one. --- roles/database_init/tasks/main.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/roles/database_init/tasks/main.yml b/roles/database_init/tasks/main.yml index ec5ce38..aa8da99 100644 --- a/roles/database_init/tasks/main.yml +++ b/roles/database_init/tasks/main.yml @@ -34,7 +34,7 @@ lc_collate: "{{ atl_jdbc_collation }}" lc_ctype: "{{ atl_jdbc_ctype }}" template: "{{ atl_jdbc_template }}" - when: atl_backup_db_dest is not defined + register: db_create - name: Restore application database postgresql_db: @@ -51,8 +51,7 @@ # Depends on fetch_backup roles state: restore target: "{{ atl_backup_db_dest }}" - target_opts: "--no-owner --role={{ atl_jdbc_user }}" - when: atl_backup_db_dest is defined + when: db_create.changed and atl_backup_db_dest is defined tags: - new_only From 72659de9ad8de6238eaaefa2d65c1cb1cc7ba564 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 10 Oct 2019 09:30:45 +1100 Subject: [PATCH 008/109] DCD-686: Install Postgres only if restoration is required. --- bin/install-ansible | 2 -- roles/fetch_backups/tasks/amazon.yml | 7 +++++++ roles/fetch_backups/tasks/main.yml | 3 +++ 3 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 roles/fetch_backups/tasks/amazon.yml diff --git a/bin/install-ansible b/bin/install-ansible index 98189a5..196562e 100755 --- a/bin/install-ansible +++ b/bin/install-ansible @@ -4,9 +4,7 @@ set -e source /etc/os-release if [[ $ID = "amzn" ]]; then - amazon-linux-extras enable postgresql9.6 yum install -y \ - postgresql \ python3-devel \ python3-pip \ python2-boto3 \ diff --git a/roles/fetch_backups/tasks/amazon.yml b/roles/fetch_backups/tasks/amazon.yml new file mode 100644 index 0000000..bf32125 --- /dev/null +++ b/roles/fetch_backups/tasks/amazon.yml @@ -0,0 +1,7 @@ +--- + +# Amazon Linux 2 supplies extra packages via a special command. +- name: Enable Postgresql from 'extras' + command: amazon-linux-extras install -y "postgresql{{ postgres_version }}" + args: + creates: /usr/bin/psql diff --git a/roles/fetch_backups/tasks/main.yml b/roles/fetch_backups/tasks/main.yml index 16b18b8..27c5efe 100644 --- a/roles/fetch_backups/tasks/main.yml +++ b/roles/fetch_backups/tasks/main.yml @@ -69,4 +69,7 @@ object: "{{ atl_backup_manifest.shared_home_dump | urlsplit('path') }}" dest: "{{ atl_backup_home_dest }}" + - name: Install distro-specific restore support packages + include_tasks: "{{ ansible_distribution|lower }}.yml" + when: atl_backup_manifest_url is defined and atl_backup_manifest_url != '' From dddf3a86ec4320dd432778cf76a47f1ccca137db Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 10 Oct 2019 09:39:50 +1100 Subject: [PATCH 009/109] DCD-686: Add comment about manifest format. --- roles/fetch_backups/tasks/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/fetch_backups/tasks/main.yml b/roles/fetch_backups/tasks/main.yml index 27c5efe..a5a87de 100644 --- a/roles/fetch_backups/tasks/main.yml +++ b/roles/fetch_backups/tasks/main.yml @@ -40,6 +40,8 @@ dest: "{{ atl_backup_manifest_dest }}" when: atl_backup_manifest_url.scheme != 's3' + # FIXME: The manifest format is still undecided; everything + # referencing this variable should be considered a placeholder. - name: Load parameters from manifest include_vars: file: "{{ atl_backup_manifest_dest }}" From 6f56925fa12c8646e733574a32e37e1eaaafd9a3 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 10 Oct 2019 11:05:20 +1100 Subject: [PATCH 010/109] DCD-686: Move restore operations into the fetch role for the time being. --- roles/database_init/tasks/main.yml | 19 +-------- roles/fetch_backups/defaults/main.yml | 4 ++ roles/fetch_backups/tasks/main.yml | 56 ++++++++++++++++++++++++++- 3 files changed, 59 insertions(+), 20 deletions(-) create mode 100644 roles/fetch_backups/defaults/main.yml diff --git a/roles/database_init/tasks/main.yml b/roles/database_init/tasks/main.yml index aa8da99..8827f99 100644 --- a/roles/database_init/tasks/main.yml +++ b/roles/database_init/tasks/main.yml @@ -34,24 +34,7 @@ lc_collate: "{{ atl_jdbc_collation }}" lc_ctype: "{{ atl_jdbc_ctype }}" template: "{{ atl_jdbc_template }}" - register: db_create - - - name: Restore application database - postgresql_db: - login_host: "{{ atl_db_host }}" - login_user: "{{ atl_db_root_user }}" - login_password: "{{ atl_db_root_password }}" - port: "{{ atl_db_port }}" - name: "{{ atl_jdbc_db_name }}" - owner: "{{ atl_jdbc_user }}" - encoding: "{{ atl_jdbc_encoding }}" - lc_collate: "{{ atl_jdbc_collation }}" - lc_ctype: "{{ atl_jdbc_ctype }}" - template: "{{ atl_jdbc_template }}" - # Depends on fetch_backup roles - state: restore - target: "{{ atl_backup_db_dest }}" - when: db_create.changed and atl_backup_db_dest is defined + register: db_created tags: - new_only diff --git a/roles/fetch_backups/defaults/main.yml b/roles/fetch_backups/defaults/main.yml new file mode 100644 index 0000000..6561c7e --- /dev/null +++ b/roles/fetch_backups/defaults/main.yml @@ -0,0 +1,4 @@ +--- + +atl_backup_home_restore_canary_filename: ".slingshot_home_restore" +atl_backup_home_restore_canary_path: "{{ atl_product_home_shared }}/{{ atl_backup_home_restore_canary_filename }}" diff --git a/roles/fetch_backups/tasks/main.yml b/roles/fetch_backups/tasks/main.yml index a5a87de..5dcdf01 100644 --- a/roles/fetch_backups/tasks/main.yml +++ b/roles/fetch_backups/tasks/main.yml @@ -3,6 +3,15 @@ # This role will attempt to fetch and load the backup manifest from a # remote HTTP or S3 URL. On successful completion the contents of JSON # or YAML document will be in the var `atl_backup_manifest`. +# +# PREREQUISITES: +# * `atl_backup_manifest_url` points at the manifest. +# * The shared home filesystem is mounted if necessary (e.g. NFS/EFS). +# * The database has been created and the variable `db_created` is +# registered with the result (i.e: `register: db_created`). +# +# NOTE: The actual DB/FS restore operations could potentially be split +# out into discrete roles, but currently that is not required. - block: @@ -40,8 +49,6 @@ dest: "{{ atl_backup_manifest_dest }}" when: atl_backup_manifest_url.scheme != 's3' - # FIXME: The manifest format is still undecided; everything - # referencing this variable should be considered a placeholder. - name: Load parameters from manifest include_vars: file: "{{ atl_backup_manifest_dest }}" @@ -49,6 +56,9 @@ - name: Define the DB and home dump destinations set_fact: + # FIXME: The manifest format is still undecided so the + # following usages will need to be updated once it settles.. + atl_backup_id: "{{ atl_backup_manifest.name }}" atl_backup_db_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.db_dump | basename }}" atl_backup_home_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.shared_home_dump | basename }}" @@ -74,4 +84,46 @@ - name: Install distro-specific restore support packages include_tasks: "{{ ansible_distribution|lower }}.yml" + + - name: Restore application database + postgresql_db: + login_host: "{{ atl_db_host }}" + login_user: "{{ atl_db_root_user }}" + login_password: "{{ atl_db_root_password }}" + port: "{{ atl_db_port }}" + name: "{{ atl_jdbc_db_name }}" + owner: "{{ atl_jdbc_user }}" + encoding: "{{ atl_jdbc_encoding }}" + lc_collate: "{{ atl_jdbc_collation }}" + lc_ctype: "{{ atl_jdbc_ctype }}" + template: "{{ atl_jdbc_template }}" + # Depends on fetch_backup roles + state: restore + target: "{{ atl_backup_db_dest }}" + when: db_created.changed and atl_backup_db_dest is defined + + + - name: Check for the restore canary file + stat: + path: "{{ atl_backup_home_restore_canary_path }}" + register: restore_canary + + - name: Create shared home if necessary + file: + path: "{{ atl_product_home_shared }}" + state: directory + mode: 0750 + owner: "{{ atl_product_user }}" + group: "{{ atl_product_user }}" + when: restore_canary.stat.exists + + - name: Restore the shared-home backup + unarchive: + path: "{{ atl_backup_home_restore_canary_path }}" + dest: "{{ atl_product_home_shared }}" + owner: "{{ atl_product_user }}" + group: "{{ atl_product_user }}" + when: restore_canary.stat.exists + + when: atl_backup_manifest_url is defined and atl_backup_manifest_url != '' From 93c359d0295b84854c8a4124d5578fde829cd916 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 10 Oct 2019 11:12:18 +1100 Subject: [PATCH 011/109] DCD-686: Rename role to reflect updated functionality and exand restore functionality. --- aws_jira_dc_node.yml | 2 +- .../defaults/main.yml | 0 .../tasks/amazon.yml | 0 .../tasks/main.yml | 37 +++++++++++-------- 4 files changed, 23 insertions(+), 16 deletions(-) rename roles/{fetch_backups => restore_backups}/defaults/main.yml (100%) rename roles/{fetch_backups => restore_backups}/tasks/amazon.yml (100%) rename roles/{fetch_backups => restore_backups}/tasks/main.yml (84%) diff --git a/aws_jira_dc_node.yml b/aws_jira_dc_node.yml index 6ef0765..b0470fb 100644 --- a/aws_jira_dc_node.yml +++ b/aws_jira_dc_node.yml @@ -21,9 +21,9 @@ - role: linux_common - role: aws_common - role: aws_shared_fs_config - - role: fetch_backups - role: product_common - role: product_install - role: database_init + - role: restore_backups - role: jira_config - role: product_startup diff --git a/roles/fetch_backups/defaults/main.yml b/roles/restore_backups/defaults/main.yml similarity index 100% rename from roles/fetch_backups/defaults/main.yml rename to roles/restore_backups/defaults/main.yml diff --git a/roles/fetch_backups/tasks/amazon.yml b/roles/restore_backups/tasks/amazon.yml similarity index 100% rename from roles/fetch_backups/tasks/amazon.yml rename to roles/restore_backups/tasks/amazon.yml diff --git a/roles/fetch_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml similarity index 84% rename from roles/fetch_backups/tasks/main.yml rename to roles/restore_backups/tasks/main.yml index 5dcdf01..a0023cf 100644 --- a/roles/fetch_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -108,22 +108,29 @@ path: "{{ atl_backup_home_restore_canary_path }}" register: restore_canary - - name: Create shared home if necessary - file: - path: "{{ atl_product_home_shared }}" - state: directory - mode: 0750 - owner: "{{ atl_product_user }}" - group: "{{ atl_product_user }}" - when: restore_canary.stat.exists + - block: - - name: Restore the shared-home backup - unarchive: - path: "{{ atl_backup_home_restore_canary_path }}" - dest: "{{ atl_product_home_shared }}" - owner: "{{ atl_product_user }}" - group: "{{ atl_product_user }}" - when: restore_canary.stat.exists + - name: Create shared home if necessary + file: + path: "{{ atl_product_home_shared }}" + state: directory + mode: 0750 + owner: "{{ atl_product_user }}" + group: "{{ atl_product_user }}" + + - name: Restore the shared-home backup + unarchive: + path: "{{ atl_backup_home_restore_canary_path }}" + dest: "{{ atl_product_home_shared }}" + owner: "{{ atl_product_user }}" + group: "{{ atl_product_user }}" + + - name: Create restore-canary if necessary + copy: + dest: "{{ atl_backup_home_restore_canary_path }}" + content: "{{ atl_backup_id }}" + + when: not restore_canary.stat.exists when: atl_backup_manifest_url is defined and atl_backup_manifest_url != '' From 8a343f7e8c66e4b4fd5b3b519495b76108c716d2 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 10 Oct 2019 11:26:47 +1100 Subject: [PATCH 012/109] DCD-686: Fix unarchive operation. --- roles/restore_backups/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index a0023cf..019c5d5 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -120,7 +120,7 @@ - name: Restore the shared-home backup unarchive: - path: "{{ atl_backup_home_restore_canary_path }}" + src: "{{ atl_backup_home_restore_canary_path }}" dest: "{{ atl_product_home_shared }}" owner: "{{ atl_product_user }}" group: "{{ atl_product_user }}" From 8254482761e5b73b1b420fd91c59495cf639a95d Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 10 Oct 2019 11:28:56 +1100 Subject: [PATCH 013/109] DCD-686: Fix tarball location. --- roles/restore_backups/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index 019c5d5..f53b685 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -120,7 +120,7 @@ - name: Restore the shared-home backup unarchive: - src: "{{ atl_backup_home_restore_canary_path }}" + src: "{{ atl_backup_home_dest }}" dest: "{{ atl_product_home_shared }}" owner: "{{ atl_product_user }}" group: "{{ atl_product_user }}" From 46a2bad4b8ce01ead1c4ba77f1532836fc6288e8 Mon Sep 17 00:00:00 2001 From: Varun Arbatti <1063972+theghostwhoforks@users.noreply.github.com> Date: Thu, 10 Oct 2019 14:51:06 +1100 Subject: [PATCH 014/109] DCD-742: Updates manifest structure in ansible --- roles/restore_backups/tasks/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index f53b685..d12d073 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -59,8 +59,8 @@ # FIXME: The manifest format is still undecided so the # following usages will need to be updated once it settles.. atl_backup_id: "{{ atl_backup_manifest.name }}" - atl_backup_db_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.db_dump | basename }}" - atl_backup_home_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.shared_home_dump | basename }}" + atl_backup_db_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.artifacts.db.location.value | basename }}" + atl_backup_home_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.artifacts.shared_home.location.value | basename }}" # FIXME: Here we fetch the backups. However we may wish to stream # these directly from S3 to the target DB/FS to avoid requiring @@ -69,16 +69,16 @@ aws_s3: mode: get overwrite: different - bucket: "{{ atl_backup_manifest.db_dump | urlsplit('hostname') }}" - object: "{{ atl_backup_manifest.db_dump | urlsplit('path') }}" + bucket: "{{ atl_backup_manifest.artifacts.db.location.value | urlsplit('hostname') }}" + object: "{{ atl_backup_manifest.artifacts.db.location.value | urlsplit('path') }}" dest: "{{ atl_backup_db_dest }}" - name: Fetch Home backup from S3 aws_s3: mode: get overwrite: different - bucket: "{{ atl_backup_manifest.shared_home_dump | urlsplit('hostname') }}" - object: "{{ atl_backup_manifest.shared_home_dump | urlsplit('path') }}" + bucket: "{{ atl_backup_manifest.artifacts.shared_home.location.value | urlsplit('hostname') }}" + object: "{{ atl_backup_manifest.artifacts.shared_home.location.value | urlsplit('path') }}" dest: "{{ atl_backup_home_dest }}" - name: Install distro-specific restore support packages From 054e171da3c8c925534d979cbe60bfebafd5f589 Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Thu, 10 Oct 2019 16:12:06 +1100 Subject: [PATCH 015/109] ITOPSENG-164 Save the binary to shared home adding a locking mechanism to avoid potential race condition. This also allows downloading of old binaries once no longer available to download --- roles/product_install/defaults/main.yml | 4 + roles/product_install/tasks/main.yml | 79 ++++++++++++++++--- .../tasks/unpack_installer.yml | 2 +- 3 files changed, 74 insertions(+), 11 deletions(-) diff --git a/roles/product_install/defaults/main.yml b/roles/product_install/defaults/main.yml index 211e76e..0741a94 100644 --- a/roles/product_install/defaults/main.yml +++ b/roles/product_install/defaults/main.yml @@ -22,6 +22,10 @@ atl_product_download_filename: "{{ atl_download_edition | default(atl_product_ed atl_product_download: "{{ atl_installer_temp }}/{{ atl_product_download_filename }}" atl_product_varfile: "{{ atl_installer_temp }}/{{ atl_product_family }}.varfile" +atl_product_home_shared_download_dir: "{{ atl_product_home_shared }}/downloads" +atl_product_home_shared_download: "{{ atl_product_home_shared_download_dir }}/{{ atl_product_download_filename }}" +atl_product_home_shared_download_lockdir: "{{ atl_product_home_shared_download }}_downloaded" + atl_marketplace_base: "https://marketplace.atlassian.com" atl_servicedesk_latest_url: "https://marketplace.atlassian.com/rest/2/products/key/jira-servicedesk/versions/latest" atl_servicedesk_versioned_url: "https://marketplace.atlassian.com/rest/2/products/key/jira-servicedesk/versions/name/{{ atl_product_version }}" diff --git a/roles/product_install/tasks/main.yml b/roles/product_install/tasks/main.yml index 77371fa..94cd6ba 100644 --- a/roles/product_install/tasks/main.yml +++ b/roles/product_install/tasks/main.yml @@ -116,6 +116,7 @@ - "{{ atl_product_home }}" - "{{ atl_product_installation_versioned }}" - "{{ atl_product_version_cache_dir }}" + - "{{ atl_product_home_shared_download_dir }}" changed_when: false # For Molecule idempotence check # At this point atl_product_version should be set, cache if necessary. @@ -125,17 +126,75 @@ dest: "{{ atl_product_version_cache }}" force: true +# For the first run a temp binary should be downloaded but moved to shared home to ensure all subsequent nodes have access +# to the same specific version binary. +# To prevent a race condition with multiple downloads at the same time a directory is used as a lockfile (atomic operation). -# Note: We don't the cache binary in the shared drive to the complexity -# around download race-conditions if multiple nodes are starting at -# the same time. When downloading from product-downloads.atlassian.com -# (which is a CDN) takes seconds anyway. -- name: Fetch product installer - get_url: - url: "{{ atl_product_download_url }}" - dest: "{{ atl_product_download }}" - mode: 0755 - force: false +- name: Assume temp binary should be downloaded + set_fact: + download_binary: yes + +# Check for product installer in home_shared and lockdir to determine if it needs to be downloaded again. +- name: Check download lock directory exists + stat: + path: "{{ atl_product_home_shared_download_lockdir }}" + register: download_lockdir + +- name: Check for presence of product installer in home_shared + stat: + path: "{{ atl_product_home_shared_download }}" + register: home_shared_downloaded + +# If binary exists and lockdir exists use this binary instead +- name: Check Lock Directory and binary exists on shared_home + set_fact: + download_binary: no + when: + - home_shared_downloaded.stat.exists + - download_lockdir.stat.isdir is defined + - download_lockdir.stat.isdir + +# If the binary was never installed, download it +- name: "Download product installer and move to shared directory" + block: + + - name: Fetch product installer + get_url: + url: "{{ atl_product_download_url }}" + dest: "{{ atl_product_download }}" + mode: 0755 + force: false + register: atl_product_downloaded + + - name: Remove lockdir to prevent nodes relying on binary when copying + file: + path: "{{ atl_product_home_shared_download_lockdir }}" + state: absent + when: atl_product_downloaded is succeeded + register: lockdir_removed + + - name: Copy temp installer to home_shared + copy: + src: "{{ atl_product_download }}" + dest: "{{ atl_product_home_shared_download }}" + remote_src: yes + when: lockdir_removed is succeeded + register: copied + + - name: Delete old temp installer + file: + path: "{{ atl_product_download }}" + state: absent + when: copied is succeeded + register: temp_deleted + + - name: Create lockdir once product installer downloaded and moved + file: + path: "{{ atl_product_home_shared_download_lockdir }}" + state: directory + when: temp_deleted is succeeded + + when: download_binary - name: Unpack the downloaded application depending on format include_tasks: "unpack_{{ atl_download_format }}.yml" diff --git a/roles/product_install/tasks/unpack_installer.yml b/roles/product_install/tasks/unpack_installer.yml index 925dca0..11d37e3 100644 --- a/roles/product_install/tasks/unpack_installer.yml +++ b/roles/product_install/tasks/unpack_installer.yml @@ -11,7 +11,7 @@ # will create 'jira1'; this potentially creates idempotency/upgrade # issues down the line. - name: Run the installer - command: /bin/sh "{{ atl_product_download }}" -q -varfile "{{ atl_product_varfile }}" + command: /bin/sh "{{ atl_product_home_shared_download }}" -q -varfile "{{ atl_product_varfile }}" args: creates: "{{ atl_product_installation_versioned }}/.install4j/" become: true From 3ed48340b3553a811fdc948c467e902f3a3c1e1a Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Thu, 10 Oct 2019 16:52:51 +1100 Subject: [PATCH 016/109] ITOPSENG-164 Yamllint fix --- roles/product_install/tasks/main.yml | 81 ++++++++++++++-------------- 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/roles/product_install/tasks/main.yml b/roles/product_install/tasks/main.yml index 94cd6ba..8a90eb3 100644 --- a/roles/product_install/tasks/main.yml +++ b/roles/product_install/tasks/main.yml @@ -126,15 +126,18 @@ dest: "{{ atl_product_version_cache }}" force: true -# For the first run a temp binary should be downloaded but moved to shared home to ensure all subsequent nodes have access +# For the first run a temp binary should be downloaded but moved to +# shared home to ensure all subsequent nodes have access # to the same specific version binary. -# To prevent a race condition with multiple downloads at the same time a directory is used as a lockfile (atomic operation). +# To prevent a race condition with multiple downloads at the same time +# a directory is used as a lockfile (atomic operation). - name: Assume temp binary should be downloaded set_fact: - download_binary: yes + download_binary: true -# Check for product installer in home_shared and lockdir to determine if it needs to be downloaded again. +# Check for product installer in home_shared and lockdir to determine +# if it needs to be downloaded again. - name: Check download lock directory exists stat: path: "{{ atl_product_home_shared_download_lockdir }}" @@ -148,51 +151,51 @@ # If binary exists and lockdir exists use this binary instead - name: Check Lock Directory and binary exists on shared_home set_fact: - download_binary: no + download_binary: false when: - - home_shared_downloaded.stat.exists - - download_lockdir.stat.isdir is defined - - download_lockdir.stat.isdir + - home_shared_downloaded.stat.exists + - download_lockdir.stat.isdir is defined + - download_lockdir.stat.isdir # If the binary was never installed, download it - name: "Download product installer and move to shared directory" block: - - name: Fetch product installer - get_url: - url: "{{ atl_product_download_url }}" - dest: "{{ atl_product_download }}" - mode: 0755 - force: false - register: atl_product_downloaded + - name: Fetch product installer + get_url: + url: "{{ atl_product_download_url }}" + dest: "{{ atl_product_download }}" + mode: 0755 + force: false + register: atl_product_downloaded - - name: Remove lockdir to prevent nodes relying on binary when copying - file: - path: "{{ atl_product_home_shared_download_lockdir }}" - state: absent - when: atl_product_downloaded is succeeded - register: lockdir_removed + - name: Remove lockdir to prevent nodes relying on binary when copying + file: + path: "{{ atl_product_home_shared_download_lockdir }}" + state: absent + when: atl_product_downloaded is succeeded + register: lockdir_removed - - name: Copy temp installer to home_shared - copy: - src: "{{ atl_product_download }}" - dest: "{{ atl_product_home_shared_download }}" - remote_src: yes - when: lockdir_removed is succeeded - register: copied + - name: Copy temp installer to home_shared + copy: + src: "{{ atl_product_download }}" + dest: "{{ atl_product_home_shared_download }}" + remote_src: true + when: lockdir_removed is succeeded + register: copied - - name: Delete old temp installer - file: - path: "{{ atl_product_download }}" - state: absent - when: copied is succeeded - register: temp_deleted + - name: Delete old temp installer + file: + path: "{{ atl_product_download }}" + state: absent + when: copied is succeeded + register: temp_deleted - - name: Create lockdir once product installer downloaded and moved - file: - path: "{{ atl_product_home_shared_download_lockdir }}" - state: directory - when: temp_deleted is succeeded + - name: Create lockdir once product installer downloaded and moved + file: + path: "{{ atl_product_home_shared_download_lockdir }}" + state: directory + when: temp_deleted is succeeded when: download_binary From b3dffa684f112196438ed29830556ce2d573c47b Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Fri, 11 Oct 2019 14:20:21 +1100 Subject: [PATCH 017/109] ITOPSENG-164 Further testing required for logic change --- roles/product_install/defaults/main.yml | 5 +- roles/product_install/tasks/main.yml | 136 +++++++++++------- .../tasks/unpack_installer.yml | 4 +- 3 files changed, 93 insertions(+), 52 deletions(-) diff --git a/roles/product_install/defaults/main.yml b/roles/product_install/defaults/main.yml index 0741a94..8798ac7 100644 --- a/roles/product_install/defaults/main.yml +++ b/roles/product_install/defaults/main.yml @@ -19,12 +19,13 @@ atl_product_base_url: "{{ atl_release_base_url }}/{{ atl_product_family }}/downl atl_product_download_url: "{{ atl_product_base_url }}/atlassian-{{ atl_download_edition | default(atl_product_edition) }}-{{ atl_product_version }}{{ atl_download_suffix }}" atl_product_download_filename: "{{ atl_download_edition | default(atl_product_edition) }}.{{ atl_product_version }}{{ atl_download_suffix }}" -atl_product_download: "{{ atl_installer_temp }}/{{ atl_product_download_filename }}" +atl_product_temp_download: "{{ atl_installer_temp }}/{{ atl_product_download_filename }}" atl_product_varfile: "{{ atl_installer_temp }}/{{ atl_product_family }}.varfile" atl_product_home_shared_download_dir: "{{ atl_product_home_shared }}/downloads" atl_product_home_shared_download: "{{ atl_product_home_shared_download_dir }}/{{ atl_product_download_filename }}" -atl_product_home_shared_download_lockdir: "{{ atl_product_home_shared_download }}_downloaded" +atl_product_home_shared_lockdir_moving: "{{ atl_product_home_shared_download }}_moving" +atl_product_home_shared_lockdir_downloaded: "{{ atl_product_home_shared_download }}_downloaded" atl_marketplace_base: "https://marketplace.atlassian.com" atl_servicedesk_latest_url: "https://marketplace.atlassian.com/rest/2/products/key/jira-servicedesk/versions/latest" diff --git a/roles/product_install/tasks/main.yml b/roles/product_install/tasks/main.yml index 8a90eb3..0d78a2c 100644 --- a/roles/product_install/tasks/main.yml +++ b/roles/product_install/tasks/main.yml @@ -132,73 +132,52 @@ # To prevent a race condition with multiple downloads at the same time # a directory is used as a lockfile (atomic operation). -- name: Assume temp binary should be downloaded +- name: Assume temp binary should be downloaded and used set_fact: download_binary: true + atl_product_download: "{{ atl_product_temp_download }}" # Check for product installer in home_shared and lockdir to determine # if it needs to be downloaded again. -- name: Check download lock directory exists +- name: Check moving lock directory does not exist stat: - path: "{{ atl_product_home_shared_download_lockdir }}" - register: download_lockdir + path: "{{ atl_product_home_shared_moving_lock }}" + register: moving_lock + +- name: Check downloaded lock directory exists + stat: + path: "{{ atl_product_home_shared_downloaded_lock }}" + register: downloaded_lock - name: Check for presence of product installer in home_shared stat: path: "{{ atl_product_home_shared_download }}" - register: home_shared_downloaded + register: home_shared_download # If binary exists and lockdir exists use this binary instead -- name: Check Lock Directory and binary exists on shared_home +- name: Check lock directory and binary exists on shared_home set_fact: download_binary: false + atl_product_download: "{{ atl_product_home_shared_download }}" when: - - home_shared_downloaded.stat.exists - - download_lockdir.stat.isdir is defined - - download_lockdir.stat.isdir - -# If the binary was never installed, download it -- name: "Download product installer and move to shared directory" - block: - - - name: Fetch product installer - get_url: - url: "{{ atl_product_download_url }}" - dest: "{{ atl_product_download }}" - mode: 0755 - force: false - register: atl_product_downloaded - - - name: Remove lockdir to prevent nodes relying on binary when copying - file: - path: "{{ atl_product_home_shared_download_lockdir }}" - state: absent - when: atl_product_downloaded is succeeded - register: lockdir_removed - - - name: Copy temp installer to home_shared - copy: - src: "{{ atl_product_download }}" - dest: "{{ atl_product_home_shared_download }}" - remote_src: true - when: lockdir_removed is succeeded - register: copied - - - name: Delete old temp installer - file: - path: "{{ atl_product_download }}" - state: absent - when: copied is succeeded - register: temp_deleted - - - name: Create lockdir once product installer downloaded and moved - file: - path: "{{ atl_product_home_shared_download_lockdir }}" - state: directory - when: temp_deleted is succeeded + - home_shared_download.stat.exists + - downloaded_lock.stat.isdir is defined + - downloaded_lock.stat.isdir + - ( moving_lock.stat.isdir is not defined or moving_lock.stat.isdir == False ) +# If the binary was never installed, download it to temp location +- name: Installer not on home_shared. Fetch it. + get_url: + url: "{{ atl_product_download_url }}" + dest: "{{ atl_product_temp_download }}" + mode: 0755 + force: false + register: atl_product_downloaded when: download_binary +# If product installer was fetched to temp, install from there +# If product installer was pre-downloaded on shared_home, install from there +# This is determined by {{ atl_product_download }} variable - name: Unpack the downloaded application depending on format include_tasks: "unpack_{{ atl_download_format }}.yml" @@ -208,3 +187,62 @@ dest: "{{ atl_product_installation_current }}" state: link force: true + +# # Product is installed. If the following are true, move to home_shared +# # 1. This node just downloaded binary. +# # 2. Another node is not already moving into place. +# - name: "Move product installer" +# block: + +# - name: Check moving lock directory does not exist +# stat: +# path: "{{ atl_product_home_shared_moving_lock }}" +# register: moving_lock + +# - name: Check downloaded lock directory exists +# stat: +# path: "{{ atl_product_home_shared_downloaded_lock }}" +# register: downloaded_lock + +# - name: Check for presence of product installer in home_shared +# stat: +# path: "{{ atl_product_home_shared_download }}" +# register: home_shared_download + + + + + + + +# - name: Remove lockdir to prevent nodes relying on binary when copying +# file: +# path: "{{ atl_product_home_shared_download_lockdir }}" +# state: absent +# when: atl_product_downloaded is succeeded +# register: lockdir_removed + +# - name: Copy temp installer to home_shared +# copy: +# src: "{{ atl_product_download }}" +# dest: "{{ atl_product_home_shared_download }}" +# remote_src: true +# when: lockdir_removed is succeeded +# register: copied + +# - name: Delete old temp installer +# file: +# path: "{{ atl_product_download }}" +# state: absent +# when: copied is succeeded +# register: temp_deleted + +# - name: Create lockdir once product installer downloaded and moved +# file: +# path: "{{ atl_product_home_shared_download_lockdir }}" +# state: directory +# when: temp_deleted is succeeded + + + + diff --git a/roles/product_install/tasks/unpack_installer.yml b/roles/product_install/tasks/unpack_installer.yml index 11d37e3..f340463 100644 --- a/roles/product_install/tasks/unpack_installer.yml +++ b/roles/product_install/tasks/unpack_installer.yml @@ -10,8 +10,10 @@ # actions. For example, if root and the 'jira' user exists then it # will create 'jira1'; this potentially creates idempotency/upgrade # issues down the line. +# The variable {{ atl_product_download }} will be on temp for first nodes and shared_home for +# subsequent nodes. - name: Run the installer - command: /bin/sh "{{ atl_product_home_shared_download }}" -q -varfile "{{ atl_product_varfile }}" + command: /bin/sh "{{ atl_product_download }}" -q -varfile "{{ atl_product_varfile }}" args: creates: "{{ atl_product_installation_versioned }}/.install4j/" become: true From 894501e9d191feb7b069b15b1c45bac5c3bcd38d Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Fri, 11 Oct 2019 14:32:50 +1100 Subject: [PATCH 018/109] ITOPSENG-164 Additional default variables --- roles/product_install/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/product_install/defaults/main.yml b/roles/product_install/defaults/main.yml index 8798ac7..61d8d04 100644 --- a/roles/product_install/defaults/main.yml +++ b/roles/product_install/defaults/main.yml @@ -24,8 +24,8 @@ atl_product_varfile: "{{ atl_installer_temp }}/{{ atl_product_family }}.varfile" atl_product_home_shared_download_dir: "{{ atl_product_home_shared }}/downloads" atl_product_home_shared_download: "{{ atl_product_home_shared_download_dir }}/{{ atl_product_download_filename }}" -atl_product_home_shared_lockdir_moving: "{{ atl_product_home_shared_download }}_moving" -atl_product_home_shared_lockdir_downloaded: "{{ atl_product_home_shared_download }}_downloaded" +atl_product_home_shared_moving_lock: "{{ atl_product_home_shared_download }}_moving" +atl_product_home_shared_downloaded_lock: "{{ atl_product_home_shared_download }}_downloaded" atl_marketplace_base: "https://marketplace.atlassian.com" atl_servicedesk_latest_url: "https://marketplace.atlassian.com/rest/2/products/key/jira-servicedesk/versions/latest" From b73381e907d125f304efe65107ff20443db082b9 Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Fri, 11 Oct 2019 14:52:35 +1100 Subject: [PATCH 019/109] ITOPSENG-164 Further testing required for logic change --- roles/product_install/tasks/main.yml | 129 +++++++++++++++------------ 1 file changed, 73 insertions(+), 56 deletions(-) diff --git a/roles/product_install/tasks/main.yml b/roles/product_install/tasks/main.yml index 0d78a2c..a23f98b 100644 --- a/roles/product_install/tasks/main.yml +++ b/roles/product_install/tasks/main.yml @@ -130,26 +130,27 @@ # shared home to ensure all subsequent nodes have access # to the same specific version binary. # To prevent a race condition with multiple downloads at the same time -# a directory is used as a lockfile (atomic operation). +# a directory is used as a lockfile (atomic operation) when moving binary. -- name: Assume temp binary should be downloaded and used +- name: Set assumptions to avoid race condition set_fact: download_binary: true + move_binary: false atl_product_download: "{{ atl_product_temp_download }}" # Check for product installer in home_shared and lockdir to determine # if it needs to be downloaded again. -- name: Check moving lock directory does not exist +- name: Check for moving lock directory stat: path: "{{ atl_product_home_shared_moving_lock }}" register: moving_lock -- name: Check downloaded lock directory exists +- name: Check for downloaded lock directory stat: path: "{{ atl_product_home_shared_downloaded_lock }}" register: downloaded_lock -- name: Check for presence of product installer in home_shared +- name: Check for product installer in home_shared stat: path: "{{ atl_product_home_shared_download }}" register: home_shared_download @@ -188,60 +189,76 @@ state: link force: true -# # Product is installed. If the following are true, move to home_shared -# # 1. This node just downloaded binary. -# # 2. Another node is not already moving into place. -# - name: "Move product installer" -# block: +# Product is installed. If the following are true, move to home_shared +# 1. This node just downloaded binary. +# 2. Another node is not already moving into place. +- name: "Move product installer" + block: -# - name: Check moving lock directory does not exist -# stat: -# path: "{{ atl_product_home_shared_moving_lock }}" -# register: moving_lock + - name: Check again for moving lock directory + stat: + path: "{{ atl_product_home_shared_moving_lock }}" + register: moving_lock_2 -# - name: Check downloaded lock directory exists -# stat: -# path: "{{ atl_product_home_shared_downloaded_lock }}" -# register: downloaded_lock + - name: Check again for downloaded lock directory + stat: + path: "{{ atl_product_home_shared_downloaded_lock }}" + register: downloaded_lock_2 -# - name: Check for presence of product installer in home_shared -# stat: -# path: "{{ atl_product_home_shared_download }}" -# register: home_shared_download - - - - - - - -# - name: Remove lockdir to prevent nodes relying on binary when copying -# file: -# path: "{{ atl_product_home_shared_download_lockdir }}" -# state: absent -# when: atl_product_downloaded is succeeded -# register: lockdir_removed - -# - name: Copy temp installer to home_shared -# copy: -# src: "{{ atl_product_download }}" -# dest: "{{ atl_product_home_shared_download }}" -# remote_src: true -# when: lockdir_removed is succeeded -# register: copied - -# - name: Delete old temp installer -# file: -# path: "{{ atl_product_download }}" -# state: absent -# when: copied is succeeded -# register: temp_deleted - -# - name: Create lockdir once product installer downloaded and moved -# file: -# path: "{{ atl_product_home_shared_download_lockdir }}" -# state: directory -# when: temp_deleted is succeeded + - name: Check again for product installer in home_shared + stat: + path: "{{ atl_product_home_shared_download }}" + register: home_shared_download_2 + + # If binary exists and lockdir exists use this binary instead + - name: Check lock directory and binary exists on shared_home + set_fact: + move_binary: true + atl_product_download: "{{ atl_product_home_shared_download }}" + when: + - home_shared_download.stat.exists == False + - ( downloaded_lock.stat.isdir is not defined or downloaded_lock.stat.isdir == False ) + - ( moving_lock.stat.isdir is not defined or moving_lock.stat.isdir == False ) + + - name: Create moving_lock to ensure other nodes skip + file: + path: "{{ atl_product_home_shared_moving_lock }}" + state: directory + when: move_binary + register: moving_lock_created + + - name: Copy temp installer to home_shared + copy: + src: "{{ atl_product_temp_download }}" + dest: "{{ atl_product_home_shared_download }}" + remote_src: true + when: moving_lock_created is succeeded + register: copied + + - name: Create downloaded_lock once product installer downloaded and copied + file: + path: "{{ atl_product_home_shared_downloaded_lock }}" + state: directory + when: copied is succeeded + register: downloaded_lock_created + + - name: Remove moving_lock to show that binary is completed + file: + path: "{{ atl_product_home_shared_moving_lock }}" + state: absent + when: + - downloaded_lock_created is succeeded + - copied is succeeded + register: moving_lock_removed + + - name: Delete old temp installer + file: + path: "{{ atl_product_temp_download }}" + state: absent + when: moving_lock_removed is succeeded + register: temp_deleted + + when: move_binary From 5d10948feb5ece672309e3dc58a458386a93180f Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Fri, 11 Oct 2019 14:54:59 +1100 Subject: [PATCH 020/109] ITOPSENG-164 Further testing required for logic change --- roles/product_install/tasks/main.yml | 66 ++++++++++++++-------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/roles/product_install/tasks/main.yml b/roles/product_install/tasks/main.yml index a23f98b..d6c7b61 100644 --- a/roles/product_install/tasks/main.yml +++ b/roles/product_install/tasks/main.yml @@ -220,43 +220,43 @@ - ( downloaded_lock.stat.isdir is not defined or downloaded_lock.stat.isdir == False ) - ( moving_lock.stat.isdir is not defined or moving_lock.stat.isdir == False ) - - name: Create moving_lock to ensure other nodes skip - file: - path: "{{ atl_product_home_shared_moving_lock }}" - state: directory - when: move_binary - register: moving_lock_created + - name: Create moving_lock to ensure other nodes skip + file: + path: "{{ atl_product_home_shared_moving_lock }}" + state: directory + when: move_binary + register: moving_lock_created - - name: Copy temp installer to home_shared - copy: - src: "{{ atl_product_temp_download }}" - dest: "{{ atl_product_home_shared_download }}" - remote_src: true - when: moving_lock_created is succeeded - register: copied + - name: Copy temp installer to home_shared + copy: + src: "{{ atl_product_temp_download }}" + dest: "{{ atl_product_home_shared_download }}" + remote_src: true + when: moving_lock_created is succeeded + register: copied - - name: Create downloaded_lock once product installer downloaded and copied - file: - path: "{{ atl_product_home_shared_downloaded_lock }}" - state: directory - when: copied is succeeded - register: downloaded_lock_created + - name: Create downloaded_lock once product installer downloaded and copied + file: + path: "{{ atl_product_home_shared_downloaded_lock }}" + state: directory + when: copied is succeeded + register: downloaded_lock_created - - name: Remove moving_lock to show that binary is completed - file: - path: "{{ atl_product_home_shared_moving_lock }}" - state: absent - when: - - downloaded_lock_created is succeeded - - copied is succeeded - register: moving_lock_removed + - name: Remove moving_lock to show that binary is completed + file: + path: "{{ atl_product_home_shared_moving_lock }}" + state: absent + when: + - downloaded_lock_created is succeeded + - copied is succeeded + register: moving_lock_removed - - name: Delete old temp installer - file: - path: "{{ atl_product_temp_download }}" - state: absent - when: moving_lock_removed is succeeded - register: temp_deleted + - name: Delete old temp installer + file: + path: "{{ atl_product_temp_download }}" + state: absent + when: moving_lock_removed is succeeded + register: temp_deleted when: move_binary From fbb9316c26eea2383df30345a34b4e6b69da4c69 Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Fri, 11 Oct 2019 16:37:28 +1100 Subject: [PATCH 021/109] ITOPSENG-164 Further testing required for logic change --- roles/product_install/tasks/main.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/roles/product_install/tasks/main.yml b/roles/product_install/tasks/main.yml index d6c7b61..f12f987 100644 --- a/roles/product_install/tasks/main.yml +++ b/roles/product_install/tasks/main.yml @@ -189,10 +189,12 @@ state: link force: true -# Product is installed. If the following are true, move to home_shared +# Temp product was downloaded and installed. +# If the following conditions are true, move to home_shared # 1. This node just downloaded binary. # 2. Another node is not already moving into place. -- name: "Move product installer" +# 3. The binary is downloaded and lockdir in place. +- name: "Check move product installer" block: - name: Check again for moving lock directory @@ -214,11 +216,15 @@ - name: Check lock directory and binary exists on shared_home set_fact: move_binary: true - atl_product_download: "{{ atl_product_home_shared_download }}" when: - - home_shared_download.stat.exists == False - - ( downloaded_lock.stat.isdir is not defined or downloaded_lock.stat.isdir == False ) - - ( moving_lock.stat.isdir is not defined or moving_lock.stat.isdir == False ) + - ( home_shared_download.stat.exists == False or + downloaded_lock.stat.isdir is not defined or downloaded_lock.stat.isdir == False ) + - ( moving_lock.stat.isdir is not defined or moving_lock.stat.isdir == False ) + + when: download_binary + +- name: "Move product installer if required" + block: - name: Create moving_lock to ensure other nodes skip file: From 0a6501d781fe78e17c44982730006f436da3a3dd Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Mon, 14 Oct 2019 17:29:16 +1100 Subject: [PATCH 022/109] ITOPSENG-164 Debug some changes to logic --- roles/product_install/tasks/main.yml | 192 +++++++++++++++------------ 1 file changed, 104 insertions(+), 88 deletions(-) diff --git a/roles/product_install/tasks/main.yml b/roles/product_install/tasks/main.yml index f12f987..040b2ef 100644 --- a/roles/product_install/tasks/main.yml +++ b/roles/product_install/tasks/main.yml @@ -138,17 +138,11 @@ move_binary: false atl_product_download: "{{ atl_product_temp_download }}" -# Check for product installer in home_shared and lockdir to determine -# if it needs to be downloaded again. -- name: Check for moving lock directory +# Check for pre-downloaded binary on shared_home and completed lock dir. +- name: Check for completed lock directory stat: - path: "{{ atl_product_home_shared_moving_lock }}" - register: moving_lock - -- name: Check for downloaded lock directory - stat: - path: "{{ atl_product_home_shared_downloaded_lock }}" - register: downloaded_lock + path: "{{ atl_product_home_shared_completed_lock }}" + register: completed_lock - name: Check for product installer in home_shared stat: @@ -162,9 +156,8 @@ atl_product_download: "{{ atl_product_home_shared_download }}" when: - home_shared_download.stat.exists - - downloaded_lock.stat.isdir is defined - - downloaded_lock.stat.isdir - - ( moving_lock.stat.isdir is not defined or moving_lock.stat.isdir == False ) + - completed_lock.stat.isdir is defined + - completed_lock.stat.isdir # If the binary was never installed, download it to temp location - name: Installer not on home_shared. Fetch it. @@ -173,98 +166,121 @@ dest: "{{ atl_product_temp_download }}" mode: 0755 force: false - register: atl_product_downloaded + register: atl_product_completed when: download_binary -# If product installer was fetched to temp, install from there -# If product installer was pre-downloaded on shared_home, install from there -# This is determined by {{ atl_product_download }} variable -- name: Unpack the downloaded application depending on format - include_tasks: "unpack_{{ atl_download_format }}.yml" +# If product installer was fetched +# Make the moving directory +# - failure, continue and install from temp +# - success, move binary and install from shared_home -- name: Symlink the installed version to current +- name: Create moving_lock. file: - src: "{{ atl_product_installation_versioned }}" - dest: "{{ atl_product_installation_current }}" - state: link - force: true + path: "{{ atl_product_home_shared_moving_lock }}" + state: directory + when: download_binary is succeeded + register: moving_lock_created -# Temp product was downloaded and installed. -# If the following conditions are true, move to home_shared -# 1. This node just downloaded binary. -# 2. Another node is not already moving into place. -# 3. The binary is downloaded and lockdir in place. -- name: "Check move product installer" - block: +- name: Debug Scenario A - lock created + debug: lock created + when: moving_lock_created is succeeded - - name: Check again for moving lock directory - stat: - path: "{{ atl_product_home_shared_moving_lock }}" - register: moving_lock_2 +- name: Debug Scenario B - lock cannot created + debug: lock not created + when: moving_lock_created is failed + + + + + +# # If product installer was pre-downloaded on shared_home, install from there +# # This is determined by {{ atl_product_download }} variable +# - name: Unpack the downloaded application depending on format +# include_tasks: "unpack_{{ atl_download_format }}.yml" + +# - name: Symlink the installed version to current +# file: +# src: "{{ atl_product_installation_versioned }}" +# dest: "{{ atl_product_installation_current }}" +# state: link +# force: true + +# # Temp product was downloaded and installed. +# # If the following conditions are true, move to home_shared +# # 1. This node just downloaded binary. +# # 2. Another node is not already moving into place. +# # 3. The binary is downloaded and lockdir in place. +# - name: "Check move product installer" +# block: + +# - name: Check again for moving lock directory +# stat: +# path: "{{ atl_product_home_shared_moving_lock }}" +# register: moving_lock_2 - - name: Check again for downloaded lock directory - stat: - path: "{{ atl_product_home_shared_downloaded_lock }}" - register: downloaded_lock_2 +# - name: Check again for completed lock directory +# stat: +# path: "{{ atl_product_home_shared_completed_lock }}" +# register: completed_lock_2 - - name: Check again for product installer in home_shared - stat: - path: "{{ atl_product_home_shared_download }}" - register: home_shared_download_2 +# - name: Check again for product installer in home_shared +# stat: +# path: "{{ atl_product_home_shared_download }}" +# register: home_shared_download_2 - # If binary exists and lockdir exists use this binary instead - - name: Check lock directory and binary exists on shared_home - set_fact: - move_binary: true - when: - - ( home_shared_download.stat.exists == False or - downloaded_lock.stat.isdir is not defined or downloaded_lock.stat.isdir == False ) - - ( moving_lock.stat.isdir is not defined or moving_lock.stat.isdir == False ) +# # If binary exists and lockdir exists use this binary instead +# - name: Check lock directory and binary exists on shared_home +# set_fact: +# move_binary: true +# when: +# - ( home_shared_download.stat.exists == False or +# completed_lock.stat.isdir is not defined or completed_lock.stat.isdir == False ) +# - ( moving_lock.stat.isdir is not defined or moving_lock.stat.isdir == False ) - when: download_binary +# when: download_binary -- name: "Move product installer if required" - block: +# - name: "Move product installer if required" +# block: - - name: Create moving_lock to ensure other nodes skip - file: - path: "{{ atl_product_home_shared_moving_lock }}" - state: directory - when: move_binary - register: moving_lock_created +# - name: Create moving_lock to ensure other nodes skip +# file: +# path: "{{ atl_product_home_shared_moving_lock }}" +# state: directory +# when: move_binary +# register: moving_lock_created - - name: Copy temp installer to home_shared - copy: - src: "{{ atl_product_temp_download }}" - dest: "{{ atl_product_home_shared_download }}" - remote_src: true - when: moving_lock_created is succeeded - register: copied +# - name: Copy temp installer to home_shared +# copy: +# src: "{{ atl_product_temp_download }}" +# dest: "{{ atl_product_home_shared_download }}" +# remote_src: true +# when: moving_lock_created is succeeded +# register: copied - - name: Create downloaded_lock once product installer downloaded and copied - file: - path: "{{ atl_product_home_shared_downloaded_lock }}" - state: directory - when: copied is succeeded - register: downloaded_lock_created +# - name: Create completed_lock once product installer downloaded and copied +# file: +# path: "{{ atl_product_home_shared_completed_lock }}" +# state: directory +# when: copied is succeeded +# register: completed_lock_created - - name: Remove moving_lock to show that binary is completed - file: - path: "{{ atl_product_home_shared_moving_lock }}" - state: absent - when: - - downloaded_lock_created is succeeded - - copied is succeeded - register: moving_lock_removed +# - name: Remove moving_lock to show that binary is completed +# file: +# path: "{{ atl_product_home_shared_moving_lock }}" +# state: absent +# when: +# - completed_lock_created is succeeded +# - copied is succeeded +# register: moving_lock_removed - - name: Delete old temp installer - file: - path: "{{ atl_product_temp_download }}" - state: absent - when: moving_lock_removed is succeeded - register: temp_deleted +# - name: Delete old temp installer +# file: +# path: "{{ atl_product_temp_download }}" +# state: absent +# when: moving_lock_removed is succeeded +# register: temp_deleted - when: move_binary +# when: move_binary From 70a48e58c4a2d178b1043b4218bc9e6b15a89823 Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Mon, 14 Oct 2019 17:34:05 +1100 Subject: [PATCH 023/109] ITOPSENG-164 Debug some changes to logic --- roles/product_install/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/product_install/defaults/main.yml b/roles/product_install/defaults/main.yml index 61d8d04..931e639 100644 --- a/roles/product_install/defaults/main.yml +++ b/roles/product_install/defaults/main.yml @@ -25,7 +25,7 @@ atl_product_varfile: "{{ atl_installer_temp }}/{{ atl_product_family }}.varfile" atl_product_home_shared_download_dir: "{{ atl_product_home_shared }}/downloads" atl_product_home_shared_download: "{{ atl_product_home_shared_download_dir }}/{{ atl_product_download_filename }}" atl_product_home_shared_moving_lock: "{{ atl_product_home_shared_download }}_moving" -atl_product_home_shared_downloaded_lock: "{{ atl_product_home_shared_download }}_downloaded" +atl_product_home_shared_completed_lock: "{{ atl_product_home_shared_download }}_completed" atl_marketplace_base: "https://marketplace.atlassian.com" atl_servicedesk_latest_url: "https://marketplace.atlassian.com/rest/2/products/key/jira-servicedesk/versions/latest" From 9dea02808c49ad56d52b73b3bbc8c9f237efc351 Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Tue, 15 Oct 2019 10:16:52 +1100 Subject: [PATCH 024/109] ITOPSENG-164 Debug some changes to logic --- roles/product_install/tasks/main.yml | 32 +++++++++++++++++----------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/roles/product_install/tasks/main.yml b/roles/product_install/tasks/main.yml index 040b2ef..54e74c1 100644 --- a/roles/product_install/tasks/main.yml +++ b/roles/product_install/tasks/main.yml @@ -169,25 +169,33 @@ register: atl_product_completed when: download_binary -# If product installer was fetched -# Make the moving directory -# - failure, continue and install from temp -# - success, move binary and install from shared_home - +# If product installer was fetched make the moving directory - name: Create moving_lock. file: path: "{{ atl_product_home_shared_moving_lock }}" state: directory - when: download_binary is succeeded + when: + - atl_product_completed is succeeded register: moving_lock_created -- name: Debug Scenario A - lock created - debug: lock created - when: moving_lock_created is succeeded +# Directory lock was created by this run +# - Move binary +- name: Move binary Scenario - lock created by this run + debug: + msg: lock created + set_fact: + move_binary: true + when: + - moving_lock_created is succeeded + - moving_lock_created.changed == True -- name: Debug Scenario B - lock cannot created - debug: lock not created - when: moving_lock_created is failed +# Directory lock was either already created or could not be +# - Continue and install from temp +- name: Continue Scenario - lock cannot be created or created by previous run + debug: + msg: lock not created + when: moving_lock_created is not succeeded or + ( moving_lock_created is succeeded and moving_lock_created.changed == False ) From eedea5e32647adba15bd004e7b509f7fad7e946e Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Tue, 15 Oct 2019 10:23:06 +1100 Subject: [PATCH 025/109] ITOPSENG-164 Debug some changes to logic --- roles/product_install/tasks/main.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/roles/product_install/tasks/main.yml b/roles/product_install/tasks/main.yml index 54e74c1..ad490f6 100644 --- a/roles/product_install/tasks/main.yml +++ b/roles/product_install/tasks/main.yml @@ -181,8 +181,6 @@ # Directory lock was created by this run # - Move binary - name: Move binary Scenario - lock created by this run - debug: - msg: lock created set_fact: move_binary: true when: From 2ac847dd3ab88ceb54fdeb3dc6c8a19082297b7f Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Tue, 15 Oct 2019 10:28:48 +1100 Subject: [PATCH 026/109] ITOPSENG-164 Debug some changes to logic --- roles/product_install/tasks/main.yml | 74 ++++++++++++++++------------ 1 file changed, 42 insertions(+), 32 deletions(-) diff --git a/roles/product_install/tasks/main.yml b/roles/product_install/tasks/main.yml index ad490f6..288b0f6 100644 --- a/roles/product_install/tasks/main.yml +++ b/roles/product_install/tasks/main.yml @@ -196,6 +196,48 @@ ( moving_lock_created is succeeded and moving_lock_created.changed == False ) +# Move binary block +- name: Move product installer to home_shared + block: + + - name: Copy temp installer to home_shared + copy: + src: "{{ atl_product_temp_download }}" + dest: "{{ atl_product_home_shared_download }}" + remote_src: true + when: + - moving_lock_created is succeeded + - moving_lock_created.changed == True + register: copied + + - name: Create completed_lock once product installer downloaded and copied + file: + path: "{{ atl_product_home_shared_completed_lock }}" + state: directory + when: copied is succeeded + register: completed_lock_created + + - name: Remove moving_lock to show that binary is completed + file: + path: "{{ atl_product_home_shared_moving_lock }}" + state: absent + when: + - completed_lock_created is succeeded + - copied is succeeded + register: moving_lock_removed + + - name: Delete old temp installer + file: + path: "{{ atl_product_temp_download }}" + state: absent + when: moving_lock_removed is succeeded + register: temp_deleted + + when: move_binary + + + + @@ -255,38 +297,6 @@ # when: move_binary # register: moving_lock_created -# - name: Copy temp installer to home_shared -# copy: -# src: "{{ atl_product_temp_download }}" -# dest: "{{ atl_product_home_shared_download }}" -# remote_src: true -# when: moving_lock_created is succeeded -# register: copied - -# - name: Create completed_lock once product installer downloaded and copied -# file: -# path: "{{ atl_product_home_shared_completed_lock }}" -# state: directory -# when: copied is succeeded -# register: completed_lock_created - -# - name: Remove moving_lock to show that binary is completed -# file: -# path: "{{ atl_product_home_shared_moving_lock }}" -# state: absent -# when: -# - completed_lock_created is succeeded -# - copied is succeeded -# register: moving_lock_removed - -# - name: Delete old temp installer -# file: -# path: "{{ atl_product_temp_download }}" -# state: absent -# when: moving_lock_removed is succeeded -# register: temp_deleted - -# when: move_binary From 46e3665cd561cd0b4f0425ce0ff18b996b0fcab2 Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Tue, 15 Oct 2019 10:50:48 +1100 Subject: [PATCH 027/109] ITOPSENG-164 Debug some changes to logic --- roles/product_install/tasks/main.yml | 72 +++++++++++++++------------- 1 file changed, 38 insertions(+), 34 deletions(-) diff --git a/roles/product_install/tasks/main.yml b/roles/product_install/tasks/main.yml index 288b0f6..3665698 100644 --- a/roles/product_install/tasks/main.yml +++ b/roles/product_install/tasks/main.yml @@ -159,42 +159,46 @@ - completed_lock.stat.isdir is defined - completed_lock.stat.isdir -# If the binary was never installed, download it to temp location -- name: Installer not on home_shared. Fetch it. - get_url: - url: "{{ atl_product_download_url }}" - dest: "{{ atl_product_temp_download }}" - mode: 0755 - force: false - register: atl_product_completed +# Fetch binary if required +- name: download_binary is true so fetch and do all the things + block: + + - name: Fetch binary + get_url: + url: "{{ atl_product_download_url }}" + dest: "{{ atl_product_temp_download }}" + mode: 0755 + force: false + register: atl_product_completed + + # If product installer was fetched make the moving directory + - name: Create moving_lock. + file: + path: "{{ atl_product_home_shared_moving_lock }}" + state: directory + when: + - atl_product_completed is succeeded + register: moving_lock_created + + # Directory lock was created by this run + # - Move binary + - name: Move binary Scenario - lock created by this run + set_fact: + move_binary: true + when: + - moving_lock_created is succeeded + - moving_lock_created.changed == True + + # Directory lock was either already created or could not be + # - Continue and install from temp + - name: Continue Scenario - lock cannot be created or created by previous run + debug: + msg: lock not created + when: moving_lock_created is not succeeded or + ( moving_lock_created is succeeded and moving_lock_created.changed == False ) + when: download_binary -# If product installer was fetched make the moving directory -- name: Create moving_lock. - file: - path: "{{ atl_product_home_shared_moving_lock }}" - state: directory - when: - - atl_product_completed is succeeded - register: moving_lock_created - -# Directory lock was created by this run -# - Move binary -- name: Move binary Scenario - lock created by this run - set_fact: - move_binary: true - when: - - moving_lock_created is succeeded - - moving_lock_created.changed == True - -# Directory lock was either already created or could not be -# - Continue and install from temp -- name: Continue Scenario - lock cannot be created or created by previous run - debug: - msg: lock not created - when: moving_lock_created is not succeeded or - ( moving_lock_created is succeeded and moving_lock_created.changed == False ) - # Move binary block - name: Move product installer to home_shared From b9c5389ef5fc33758ca5eeeed959940475c921e6 Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Tue, 15 Oct 2019 10:53:25 +1100 Subject: [PATCH 028/109] ITOPSENG-164 Debug some changes to logic --- roles/product_install/tasks/main.yml | 77 ++++------------------------ 1 file changed, 11 insertions(+), 66 deletions(-) diff --git a/roles/product_install/tasks/main.yml b/roles/product_install/tasks/main.yml index 3665698..eba1b6b 100644 --- a/roles/product_install/tasks/main.yml +++ b/roles/product_install/tasks/main.yml @@ -199,7 +199,6 @@ when: download_binary - # Move binary block - name: Move product installer to home_shared block: @@ -239,69 +238,15 @@ when: move_binary +# At this point the binary is in place and can be used to install +# The location is determined by {{ atl_product_download }} variable +# set above +- name: Unpack the downloaded application depending on format + include_tasks: "unpack_{{ atl_download_format }}.yml" - - - - - -# # If product installer was pre-downloaded on shared_home, install from there -# # This is determined by {{ atl_product_download }} variable -# - name: Unpack the downloaded application depending on format -# include_tasks: "unpack_{{ atl_download_format }}.yml" - -# - name: Symlink the installed version to current -# file: -# src: "{{ atl_product_installation_versioned }}" -# dest: "{{ atl_product_installation_current }}" -# state: link -# force: true - -# # Temp product was downloaded and installed. -# # If the following conditions are true, move to home_shared -# # 1. This node just downloaded binary. -# # 2. Another node is not already moving into place. -# # 3. The binary is downloaded and lockdir in place. -# - name: "Check move product installer" -# block: - -# - name: Check again for moving lock directory -# stat: -# path: "{{ atl_product_home_shared_moving_lock }}" -# register: moving_lock_2 - -# - name: Check again for completed lock directory -# stat: -# path: "{{ atl_product_home_shared_completed_lock }}" -# register: completed_lock_2 - -# - name: Check again for product installer in home_shared -# stat: -# path: "{{ atl_product_home_shared_download }}" -# register: home_shared_download_2 - -# # If binary exists and lockdir exists use this binary instead -# - name: Check lock directory and binary exists on shared_home -# set_fact: -# move_binary: true -# when: -# - ( home_shared_download.stat.exists == False or -# completed_lock.stat.isdir is not defined or completed_lock.stat.isdir == False ) -# - ( moving_lock.stat.isdir is not defined or moving_lock.stat.isdir == False ) - -# when: download_binary - -# - name: "Move product installer if required" -# block: - -# - name: Create moving_lock to ensure other nodes skip -# file: -# path: "{{ atl_product_home_shared_moving_lock }}" -# state: directory -# when: move_binary -# register: moving_lock_created - - - - - +- name: Symlink the installed version to current + file: + src: "{{ atl_product_installation_versioned }}" + dest: "{{ atl_product_installation_current }}" + state: link + force: true From 2c1ce5bc2e18b1f883a96e978576443fb8b68292 Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Tue, 15 Oct 2019 10:57:51 +1100 Subject: [PATCH 029/109] ITOPSENG-164 Debug some changes to logic --- roles/product_install/tasks/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/product_install/tasks/main.yml b/roles/product_install/tasks/main.yml index eba1b6b..30d3cce 100644 --- a/roles/product_install/tasks/main.yml +++ b/roles/product_install/tasks/main.yml @@ -163,6 +163,10 @@ - name: download_binary is true so fetch and do all the things block: + - name: debug + msg: + - Download the binary + - name: Fetch binary get_url: url: "{{ atl_product_download_url }}" From cc5b3c8a70662a3dee8141650770599851b5f523 Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Tue, 15 Oct 2019 11:03:03 +1100 Subject: [PATCH 030/109] ITOPSENG-164 Debug some changes to logic --- roles/product_install/tasks/main.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/roles/product_install/tasks/main.yml b/roles/product_install/tasks/main.yml index 30d3cce..349df3e 100644 --- a/roles/product_install/tasks/main.yml +++ b/roles/product_install/tasks/main.yml @@ -164,8 +164,8 @@ block: - name: debug - msg: - - Download the binary + debug: + msg: Download the binary - name: Fetch binary get_url: @@ -242,6 +242,10 @@ when: move_binary +- name: debug + debug: + msg: Install from {{ atl_product_download }} + # At this point the binary is in place and can be used to install # The location is determined by {{ atl_product_download }} variable # set above From 37ae0eadc5026a0afe2dce7bc40c973351a48321 Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Tue, 15 Oct 2019 11:10:47 +1100 Subject: [PATCH 031/109] ITOPSENG-164 Debug some changes to logic --- roles/product_install/tasks/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/product_install/tasks/main.yml b/roles/product_install/tasks/main.yml index 349df3e..5da046f 100644 --- a/roles/product_install/tasks/main.yml +++ b/roles/product_install/tasks/main.yml @@ -240,6 +240,11 @@ when: moving_lock_removed is succeeded register: temp_deleted + - name: Set install to home_shared location + set_fact: + atl_product_download: "{{ atl_product_home_shared_download }}" + when: temp_deleted is succeeded + when: move_binary - name: debug From 5f2e5929ab3f48305f3d41d316a5ea95f23cf4e0 Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Tue, 15 Oct 2019 15:27:41 +1100 Subject: [PATCH 032/109] ITOPSENG-164 Remove all the debug now that logic is fixed --- roles/product_install/tasks/main.yml | 32 ++++++++-------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/roles/product_install/tasks/main.yml b/roles/product_install/tasks/main.yml index 5da046f..068a3ee 100644 --- a/roles/product_install/tasks/main.yml +++ b/roles/product_install/tasks/main.yml @@ -163,10 +163,7 @@ - name: download_binary is true so fetch and do all the things block: - - name: debug - debug: - msg: Download the binary - + # Fetch binary and copy to temp - name: Fetch binary get_url: url: "{{ atl_product_download_url }}" @@ -175,7 +172,7 @@ force: false register: atl_product_completed - # If product installer was fetched make the moving directory + # If product installer was fetched make the lock directory - name: Create moving_lock. file: path: "{{ atl_product_home_shared_moving_lock }}" @@ -184,26 +181,20 @@ - atl_product_completed is succeeded register: moving_lock_created - # Directory lock was created by this run - # - Move binary + # Directory lock was created by this run? + # If so, then set a fact intending to move binary - name: Move binary Scenario - lock created by this run set_fact: move_binary: true when: - moving_lock_created is succeeded - moving_lock_created.changed == True - - # Directory lock was either already created or could not be - # - Continue and install from temp - - name: Continue Scenario - lock cannot be created or created by previous run - debug: - msg: lock not created - when: moving_lock_created is not succeeded or - ( moving_lock_created is succeeded and moving_lock_created.changed == False ) + # Otherwise directory lock was either already created or + # could not be created. Fall back is to continue and install from temp when: download_binary -# Move binary block +# If the intention is to move binary to home_shared - name: Move product installer to home_shared block: @@ -247,13 +238,8 @@ when: move_binary -- name: debug - debug: - msg: Install from {{ atl_product_download }} - -# At this point the binary is in place and can be used to install -# The location is determined by {{ atl_product_download }} variable -# set above +# At this point the binary is in {{ atl_product_download }} +# (which is either on home_shared or temp) - name: Unpack the downloaded application depending on format include_tasks: "unpack_{{ atl_download_format }}.yml" From 1dedfa827e116cb4a5ea5d960857f6a7d880b106 Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Wed, 16 Oct 2019 14:37:05 +1100 Subject: [PATCH 033/109] ITOPSENG-164 yamllint fixes --- roles/product_install/tasks/main.yml | 118 +++++++++++++-------------- 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/roles/product_install/tasks/main.yml b/roles/product_install/tasks/main.yml index 068a3ee..388164c 100644 --- a/roles/product_install/tasks/main.yml +++ b/roles/product_install/tasks/main.yml @@ -163,34 +163,34 @@ - name: download_binary is true so fetch and do all the things block: - # Fetch binary and copy to temp - - name: Fetch binary - get_url: - url: "{{ atl_product_download_url }}" - dest: "{{ atl_product_temp_download }}" - mode: 0755 - force: false - register: atl_product_completed + # Fetch binary and copy to temp + - name: Fetch binary + get_url: + url: "{{ atl_product_download_url }}" + dest: "{{ atl_product_temp_download }}" + mode: 0755 + force: false + register: atl_product_completed - # If product installer was fetched make the lock directory - - name: Create moving_lock. - file: - path: "{{ atl_product_home_shared_moving_lock }}" - state: directory - when: - - atl_product_completed is succeeded - register: moving_lock_created + # If product installer was fetched make the lock directory + - name: Create moving_lock. + file: + path: "{{ atl_product_home_shared_moving_lock }}" + state: directory + when: + - atl_product_completed is succeeded + register: moving_lock_created - # Directory lock was created by this run? - # If so, then set a fact intending to move binary - - name: Move binary Scenario - lock created by this run - set_fact: - move_binary: true - when: - - moving_lock_created is succeeded - - moving_lock_created.changed == True - # Otherwise directory lock was either already created or - # could not be created. Fall back is to continue and install from temp + # Directory lock was created by this run? + # If so, then set a fact intending to move binary + - name: Move binary Scenario - lock created by this run + set_fact: + move_binary: true + when: + - moving_lock_created is succeeded + - moving_lock_created.changed == True + # Otherwise directory lock was either already created or + # could not be created. Fall back is to continue and install from temp when: download_binary @@ -198,43 +198,43 @@ - name: Move product installer to home_shared block: - - name: Copy temp installer to home_shared - copy: - src: "{{ atl_product_temp_download }}" - dest: "{{ atl_product_home_shared_download }}" - remote_src: true - when: - - moving_lock_created is succeeded - - moving_lock_created.changed == True - register: copied + - name: Copy temp installer to home_shared + copy: + src: "{{ atl_product_temp_download }}" + dest: "{{ atl_product_home_shared_download }}" + remote_src: true + when: + - moving_lock_created is succeeded + - moving_lock_created.changed == True + register: copied - - name: Create completed_lock once product installer downloaded and copied - file: - path: "{{ atl_product_home_shared_completed_lock }}" - state: directory - when: copied is succeeded - register: completed_lock_created + - name: Create completed_lock once product installer downloaded and copied + file: + path: "{{ atl_product_home_shared_completed_lock }}" + state: directory + when: copied is succeeded + register: completed_lock_created - - name: Remove moving_lock to show that binary is completed - file: - path: "{{ atl_product_home_shared_moving_lock }}" - state: absent - when: - - completed_lock_created is succeeded - - copied is succeeded - register: moving_lock_removed + - name: Remove moving_lock to show that binary is completed + file: + path: "{{ atl_product_home_shared_moving_lock }}" + state: absent + when: + - completed_lock_created is succeeded + - copied is succeeded + register: moving_lock_removed - - name: Delete old temp installer - file: - path: "{{ atl_product_temp_download }}" - state: absent - when: moving_lock_removed is succeeded - register: temp_deleted + - name: Delete old temp installer + file: + path: "{{ atl_product_temp_download }}" + state: absent + when: moving_lock_removed is succeeded + register: temp_deleted - - name: Set install to home_shared location - set_fact: - atl_product_download: "{{ atl_product_home_shared_download }}" - when: temp_deleted is succeeded + - name: Set install to home_shared location + set_fact: + atl_product_download: "{{ atl_product_home_shared_download }}" + when: temp_deleted is succeeded when: move_binary From f24fb3fb6574b9acda4e2ccffeed557eb42063cd Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Wed, 16 Oct 2019 14:45:50 +1100 Subject: [PATCH 034/109] ITOPSENG-164 ansible-lint fixes --- roles/product_install/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/product_install/tasks/main.yml b/roles/product_install/tasks/main.yml index 388164c..f136128 100644 --- a/roles/product_install/tasks/main.yml +++ b/roles/product_install/tasks/main.yml @@ -188,7 +188,7 @@ move_binary: true when: - moving_lock_created is succeeded - - moving_lock_created.changed == True + - moving_lock_created.changed # Otherwise directory lock was either already created or # could not be created. Fall back is to continue and install from temp @@ -205,7 +205,7 @@ remote_src: true when: - moving_lock_created is succeeded - - moving_lock_created.changed == True + - moving_lock_created.changed register: copied - name: Create completed_lock once product installer downloaded and copied From 2a67eae9c87b7120fe65e896a432d8f33a41dea9 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Thu, 17 Oct 2019 14:51:32 +1100 Subject: [PATCH 035/109] ITOPSENG-101 jsd_as_obr support --- aws_jira_dc_node.yml | 1 + .../tasks/jira-servicedesk_as_obr.yml | 30 +++++++++++++++++++ roles/product_install/tasks/main.yml | 4 +++ 3 files changed, 35 insertions(+) create mode 100644 roles/product_install/tasks/jira-servicedesk_as_obr.yml diff --git a/aws_jira_dc_node.yml b/aws_jira_dc_node.yml index f0af8a4..08d9657 100644 --- a/aws_jira_dc_node.yml +++ b/aws_jira_dc_node.yml @@ -7,6 +7,7 @@ atl_product_family: "jira" atl_product_user: "jira" atl_product_edition: "jira-{{ lookup('env', 'ATL_PRODUCT_EDITION') | lower }}" + atl_install_jsd_as_obr: "{{ lookup('env', 'ATL_JSD_ASOBR')" atl_systemd_service_name: "jira.service" atl_startup_systemd_params: diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml new file mode 100644 index 0000000..012c6d9 --- /dev/null +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -0,0 +1,30 @@ +--- + +- name: Get the installer product version info + uri: + url="https://marketplace.atlassian.com/rest/2/products/key/jira-software/versions/name/{{ atl_product_version }}" + return_content=yes + register: atl_product_version_info + +- name: lets grab the build number + debug: + msg="buildNumber={{ atl_product_version_info.json.buildNumber }}" + +- name: Get the JSD build version info + uri: + url="https://marketplace.atlassian.com/rest/2/products/key/jira-servicedesk/versions/latest?application=jira&applicationBuild={{ atl_product_version_info.json.buildNumber }}" + return_content=yes + register: atl_jsd_build_info + +- name: lets grab the obr binary href + debug: + msg="obr_ref={{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}" + +- name: grab the jsd obr + get_url: + url: "{{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}" + dest: "{{ atl_installer_temp }}" + +- name: override the atl_product_edition to jira-software + set_fact: + atl_product_edition: "jira-software" diff --git a/roles/product_install/tasks/main.yml b/roles/product_install/tasks/main.yml index 77371fa..b685a28 100644 --- a/roles/product_install/tasks/main.yml +++ b/roles/product_install/tasks/main.yml @@ -146,3 +146,7 @@ dest: "{{ atl_product_installation_current }}" state: link force: true + +- name: Include if jsd is requested to be installed from OBR + include_tasks: "jira-servicedesk_as_obr.yml" + when: atl_install_jsd_as_obr From 19329447a800d424ec7c6ec0dcd325023799b881 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 18 Oct 2019 13:57:13 +1100 Subject: [PATCH 036/109] ITOPSENG-101 logic to fetch and unpack jsd as an obr --- .../tasks/jira-servicedesk_as_obr.yml | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index 012c6d9..efb1711 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -6,7 +6,7 @@ return_content=yes register: atl_product_version_info -- name: lets grab the build number +- name: Show the returned build number debug: msg="buildNumber={{ atl_product_version_info.json.buildNumber }}" @@ -16,15 +16,33 @@ return_content=yes register: atl_jsd_build_info -- name: lets grab the obr binary href +- name: Show the returned obr binary href debug: msg="obr_ref={{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}" -- name: grab the jsd obr +- name: Fetch the jsd obr get_url: url: "{{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}" dest: "{{ atl_installer_temp }}" + register: jsdobr -- name: override the atl_product_edition to jira-software - set_fact: - atl_product_edition: "jira-software" +- name: check the name of the downloaded file + debug: + msg="{{ jsdobr.dest }}" + +- name: Copy the obr to shared_home + copy: + src: "{{ jsdobr.dest }}" + dest: "{{ atl_product_version_cache_dir }}" + remote_src: true + +- name: Unpack the obr into the installed-plugins dir + unarchive: + remote_src: true + src: "{{ jsdobr.dest }}" + dest: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins" + owner: "{{ atl_product_user }}" + group: "{{ atl_product_user }}" + mode: 0750 + extra_opts: + - "-j" From 9c032469f5a90283eb19bb2c046eba6b649cb346 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 18 Oct 2019 14:24:58 +1100 Subject: [PATCH 037/109] ITOPSENG-101 split some strings across lines to pass line length linting --- roles/product_install/tasks/jira-servicedesk_as_obr.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index efb1711..cbbb753 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -2,8 +2,8 @@ - name: Get the installer product version info uri: - url="https://marketplace.atlassian.com/rest/2/products/key/jira-software/versions/name/{{ atl_product_version }}" - return_content=yes + url: "https://marketplace.atlassian.com/rest/2/products/key/jira-software/versions/name/{{ atl_product_version }}" + return_content: yes register: atl_product_version_info - name: Show the returned build number @@ -12,8 +12,9 @@ - name: Get the JSD build version info uri: - url="https://marketplace.atlassian.com/rest/2/products/key/jira-servicedesk/versions/latest?application=jira&applicationBuild={{ atl_product_version_info.json.buildNumber }}" - return_content=yes + url: "https://marketplace.atlassian.com/rest/2/products/key/jira-servicedesk/versions/latest?application=\ + jira&applicationBuild={{ atl_product_version_info.json.buildNumber }}" + return_content: yes register: atl_jsd_build_info - name: Show the returned obr binary href From eae65657cebb2bae0b52b9191cc4d94dafaaab88 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 18 Oct 2019 14:44:43 +1100 Subject: [PATCH 038/109] ITOPSENG-101 fixed missing braces --- aws_jira_dc_node.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_jira_dc_node.yml b/aws_jira_dc_node.yml index 08d9657..c349bb8 100644 --- a/aws_jira_dc_node.yml +++ b/aws_jira_dc_node.yml @@ -7,7 +7,7 @@ atl_product_family: "jira" atl_product_user: "jira" atl_product_edition: "jira-{{ lookup('env', 'ATL_PRODUCT_EDITION') | lower }}" - atl_install_jsd_as_obr: "{{ lookup('env', 'ATL_JSD_ASOBR')" + atl_install_jsd_as_obr: "{{ lookup('env', 'ATL_JSD_ASOBR') }}" atl_systemd_service_name: "jira.service" atl_startup_systemd_params: From 0245f9059d8f53340618bcd88e48cfe14e79f844 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 18 Oct 2019 14:59:55 +1100 Subject: [PATCH 039/109] ITOPSENG-101 ensure installed-plugins exists --- roles/product_install/tasks/jira-servicedesk_as_obr.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index cbbb753..7f9702d 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -37,6 +37,14 @@ dest: "{{ atl_product_version_cache_dir }}" remote_src: true +- name: Ensure instaled-plugins dir exists + file: + path: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins" + state: directory + mode: 0750 + owner: "{{ atl_product_user }}" + group: "{{ atl_product_user }}" + - name: Unpack the obr into the installed-plugins dir unarchive: remote_src: true From 3ae840cf13ee4109462adba3ee566b11ec45fb75 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 18 Oct 2019 18:11:00 +1100 Subject: [PATCH 040/109] ITOPSENG-101 work around ansible not handling unzip -j --- .../tasks/jira-servicedesk_as_obr.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index 7f9702d..b8c0369 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -53,5 +53,13 @@ owner: "{{ atl_product_user }}" group: "{{ atl_product_user }}" mode: 0750 - extra_opts: - - "-j" + # Ansible unarchive task validation does not work with the -j flag so we need to achieve the same result in 2 steps + # extra_opts: + # - "-j" + +- name: Copy JSD dependency jars into the installed-plugins dir + copy: + src: "{{ item }}" + dest: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins" + remote_src: true + with_fileglob: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins/dependencies/*.jar" From 244e6664e61e5ba9263d549cbcfb2d26d6446012 Mon Sep 17 00:00:00 2001 From: Geoff Jacobs Date: Mon, 21 Oct 2019 10:46:56 +1100 Subject: [PATCH 041/109] ITOPSENG-258 setting the Crowd node name via CATALINA_OPTS --- roles/crowd_config/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/crowd_config/tasks/main.yml b/roles/crowd_config/tasks/main.yml index ef7dcc2..9df29b9 100644 --- a/roles/crowd_config/tasks/main.yml +++ b/roles/crowd_config/tasks/main.yml @@ -25,9 +25,9 @@ lineinfile: path: "{{ atl_product_installation_versioned }}/apache-tomcat/bin/setenv.sh" insertafter: "EOF" - line: 'export CATALINA_OPTS="${CATALINA_OPTS} {{ atl_catalina_opts }} {{ atl_catalina_opts_extra }}"' + line: 'export CATALINA_OPTS="${CATALINA_OPTS} {{ atl_catalina_opts }} {{ atl_catalina_opts_extra }} -Dcluster.node.name={{ ansible_ec2_instance_id }}-{{ ansible_ec2_local_ipv4 }}"' -- name: Set JAVA_HOME #FIXME +- name: Set JAVA_HOME lineinfile: path: "{{ atl_product_installation_versioned }}/apache-tomcat/bin/setenv.sh" insertafter: "EOF" From 1df244d7788d439b480cc3b265d9c17c2e7f54be Mon Sep 17 00:00:00 2001 From: Geoff Jacobs Date: Mon, 21 Oct 2019 10:49:53 +1100 Subject: [PATCH 042/109] ITOPSENG-258 splitting to a new task to appease the linting overlords --- roles/crowd_config/tasks/main.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/roles/crowd_config/tasks/main.yml b/roles/crowd_config/tasks/main.yml index 9df29b9..f6e8493 100644 --- a/roles/crowd_config/tasks/main.yml +++ b/roles/crowd_config/tasks/main.yml @@ -25,7 +25,13 @@ lineinfile: path: "{{ atl_product_installation_versioned }}/apache-tomcat/bin/setenv.sh" insertafter: "EOF" - line: 'export CATALINA_OPTS="${CATALINA_OPTS} {{ atl_catalina_opts }} {{ atl_catalina_opts_extra }} -Dcluster.node.name={{ ansible_ec2_instance_id }}-{{ ansible_ec2_local_ipv4 }}"' + line: 'export CATALINA_OPTS="${CATALINA_OPTS} {{ atl_catalina_opts }} {{ atl_catalina_opts_extra }}"' + +- name: Set the Crowd node name via CATALINA_OPTS + lineinfile: + path: "{{ atl_product_installation_versioned }}/apache-tomcat/bin/setenv.sh" + insertafter: "EOF" + line: export CATALINA_OPTS="${CATALINA_OPTS} -Dcluster.node.name={{ ansible_ec2_instance_id }}-{{ ansible_ec2_local_ipv4 }}" - name: Set JAVA_HOME lineinfile: From 7e4d1301e10ad2c133341b0275913d85c7bf9f67 Mon Sep 17 00:00:00 2001 From: Varun Arbatti <1063972+theghostwhoforks@users.noreply.github.com> Date: Mon, 21 Oct 2019 11:08:28 +1100 Subject: [PATCH 043/109] DCD-686: Changes case of shared home from snakeCase to lower camel case --- roles/restore_backups/tasks/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index d12d073..18f2169 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -60,7 +60,7 @@ # following usages will need to be updated once it settles.. atl_backup_id: "{{ atl_backup_manifest.name }}" atl_backup_db_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.artifacts.db.location.value | basename }}" - atl_backup_home_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.artifacts.shared_home.location.value | basename }}" + atl_backup_home_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.artifacts.sharedHome.location.value | basename }}" # FIXME: Here we fetch the backups. However we may wish to stream # these directly from S3 to the target DB/FS to avoid requiring @@ -77,8 +77,8 @@ aws_s3: mode: get overwrite: different - bucket: "{{ atl_backup_manifest.artifacts.shared_home.location.value | urlsplit('hostname') }}" - object: "{{ atl_backup_manifest.artifacts.shared_home.location.value | urlsplit('path') }}" + bucket: "{{ atl_backup_manifest.artifacts.sharedHome.location.value | urlsplit('hostname') }}" + object: "{{ atl_backup_manifest.artifacts.sharedHome.location.value | urlsplit('path') }}" dest: "{{ atl_backup_home_dest }}" - name: Install distro-specific restore support packages From d6a9a2c8415a6678709cfb83b21b8e63f6c5c643 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Mon, 21 Oct 2019 12:54:39 +1100 Subject: [PATCH 044/109] ITOPSENG-101 ensure the obr var exists and is true before including the associated tasks --- roles/product_install/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/product_install/tasks/main.yml b/roles/product_install/tasks/main.yml index b685a28..d84895d 100644 --- a/roles/product_install/tasks/main.yml +++ b/roles/product_install/tasks/main.yml @@ -149,4 +149,4 @@ - name: Include if jsd is requested to be installed from OBR include_tasks: "jira-servicedesk_as_obr.yml" - when: atl_install_jsd_as_obr + when: atl_install_jsd_as_obr.exists and atl_install_jsd_as_obr From eec68f6357cda6031216a0965d830e75f57617ef Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Mon, 21 Oct 2019 13:06:46 +1100 Subject: [PATCH 045/109] ITOPSENG-101 ensure the obr var exists and is true before including the associated tasks --- roles/product_install/tasks/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/roles/product_install/tasks/main.yml b/roles/product_install/tasks/main.yml index d84895d..6cea478 100644 --- a/roles/product_install/tasks/main.yml +++ b/roles/product_install/tasks/main.yml @@ -149,4 +149,6 @@ - name: Include if jsd is requested to be installed from OBR include_tasks: "jira-servicedesk_as_obr.yml" - when: atl_install_jsd_as_obr.exists and atl_install_jsd_as_obr + when: + - atl_install_jsd_as_obr is defined + - atl_install_jsd_as_obr From f8e8f94946a811fa4a9d7b592bfc8ca638efb8e8 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Mon, 21 Oct 2019 14:22:52 +1100 Subject: [PATCH 046/109] ITOPSENG-101 cleanup for PR comments --- roles/product_install/defaults/main.yml | 1 + roles/product_install/tasks/jira-servicedesk_as_obr.yml | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/roles/product_install/defaults/main.yml b/roles/product_install/defaults/main.yml index 211e76e..4b62635 100644 --- a/roles/product_install/defaults/main.yml +++ b/roles/product_install/defaults/main.yml @@ -23,6 +23,7 @@ atl_product_download: "{{ atl_installer_temp }}/{{ atl_product_download_filename atl_product_varfile: "{{ atl_installer_temp }}/{{ atl_product_family }}.varfile" atl_marketplace_base: "https://marketplace.atlassian.com" +atl_mpac_products: "https://marketplace.atlassian.com/rest/2/products" atl_servicedesk_latest_url: "https://marketplace.atlassian.com/rest/2/products/key/jira-servicedesk/versions/latest" atl_servicedesk_versioned_url: "https://marketplace.atlassian.com/rest/2/products/key/jira-servicedesk/versions/name/{{ atl_product_version }}" atl_servicedesk_url_map: diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index b8c0369..ca013e6 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -2,7 +2,7 @@ - name: Get the installer product version info uri: - url: "https://marketplace.atlassian.com/rest/2/products/key/jira-software/versions/name/{{ atl_product_version }}" + url: "{{ atl_mpac_products }}/key/jira-software/versions/name/{{ atl_product_version }}" return_content: yes register: atl_product_version_info @@ -12,7 +12,7 @@ - name: Get the JSD build version info uri: - url: "https://marketplace.atlassian.com/rest/2/products/key/jira-servicedesk/versions/latest?application=\ + url: "{{ atl_mpac_products }}/key/jira-servicedesk/versions/latest?application=\ jira&applicationBuild={{ atl_product_version_info.json.buildNumber }}" return_content: yes register: atl_jsd_build_info @@ -53,9 +53,6 @@ owner: "{{ atl_product_user }}" group: "{{ atl_product_user }}" mode: 0750 - # Ansible unarchive task validation does not work with the -j flag so we need to achieve the same result in 2 steps - # extra_opts: - # - "-j" - name: Copy JSD dependency jars into the installed-plugins dir copy: From 2513450d07a15b30f44af06a54ab0c016dc8e964 Mon Sep 17 00:00:00 2001 From: Geoff Jacobs Date: Wed, 30 Oct 2019 10:49:39 +1100 Subject: [PATCH 047/109] ITOPSENG-283 updating the method for managing the xmx and xms settings in setenv.sh --- roles/crowd_config/tasks/main.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/roles/crowd_config/tasks/main.yml b/roles/crowd_config/tasks/main.yml index f6e8493..ea31775 100644 --- a/roles/crowd_config/tasks/main.yml +++ b/roles/crowd_config/tasks/main.yml @@ -5,16 +5,19 @@ src: server.xml.j2 dest: "{{ atl_product_installation_versioned }}/apache-tomcat/conf/server.xml" -- name: Override JVM memory settings. - # Ugly but necessary as the product installs this file so we need to make the change here. +- name: Set the minimum heap size (Xms) lineinfile: path: "{{ atl_product_installation_versioned }}/apache-tomcat/bin/setenv.sh" - backrefs: true - regexp: "^{{ item }}=" - line: "{{ item }}=\"{{ atl_jvm_heap }}\"" - with_items: - - 'JVM_MINIMUM_MEMORY' - - 'JVM_MAXIMUM_MEMORY' + regexp: '^(.*)Xmx(\\d+\\w)(\\s.*)$' + line: '\1Xms${atl_jvm_heap}\3' + backrefs: yes + +- name: Set the maxmimum heap size (Xmx) + lineinfile: + path: "{{ atl_product_installation_versioned }}/apache-tomcat/bin/setenv.sh" + regexp: '^(.*)Xmx(\\d+\\w)(\\s.*)$' + line: '\1Xmx${xms}m\3' + backrefs: yes - name: Set Crowd home directory in crowd-init.properties file lineinfile: From 4c0cec450909f8c6382cb6689c166facb7c70504 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 30 Oct 2019 11:01:18 +1100 Subject: [PATCH 048/109] DCD-686: Check for no-op when no manifest URL specified. --- group_vars/aws_node_local.yml | 1 + roles/restore_backups/.yamllint | 12 +++++++ .../molecule/default/Dockerfile.j2 | 14 ++++++++ .../molecule/default/molecule.yml | 36 +++++++++++++++++++ .../molecule/default/playbook.yml | 10 ++++++ .../molecule/default/tests/test_default.py | 10 ++++++ roles/restore_backups/tasks/main.yml | 2 +- 7 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 roles/restore_backups/.yamllint create mode 100644 roles/restore_backups/molecule/default/Dockerfile.j2 create mode 100644 roles/restore_backups/molecule/default/molecule.yml create mode 100644 roles/restore_backups/molecule/default/playbook.yml create mode 100644 roles/restore_backups/molecule/default/tests/test_default.py diff --git a/group_vars/aws_node_local.yml b/group_vars/aws_node_local.yml index 69285f9..1675ba8 100644 --- a/group_vars/aws_node_local.yml +++ b/group_vars/aws_node_local.yml @@ -128,3 +128,4 @@ atl_rds_subnet_group_name: "{{ lookup('env', 'ATL_RDS_SUBNET_GROUP_NAME') }}" atl_rds_security_group: "{{ lookup('env', 'ATL_RDS_SECURITY_GROUP') }}" atl_backup_manifest_url: "{{ lookup('env', 'ATL_BACKUP_MANIFEST_URL') }}" +atl_restore_required: "{{ atl_backup_manifest_url is defined and atl_backup_manifest_url != '' }}" diff --git a/roles/restore_backups/.yamllint b/roles/restore_backups/.yamllint new file mode 100644 index 0000000..a87f8ff --- /dev/null +++ b/roles/restore_backups/.yamllint @@ -0,0 +1,12 @@ +extends: default + +rules: + braces: + max-spaces-inside: 1 + level: error + brackets: + max-spaces-inside: 1 + level: error + line-length: disable + truthy: disable + trailing-spaces: false diff --git a/roles/restore_backups/molecule/default/Dockerfile.j2 b/roles/restore_backups/molecule/default/Dockerfile.j2 new file mode 100644 index 0000000..e6aa95d --- /dev/null +++ b/roles/restore_backups/molecule/default/Dockerfile.j2 @@ -0,0 +1,14 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi diff --git a/roles/restore_backups/molecule/default/molecule.yml b/roles/restore_backups/molecule/default/molecule.yml new file mode 100644 index 0000000..7f082f6 --- /dev/null +++ b/roles/restore_backups/molecule/default/molecule.yml @@ -0,0 +1,36 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: amazon_linux2 + image: amazonlinux:2 + groups: + - aws_node_local + ulimits: + - nofile:262144:262144 + - name: ubuntu_lts + image: ubuntu:bionic + groups: + - aws_node_local + ulimits: + - nofile:262144:262144 +provisioner: + name: ansible + options: + skip-tags: runtime_pkg + lint: + name: ansible-lint + options: + x: ["701"] + inventory: + links: + group_vars: ../../../../group_vars/ +verifier: + name: testinfra + lint: + name: flake8 + enabled: false diff --git a/roles/restore_backups/molecule/default/playbook.yml b/roles/restore_backups/molecule/default/playbook.yml new file mode 100644 index 0000000..ffd0c12 --- /dev/null +++ b/roles/restore_backups/molecule/default/playbook.yml @@ -0,0 +1,10 @@ +--- +- name: Converge + hosts: all + vars: + atl_backup_manifest_url: '' + atl_backup_home_restore_canary_path: '/tmp/canary.tmp' + + roles: + # Should be no-op + - role: restore_backups diff --git a/roles/restore_backups/molecule/default/tests/test_default.py b/roles/restore_backups/molecule/default/tests/test_default.py new file mode 100644 index 0000000..0a7276f --- /dev/null +++ b/roles/restore_backups/molecule/default/tests/test_default.py @@ -0,0 +1,10 @@ +import os + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +def test_no_canary_file(host): + assert not host.file('atl_backup_home_restore_canary_path').exists diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index 18f2169..8767b4d 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -133,4 +133,4 @@ when: not restore_canary.stat.exists - when: atl_backup_manifest_url is defined and atl_backup_manifest_url != '' + when: atl_restore_required From e440daa1a5de40f9dae6d0fd6e10d4c09e61f676 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 30 Oct 2019 11:03:19 +1100 Subject: [PATCH 049/109] DCD-686: Update pipeline tests. --- bitbucket-pipelines.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index ec993c6..d28b140 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -14,7 +14,7 @@ pipelines: - step: name: Pre Parallelization stage script: - - echo "Running tests in 28 batches" + - echo "Running tests in 29 batches" - step: name: Check if number of batches match actual number of scenarios script: @@ -251,4 +251,12 @@ pipelines: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 28 + - step: + name: Molecule Test Batch - 29 + services: + - docker + script: + - apt-get update && ./bin/install-ansible --dev + - ./bin/run-tests-in-batches --batch 29 + From 5032371fed22d753cb2f3e9936be533bf26ec85f Mon Sep 17 00:00:00 2001 From: Geoff Jacobs Date: Wed, 30 Oct 2019 11:14:39 +1100 Subject: [PATCH 050/109] ITOPSENG-283 fixing typo and adding debug --- roles/crowd_config/tasks/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/roles/crowd_config/tasks/main.yml b/roles/crowd_config/tasks/main.yml index ea31775..3b5afdc 100644 --- a/roles/crowd_config/tasks/main.yml +++ b/roles/crowd_config/tasks/main.yml @@ -11,13 +11,15 @@ regexp: '^(.*)Xmx(\\d+\\w)(\\s.*)$' line: '\1Xms${atl_jvm_heap}\3' backrefs: yes + verbosity: 4 - name: Set the maxmimum heap size (Xmx) lineinfile: path: "{{ atl_product_installation_versioned }}/apache-tomcat/bin/setenv.sh" regexp: '^(.*)Xmx(\\d+\\w)(\\s.*)$' - line: '\1Xmx${xms}m\3' + line: '\1Xmx${atl_jvm_heap}\3' backrefs: yes + verbosity: 4 - name: Set Crowd home directory in crowd-init.properties file lineinfile: From 7d90e8b65a91dee94a4a5dfea90561f7c4a1cd01 Mon Sep 17 00:00:00 2001 From: Geoff Jacobs Date: Wed, 30 Oct 2019 11:21:34 +1100 Subject: [PATCH 051/109] ITOPSENG-283 more debug --- bin/ansible-with-atl-env | 2 +- roles/crowd_config/tasks/main.yml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/bin/ansible-with-atl-env b/bin/ansible-with-atl-env index 3685381..b3e05e0 100755 --- a/bin/ansible-with-atl-env +++ b/bin/ansible-with-atl-env @@ -18,7 +18,7 @@ set +a # Use Ansible from virtualenv if provided pipenv run \ - ansible-playbook -v \ + ansible-playbook -vvvv \ $ATL_DEPLOYMENT_REPOSITORY_CUSTOM_PARAMS \ -i $INV \ $PLAYBOOK \ diff --git a/roles/crowd_config/tasks/main.yml b/roles/crowd_config/tasks/main.yml index 3b5afdc..817ad2e 100644 --- a/roles/crowd_config/tasks/main.yml +++ b/roles/crowd_config/tasks/main.yml @@ -11,7 +11,6 @@ regexp: '^(.*)Xmx(\\d+\\w)(\\s.*)$' line: '\1Xms${atl_jvm_heap}\3' backrefs: yes - verbosity: 4 - name: Set the maxmimum heap size (Xmx) lineinfile: @@ -19,7 +18,6 @@ regexp: '^(.*)Xmx(\\d+\\w)(\\s.*)$' line: '\1Xmx${atl_jvm_heap}\3' backrefs: yes - verbosity: 4 - name: Set Crowd home directory in crowd-init.properties file lineinfile: From ee759efeadefd9a0d7cebb9656660770f23613a0 Mon Sep 17 00:00:00 2001 From: Geoff Jacobs Date: Wed, 30 Oct 2019 11:45:57 +1100 Subject: [PATCH 052/109] ITOPSENG-283 upstream ansible example had wrong syntax :facepalm: --- bin/ansible-with-atl-env | 2 +- roles/crowd_config/tasks/main.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/ansible-with-atl-env b/bin/ansible-with-atl-env index b3e05e0..3685381 100755 --- a/bin/ansible-with-atl-env +++ b/bin/ansible-with-atl-env @@ -18,7 +18,7 @@ set +a # Use Ansible from virtualenv if provided pipenv run \ - ansible-playbook -vvvv \ + ansible-playbook -v \ $ATL_DEPLOYMENT_REPOSITORY_CUSTOM_PARAMS \ -i $INV \ $PLAYBOOK \ diff --git a/roles/crowd_config/tasks/main.yml b/roles/crowd_config/tasks/main.yml index 817ad2e..b5b987c 100644 --- a/roles/crowd_config/tasks/main.yml +++ b/roles/crowd_config/tasks/main.yml @@ -8,15 +8,15 @@ - name: Set the minimum heap size (Xms) lineinfile: path: "{{ atl_product_installation_versioned }}/apache-tomcat/bin/setenv.sh" - regexp: '^(.*)Xmx(\\d+\\w)(\\s.*)$' - line: '\1Xms${atl_jvm_heap}\3' + regexp: '^(.*)Xms(\d+\w)(\s.*)$' + line: '\1Xms{{ atl_jvm_heap }}\3' backrefs: yes - name: Set the maxmimum heap size (Xmx) lineinfile: path: "{{ atl_product_installation_versioned }}/apache-tomcat/bin/setenv.sh" - regexp: '^(.*)Xmx(\\d+\\w)(\\s.*)$' - line: '\1Xmx${atl_jvm_heap}\3' + regexp: '^(.*)Xmx(\d+\w)(\s.*)$' + line: '\1Xmx{{ atl_jvm_heap }}\3' backrefs: yes - name: Set Crowd home directory in crowd-init.properties file From 1ea30531b6806ae4753cbb1c20efdcecd314a987 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 31 Oct 2019 10:30:17 +1100 Subject: [PATCH 053/109] DCD-686: Remove HTTP manifest download for now. --- roles/restore_backups/tasks/main.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index 8767b4d..4d6865f 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -1,8 +1,8 @@ --- # This role will attempt to fetch and load the backup manifest from a -# remote HTTP or S3 URL. On successful completion the contents of JSON -# or YAML document will be in the var `atl_backup_manifest`. +# remote S3 URL. On successful completion the contents of JSON or YAML +# document will be in the var `atl_backup_manifest`. # # PREREQUISITES: # * `atl_backup_manifest_url` points at the manifest. @@ -12,6 +12,9 @@ # # NOTE: The actual DB/FS restore operations could potentially be split # out into discrete roles, but currently that is not required. +# +# TODO: Support HTTPS with authentication. Deferred until after the +# initial testing release. - block: @@ -43,12 +46,6 @@ dest: "{{ atl_backup_manifest_dest }}" when: atl_backup_manifest_url.scheme == 's3' - - name: Fetch the manifest from remote host - get_url: - url: "{{ atl_backup_manifest_url }}" - dest: "{{ atl_backup_manifest_dest }}" - when: atl_backup_manifest_url.scheme != 's3' - - name: Load parameters from manifest include_vars: file: "{{ atl_backup_manifest_dest }}" From 2d2fa39cbe31ba6a09f21537391baf4247d74557 Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Thu, 31 Oct 2019 16:28:05 +1100 Subject: [PATCH 054/109] ITOPSENG-164 Testing change to molecule tests --- .../molecule/bitbucket_latest/tests/test_default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/product_install/molecule/bitbucket_latest/tests/test_default.py b/roles/product_install/molecule/bitbucket_latest/tests/test_default.py index 55c71c6..4229b1f 100644 --- a/roles/product_install/molecule/bitbucket_latest/tests/test_default.py +++ b/roles/product_install/molecule/bitbucket_latest/tests/test_default.py @@ -37,6 +37,6 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/opt/atlassian/tmp/bitbucket.' + upstream + '-x64.bin') + installer = host.file('/media/atl/bitbucket/shared/downloads/bitbucket.' + upstream + '-x64.bin') assert installer.exists assert installer.user == 'root' From 6e99ec440fe1eb4e30a9a37c17abb128ffcf3378 Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Thu, 31 Oct 2019 16:40:26 +1100 Subject: [PATCH 055/109] Revert "ITOPSENG-164 Testing change to molecule tests" This reverts commit 2d2fa39cbe31ba6a09f21537391baf4247d74557. --- .../molecule/bitbucket_latest/tests/test_default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/product_install/molecule/bitbucket_latest/tests/test_default.py b/roles/product_install/molecule/bitbucket_latest/tests/test_default.py index 4229b1f..55c71c6 100644 --- a/roles/product_install/molecule/bitbucket_latest/tests/test_default.py +++ b/roles/product_install/molecule/bitbucket_latest/tests/test_default.py @@ -37,6 +37,6 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/media/atl/bitbucket/shared/downloads/bitbucket.' + upstream + '-x64.bin') + installer = host.file('/opt/atlassian/tmp/bitbucket.' + upstream + '-x64.bin') assert installer.exists assert installer.user == 'root' From 36f870f20096567dab33c37cfe300dcadd6f8dda Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Fri, 1 Nov 2019 10:57:06 +1100 Subject: [PATCH 056/109] ITOPSENG-164 Testing changes --- .../molecule/bitbucket_latest/tests/test_default.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/product_install/molecule/bitbucket_latest/tests/test_default.py b/roles/product_install/molecule/bitbucket_latest/tests/test_default.py index 55c71c6..ed87338 100644 --- a/roles/product_install/molecule/bitbucket_latest/tests/test_default.py +++ b/roles/product_install/molecule/bitbucket_latest/tests/test_default.py @@ -37,6 +37,6 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/opt/atlassian/tmp/bitbucket.' + upstream + '-x64.bin') + installer = host.file('/media/atl/bitbucket/shared/downloads/bitbucket.' + upstream + '-x64.bin') assert installer.exists - assert installer.user == 'root' + assert installer.user == 'root' From c015984acec4d9aa5ad9ae49e595f7e1d4041453 Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Fri, 1 Nov 2019 11:32:38 +1100 Subject: [PATCH 057/109] ITOPSENG-164 Testing changes --- .../molecule/confluence_latest/tests/test_default.py | 2 +- roles/product_install/molecule/default/tests/test_default.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/product_install/molecule/confluence_latest/tests/test_default.py b/roles/product_install/molecule/confluence_latest/tests/test_default.py index 47245a4..0b6c684 100644 --- a/roles/product_install/molecule/confluence_latest/tests/test_default.py +++ b/roles/product_install/molecule/confluence_latest/tests/test_default.py @@ -35,6 +35,6 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/opt/atlassian/tmp/confluence.'+upstream+'-x64.bin') + installer = host.file('/media/atl/confluence/shared-home/downloads/confluence.'+upstream+'-x64.bin') assert installer.exists assert installer.user == 'root' diff --git a/roles/product_install/molecule/default/tests/test_default.py b/roles/product_install/molecule/default/tests/test_default.py index 70839b9..08626e3 100644 --- a/roles/product_install/molecule/default/tests/test_default.py +++ b/roles/product_install/molecule/default/tests/test_default.py @@ -23,6 +23,6 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/opt/atlassian/tmp/jira-core.'+upstream+'-x64.bin') + installer = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'-x64.bin') assert installer.exists assert installer.user == 'root' From 788bbd93dda7173131f30cd08169faecd9184ef5 Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Fri, 1 Nov 2019 11:58:31 +1100 Subject: [PATCH 058/109] ITOPSENG-164 Testing changes --- .../molecule/jira_version_latest/tests/test_default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/product_install/molecule/jira_version_latest/tests/test_default.py b/roles/product_install/molecule/jira_version_latest/tests/test_default.py index 70839b9..08626e3 100644 --- a/roles/product_install/molecule/jira_version_latest/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_latest/tests/test_default.py @@ -23,6 +23,6 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/opt/atlassian/tmp/jira-core.'+upstream+'-x64.bin') + installer = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'-x64.bin') assert installer.exists assert installer.user == 'root' From 8022b591573967fcb1e2773757f8d1d466504210 Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Fri, 1 Nov 2019 12:06:08 +1100 Subject: [PATCH 059/109] ITOPS-2158 testing --- .../product_install/molecule/jira_tarball/tests/test_default.py | 2 +- .../molecule/jira_version_from_file/tests/test_default.py | 2 +- .../molecule/jira_version_override/tests/test_default.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/product_install/molecule/jira_tarball/tests/test_default.py b/roles/product_install/molecule/jira_tarball/tests/test_default.py index 2f5d09b..d534aa5 100644 --- a/roles/product_install/molecule/jira_tarball/tests/test_default.py +++ b/roles/product_install/molecule/jira_tarball/tests/test_default.py @@ -23,6 +23,6 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/opt/atlassian/tmp/jira-core.'+upstream+'.tar.gz') + installer = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'.tar.gz') assert installer.exists assert installer.user == 'root' diff --git a/roles/product_install/molecule/jira_version_from_file/tests/test_default.py b/roles/product_install/molecule/jira_version_from_file/tests/test_default.py index 5f00577..902b5f5 100644 --- a/roles/product_install/molecule/jira_version_from_file/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_from_file/tests/test_default.py @@ -14,7 +14,7 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "7.9.0" def test_is_downloaded(host): - installer = host.file('/opt/atlassian/tmp/jira-core.7.9.0-x64.bin') + installer = host.file('/media/atl/jira/shared/downloads/jira-core.7.9.0-x64.bin') assert installer.exists assert installer.user == 'root' diff --git a/roles/product_install/molecule/jira_version_override/tests/test_default.py b/roles/product_install/molecule/jira_version_override/tests/test_default.py index 8b5c7a4..c2e28c5 100644 --- a/roles/product_install/molecule/jira_version_override/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_override/tests/test_default.py @@ -14,7 +14,7 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "7.13.2" def test_is_downloaded(host): - installer = host.file('/opt/atlassian/tmp/jira-core.7.13.2-x64.bin') + installer = host.file('/media/atl/jira/shared/downloads/jira-core.7.13.2-x64.bin') assert installer.exists assert installer.user == 'root' From 17366378de4895b2dd6aa28ea16fdf95cc63b799 Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Fri, 1 Nov 2019 12:22:50 +1100 Subject: [PATCH 060/109] ITOPS-2158 testing --- .../molecule/jira_cached_with_downgrade/tests/test_default.py | 2 +- .../molecule/jira_cached_with_upgrade/tests/test_default.py | 2 +- .../molecule/jira_software_latest/tests/test_default.py | 2 +- .../product_install/molecule/servicedesk3/tests/test_default.py | 2 +- .../product_install/molecule/servicedesk4/tests/test_default.py | 2 +- .../molecule/servicedesk_latest/tests/test_default.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py b/roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py index 05a6bb3..aad72bf 100644 --- a/roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py +++ b/roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py @@ -14,7 +14,7 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "7.10.2" def test_is_downloaded(host): - installer = host.file('/opt/atlassian/tmp/jira-core.7.10.2-x64.bin') + installer = host.file('/media/atl/jira/shared/downloads/jira-core.7.10.2-x64.bin') assert installer.exists assert installer.user == 'root' diff --git a/roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py b/roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py index ead6adf..c828961 100644 --- a/roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py +++ b/roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py @@ -14,7 +14,7 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "7.10.1" def test_is_downloaded(host): - installer = host.file('/opt/atlassian/tmp/jira-core.7.10.1-x64.bin') + installer = host.file('/media/atl/jira/shared/downloads/jira-core.7.10.1-x64.bin') assert installer.exists assert installer.user == 'root' diff --git a/roles/product_install/molecule/jira_software_latest/tests/test_default.py b/roles/product_install/molecule/jira_software_latest/tests/test_default.py index 63451c8..9a5b161 100644 --- a/roles/product_install/molecule/jira_software_latest/tests/test_default.py +++ b/roles/product_install/molecule/jira_software_latest/tests/test_default.py @@ -35,6 +35,6 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/opt/atlassian/tmp/jira-software.'+upstream+'-x64.bin') + installer = host.file('/media/atl/jira/shared/downloads/jira-software.'+upstream+'-x64.bin') assert installer.exists assert installer.user == 'root' diff --git a/roles/product_install/molecule/servicedesk3/tests/test_default.py b/roles/product_install/molecule/servicedesk3/tests/test_default.py index 24afb62..ed19b29 100644 --- a/roles/product_install/molecule/servicedesk3/tests/test_default.py +++ b/roles/product_install/molecule/servicedesk3/tests/test_default.py @@ -14,7 +14,7 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "3.9.0" def test_is_downloaded(host): - installer = host.file('/opt/atlassian/tmp/servicedesk.3.9.0-x64.bin') + installer = host.file('/media/atl/jira/shared/downloads/servicedesk.3.9.0-x64.bin') assert installer.exists assert installer.user == 'root' diff --git a/roles/product_install/molecule/servicedesk4/tests/test_default.py b/roles/product_install/molecule/servicedesk4/tests/test_default.py index b660f23..e7d1179 100644 --- a/roles/product_install/molecule/servicedesk4/tests/test_default.py +++ b/roles/product_install/molecule/servicedesk4/tests/test_default.py @@ -14,7 +14,7 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "4.1.0" def test_is_downloaded(host): - installer = host.file('/opt/atlassian/tmp/servicedesk.4.1.0-x64.bin') + installer = host.file('/media/atl/jira/shared/downloads/servicedesk.4.1.0-x64.bin') assert installer.exists assert installer.user == 'root' diff --git a/roles/product_install/molecule/servicedesk_latest/tests/test_default.py b/roles/product_install/molecule/servicedesk_latest/tests/test_default.py index 2190295..ee855e3 100644 --- a/roles/product_install/molecule/servicedesk_latest/tests/test_default.py +++ b/roles/product_install/molecule/servicedesk_latest/tests/test_default.py @@ -23,7 +23,7 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == sd def test_is_downloaded(host): - installer = host.file('/opt/atlassian/tmp/servicedesk.'+sd+'-x64.bin') + installer = host.file('/media/atl/jira/shared/downloads/servicedesk.'+sd+'-x64.bin') assert installer.exists assert installer.user == 'root' From e55593bf6c50171a8618bbe89b0f77905a2860a9 Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Fri, 1 Nov 2019 13:35:30 +1100 Subject: [PATCH 061/109] ITOPS-2158 testing --- .../molecule/bitbucket_latest/tests/test_default.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/roles/product_install/molecule/bitbucket_latest/tests/test_default.py b/roles/product_install/molecule/bitbucket_latest/tests/test_default.py index ed87338..6c28b1b 100644 --- a/roles/product_install/molecule/bitbucket_latest/tests/test_default.py +++ b/roles/product_install/molecule/bitbucket_latest/tests/test_default.py @@ -39,4 +39,8 @@ def test_latest_is_downloaded(host): installer = host.file('/media/atl/bitbucket/shared/downloads/bitbucket.' + upstream + '-x64.bin') assert installer.exists - assert installer.user == 'root' + assert installer.user == 'root' + +def test_completed_lockfile(host): + verfile = host.file('/media/atl/bitbucket/shared/downloads/bitbucket.' + upstream + '-x64.bin_completed') + assert verfile.exists From ae673495276cdd1c7068458e305c352e212a7fc0 Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Fri, 1 Nov 2019 13:51:16 +1100 Subject: [PATCH 062/109] ITOPS-2158 testing --- .../molecule/bitbucket_latest/tests/test_default.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/product_install/molecule/bitbucket_latest/tests/test_default.py b/roles/product_install/molecule/bitbucket_latest/tests/test_default.py index 6c28b1b..c709022 100644 --- a/roles/product_install/molecule/bitbucket_latest/tests/test_default.py +++ b/roles/product_install/molecule/bitbucket_latest/tests/test_default.py @@ -42,5 +42,10 @@ def test_latest_is_downloaded(host): assert installer.user == 'root' def test_completed_lockfile(host): + upstream_fd = urllib.request.urlopen( + "https://marketplace.atlassian.com/rest/2/applications/bitbucket/versions/latest") + upstream_json = json.load(upstream_fd) + upstream = upstream_json['version'] + verfile = host.file('/media/atl/bitbucket/shared/downloads/bitbucket.' + upstream + '-x64.bin_completed') assert verfile.exists From c10b312148f373fff987d48858dbeb3925e7643c Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Fri, 1 Nov 2019 14:52:57 +1100 Subject: [PATCH 063/109] ITOPS-2158 Remaining molecule unit tests --- .../molecule/bitbucket_latest/tests/test_default.py | 7 ++++--- .../molecule/confluence_latest/tests/test_default.py | 9 +++++++++ .../molecule/default/tests/test_default.py | 9 +++++++++ .../jira_cached_with_downgrade/tests/test_default.py | 5 +++++ .../jira_cached_with_upgrade/tests/test_default.py | 5 +++++ .../molecule/jira_software_latest/tests/test_default.py | 9 +++++++++ .../molecule/jira_tarball/tests/test_default.py | 9 +++++++++ .../jira_version_from_file/tests/test_default.py | 5 +++++ .../molecule/jira_version_latest/tests/test_default.py | 9 +++++++++ .../molecule/jira_version_override/tests/test_default.py | 5 +++++ .../molecule/servicedesk3/tests/test_default.py | 5 +++++ .../molecule/servicedesk4/tests/test_default.py | 5 +++++ .../molecule/servicedesk_latest/tests/test_default.py | 5 +++++ 13 files changed, 84 insertions(+), 3 deletions(-) diff --git a/roles/product_install/molecule/bitbucket_latest/tests/test_default.py b/roles/product_install/molecule/bitbucket_latest/tests/test_default.py index c709022..bbd851d 100644 --- a/roles/product_install/molecule/bitbucket_latest/tests/test_default.py +++ b/roles/product_install/molecule/bitbucket_latest/tests/test_default.py @@ -46,6 +46,7 @@ def test_completed_lockfile(host): "https://marketplace.atlassian.com/rest/2/applications/bitbucket/versions/latest") upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - - verfile = host.file('/media/atl/bitbucket/shared/downloads/bitbucket.' + upstream + '-x64.bin_completed') - assert verfile.exists + + lockfile = host.file('/media/atl/bitbucket/shared/downloads/bitbucket.' + upstream + '-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' diff --git a/roles/product_install/molecule/confluence_latest/tests/test_default.py b/roles/product_install/molecule/confluence_latest/tests/test_default.py index 0b6c684..7ec072b 100644 --- a/roles/product_install/molecule/confluence_latest/tests/test_default.py +++ b/roles/product_install/molecule/confluence_latest/tests/test_default.py @@ -38,3 +38,12 @@ def test_latest_is_downloaded(host): installer = host.file('/media/atl/confluence/shared-home/downloads/confluence.'+upstream+'-x64.bin') assert installer.exists assert installer.user == 'root' + +def test_completed_lockfile(host): + upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/confluence/versions/latest") + upstream_json = json.load(upstream_fd) + upstream = upstream_json['version'] + + lockfile = host.file('/media/atl/confluence/shared-home/downloads/confluence.'+upstream+'-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' diff --git a/roles/product_install/molecule/default/tests/test_default.py b/roles/product_install/molecule/default/tests/test_default.py index 08626e3..930ab59 100644 --- a/roles/product_install/molecule/default/tests/test_default.py +++ b/roles/product_install/molecule/default/tests/test_default.py @@ -26,3 +26,12 @@ def test_latest_is_downloaded(host): installer = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'-x64.bin') assert installer.exists assert installer.user == 'root' + +def test_completed_lockfile(host): + upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/jira/versions/latest") + upstream_json = json.load(upstream_fd) + upstream = upstream_json['version'] + + lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' \ No newline at end of file diff --git a/roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py b/roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py index aad72bf..788c3de 100644 --- a/roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py +++ b/roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py @@ -18,6 +18,11 @@ def test_is_downloaded(host): assert installer.exists assert installer.user == 'root' +def test_completed_lockfile(host): + lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.7.10.2-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' + def test_is_unpacked(host): installer = host.file('/opt/atlassian/jira-core/7.10.2/atlassian-jira/') assert installer.exists diff --git a/roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py b/roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py index c828961..0818e1b 100644 --- a/roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py +++ b/roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py @@ -18,6 +18,11 @@ def test_is_downloaded(host): assert installer.exists assert installer.user == 'root' +def test_completed_lockfile(host): + lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.7.10.1-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' + def test_is_unpacked(host): installer = host.file('/opt/atlassian/jira-core/7.10.1/atlassian-jira/') assert installer.exists diff --git a/roles/product_install/molecule/jira_software_latest/tests/test_default.py b/roles/product_install/molecule/jira_software_latest/tests/test_default.py index 9a5b161..de1dca3 100644 --- a/roles/product_install/molecule/jira_software_latest/tests/test_default.py +++ b/roles/product_install/molecule/jira_software_latest/tests/test_default.py @@ -38,3 +38,12 @@ def test_latest_is_downloaded(host): installer = host.file('/media/atl/jira/shared/downloads/jira-software.'+upstream+'-x64.bin') assert installer.exists assert installer.user == 'root' + +def test_completed_lockfile(host): + upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/jira/versions/latest") + upstream_json = json.load(upstream_fd) + upstream = upstream_json['version'] + + lockfile = host.file('/media/atl/jira/shared/downloads/jira-software.'+upstream+'-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' \ No newline at end of file diff --git a/roles/product_install/molecule/jira_tarball/tests/test_default.py b/roles/product_install/molecule/jira_tarball/tests/test_default.py index d534aa5..11a7438 100644 --- a/roles/product_install/molecule/jira_tarball/tests/test_default.py +++ b/roles/product_install/molecule/jira_tarball/tests/test_default.py @@ -26,3 +26,12 @@ def test_latest_is_downloaded(host): installer = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'.tar.gz') assert installer.exists assert installer.user == 'root' + +def test_completed_lockfile(host): + upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/jira/versions/latest") + upstream_json = json.load(upstream_fd) + upstream = upstream_json['version'] + + lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'.tar.gz_completed') + assert lockfile.exists + assert lockfile.user == 'root' \ No newline at end of file diff --git a/roles/product_install/molecule/jira_version_from_file/tests/test_default.py b/roles/product_install/molecule/jira_version_from_file/tests/test_default.py index 902b5f5..b8a1966 100644 --- a/roles/product_install/molecule/jira_version_from_file/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_from_file/tests/test_default.py @@ -18,6 +18,11 @@ def test_is_downloaded(host): assert installer.exists assert installer.user == 'root' +def test_completed_lockfile(host): + lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.7.9.0-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' + def test_is_unpacked(host): installer = host.file('/opt/atlassian/jira-core/7.9.0/atlassian-jira/') assert installer.exists diff --git a/roles/product_install/molecule/jira_version_latest/tests/test_default.py b/roles/product_install/molecule/jira_version_latest/tests/test_default.py index 08626e3..930ab59 100644 --- a/roles/product_install/molecule/jira_version_latest/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_latest/tests/test_default.py @@ -26,3 +26,12 @@ def test_latest_is_downloaded(host): installer = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'-x64.bin') assert installer.exists assert installer.user == 'root' + +def test_completed_lockfile(host): + upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/jira/versions/latest") + upstream_json = json.load(upstream_fd) + upstream = upstream_json['version'] + + lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' \ No newline at end of file diff --git a/roles/product_install/molecule/jira_version_override/tests/test_default.py b/roles/product_install/molecule/jira_version_override/tests/test_default.py index c2e28c5..3f16801 100644 --- a/roles/product_install/molecule/jira_version_override/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_override/tests/test_default.py @@ -18,6 +18,11 @@ def test_is_downloaded(host): assert installer.exists assert installer.user == 'root' +def test_completed_lockfile(host): + lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.7.13.2-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' + def test_is_unpacked(host): installer = host.file('/opt/atlassian/jira-core/7.13.2') assert installer.exists diff --git a/roles/product_install/molecule/servicedesk3/tests/test_default.py b/roles/product_install/molecule/servicedesk3/tests/test_default.py index ed19b29..5f50b6e 100644 --- a/roles/product_install/molecule/servicedesk3/tests/test_default.py +++ b/roles/product_install/molecule/servicedesk3/tests/test_default.py @@ -18,6 +18,11 @@ def test_is_downloaded(host): assert installer.exists assert installer.user == 'root' +def test_completed_lockfile(host): + lockfile = host.file('/media/atl/jira/shared/downloads/servicedesk.3.9.0-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' + def test_is_unpacked(host): installer = host.file('/opt/atlassian/jira-servicedesk/3.9.0') assert installer.exists diff --git a/roles/product_install/molecule/servicedesk4/tests/test_default.py b/roles/product_install/molecule/servicedesk4/tests/test_default.py index e7d1179..5a22e8c 100644 --- a/roles/product_install/molecule/servicedesk4/tests/test_default.py +++ b/roles/product_install/molecule/servicedesk4/tests/test_default.py @@ -18,6 +18,11 @@ def test_is_downloaded(host): assert installer.exists assert installer.user == 'root' +def test_completed_lockfile(host): + lockfile = host.file('/media/atl/jira/shared/downloads/servicedesk.4.1.0-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' + def test_is_unpacked(host): installer = host.file('/opt/atlassian/jira-servicedesk/4.1.0') assert installer.exists diff --git a/roles/product_install/molecule/servicedesk_latest/tests/test_default.py b/roles/product_install/molecule/servicedesk_latest/tests/test_default.py index ee855e3..cd975f3 100644 --- a/roles/product_install/molecule/servicedesk_latest/tests/test_default.py +++ b/roles/product_install/molecule/servicedesk_latest/tests/test_default.py @@ -27,6 +27,11 @@ def test_is_downloaded(host): assert installer.exists assert installer.user == 'root' +def test_completed_lockfile(host): + lockfile = host.file('/media/atl/jira/shared/downloads/servicedesk.'+sd+'-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' + def test_is_unpacked(host): installer = host.file('/opt/atlassian/jira-servicedesk/'+sd) assert installer.exists From d7ef24eaa94ffdf08f1f055fc0d2699e0ee360b0 Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Fri, 1 Nov 2019 15:27:17 +1100 Subject: [PATCH 064/109] ITOPS-2158 Added latest crowd molecule test --- .../molecule/crowd_latest/Dockerfile.j2 | 14 ++++++ .../molecule/crowd_latest/molecule.yml | 30 ++++++++++++ .../molecule/crowd_latest/playbook.yml | 11 +++++ .../crowd_latest/tests/test_default.py | 49 +++++++++++++++++++ 4 files changed, 104 insertions(+) create mode 100644 roles/product_install/molecule/crowd_latest/Dockerfile.j2 create mode 100644 roles/product_install/molecule/crowd_latest/molecule.yml create mode 100644 roles/product_install/molecule/crowd_latest/playbook.yml create mode 100644 roles/product_install/molecule/crowd_latest/tests/test_default.py diff --git a/roles/product_install/molecule/crowd_latest/Dockerfile.j2 b/roles/product_install/molecule/crowd_latest/Dockerfile.j2 new file mode 100644 index 0000000..e6aa95d --- /dev/null +++ b/roles/product_install/molecule/crowd_latest/Dockerfile.j2 @@ -0,0 +1,14 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi diff --git a/roles/product_install/molecule/crowd_latest/molecule.yml b/roles/product_install/molecule/crowd_latest/molecule.yml new file mode 100644 index 0000000..7fd3163 --- /dev/null +++ b/roles/product_install/molecule/crowd_latest/molecule.yml @@ -0,0 +1,30 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: amazon_linux2 + image: amazonlinux:2 + groups: + - aws_node_local + - name: ubuntu_lts + image: ubuntu:bionic + groups: + - aws_node_local +provisioner: + name: ansible + options: + skip-tags: runtime_pkg + lint: + name: ansible-lint + inventory: + links: + group_vars: ../../../../group_vars/ +verifier: + name: testinfra + lint: + name: flake8 + enabled: false diff --git a/roles/product_install/molecule/crowd_latest/playbook.yml b/roles/product_install/molecule/crowd_latest/playbook.yml new file mode 100644 index 0000000..3373f8a --- /dev/null +++ b/roles/product_install/molecule/crowd_latest/playbook.yml @@ -0,0 +1,11 @@ +--- +- name: Converge + hosts: all + vars: + atl_product_family: "crowd" + atl_product_edition: "crowd" + atl_product_user: "crowd" + roles: + - role: linux_common + - role: product_common + - role: product_install diff --git a/roles/product_install/molecule/crowd_latest/tests/test_default.py b/roles/product_install/molecule/crowd_latest/tests/test_default.py new file mode 100644 index 0000000..64c8b58 --- /dev/null +++ b/roles/product_install/molecule/crowd_latest/tests/test_default.py @@ -0,0 +1,49 @@ +import os +from six.moves import urllib +import json + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + +def test_version_downloaded(host): + verfile = host.file('/media/atl/crowd/shared/crowd.version') + assert verfile.exists + +def test_symlink_created(host): + target = host.file('/opt/atlassian/crowd/current') + assert target.exists + assert target.is_symlink + +def test_unpacked(host): + verfile = host.file('/opt/atlassian/crowd/current/bin/catalina.sh') + assert verfile.exists + +def test_version_file_is_latest(host): + verfile = host.file('/media/atl/crowd/shared/crowd.version') + assert verfile.exists + + upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/crowd/versions/latest") + upstream_json = json.load(upstream_fd) + upstream = upstream_json['version'] + + assert verfile.content.decode("UTF-8").strip() == upstream.strip() + +def test_latest_is_downloaded(host): + upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/crowd/versions/latest") + upstream_json = json.load(upstream_fd) + upstream = upstream_json['version'] + + installer = host.file('/media/atl/crowd/shared/downloads/crowd.'+upstream+'-x64.bin') + assert installer.exists + assert installer.user == 'root' + +def test_completed_lockfile(host): + upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/crowd/versions/latest") + upstream_json = json.load(upstream_fd) + upstream = upstream_json['version'] + + lockfile = host.file('/media/atl/crowd/shared/downloads/crowd.'+upstream+'-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' From 2b774cd8e67c4ccb94fbe6282da38ef0eea28c96 Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Fri, 1 Nov 2019 15:35:41 +1100 Subject: [PATCH 065/109] ITOPS-2158 Added latest crowd molecule test --- bitbucket-pipelines.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index d28b140..11d18bd 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -259,4 +259,10 @@ pipelines: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 29 - + - step: + name: Molecule Test Batch - 30 + services: + - docker + script: + - apt-get update && ./bin/install-ansible --dev + - ./bin/run-tests-in-batches --batch 30 From 3032a001be895e8ff84a5283048cdb65d86f0a76 Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Fri, 1 Nov 2019 15:42:56 +1100 Subject: [PATCH 066/109] ITOPS-2158 Added latest crowd molecule test --- bitbucket-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 11d18bd..34b84dd 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -14,7 +14,7 @@ pipelines: - step: name: Pre Parallelization stage script: - - echo "Running tests in 29 batches" + - echo "Running tests in 30 batches" - step: name: Check if number of batches match actual number of scenarios script: @@ -266,3 +266,4 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 30 + From cadae8248f949f6b08ff31c0fec5f38af09446dc Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Fri, 1 Nov 2019 16:19:51 +1100 Subject: [PATCH 067/109] ITOPS-2158 Changed run_user test to be more specific --- bitbucket-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 34b84dd..408b078 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -267,3 +267,4 @@ pipelines: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 30 + From 531fb18294afbf702cfac3b441e06434a32288bb Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Mon, 4 Nov 2019 10:39:59 +1100 Subject: [PATCH 068/109] ITOPS-2158 Changed run_user test to be more specific --- .../molecule/crowd_latest/tests/test_default.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/roles/product_install/molecule/crowd_latest/tests/test_default.py b/roles/product_install/molecule/crowd_latest/tests/test_default.py index 64c8b58..51ec463 100644 --- a/roles/product_install/molecule/crowd_latest/tests/test_default.py +++ b/roles/product_install/molecule/crowd_latest/tests/test_default.py @@ -17,33 +17,36 @@ def test_symlink_created(host): assert target.is_symlink def test_unpacked(host): - verfile = host.file('/opt/atlassian/crowd/current/bin/catalina.sh') + verfile = host.file('/opt/atlassian/crowd/current/bin/start-crowd.sh') assert verfile.exists def test_version_file_is_latest(host): verfile = host.file('/media/atl/crowd/shared/crowd.version') assert verfile.exists - upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/crowd/versions/latest") + upstream_fd = urllib.request.urlopen( + "https://marketplace.atlassian.com/rest/2/applications/crowd/versions/latest") upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] assert verfile.content.decode("UTF-8").strip() == upstream.strip() def test_latest_is_downloaded(host): - upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/crowd/versions/latest") + upstream_fd = urllib.request.urlopen( + "https://marketplace.atlassian.com/rest/2/applications/crowd/versions/latest") upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/media/atl/crowd/shared/downloads/crowd.'+upstream+'-x64.bin') + installer = host.file('/media/atl/crowd/shared/downloads/crowd.' + upstream + '-x64.bin') assert installer.exists assert installer.user == 'root' def test_completed_lockfile(host): - upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/crowd/versions/latest") + upstream_fd = urllib.request.urlopen( + "https://marketplace.atlassian.com/rest/2/applications/crowd/versions/latest") upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - lockfile = host.file('/media/atl/crowd/shared/downloads/crowd.'+upstream+'-x64.bin_completed') + lockfile = host.file('/media/atl/crowd/shared/downloads/crowd.' + upstream + '-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' From 952c8ac97e55830554c494d7561074b9145b6eb8 Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Mon, 4 Nov 2019 11:03:56 +1100 Subject: [PATCH 069/109] ITOPS-2158 Added crowd test - testing --- .../product_install/molecule/crowd_latest/tests/test_default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/product_install/molecule/crowd_latest/tests/test_default.py b/roles/product_install/molecule/crowd_latest/tests/test_default.py index 51ec463..eb186dc 100644 --- a/roles/product_install/molecule/crowd_latest/tests/test_default.py +++ b/roles/product_install/molecule/crowd_latest/tests/test_default.py @@ -17,7 +17,7 @@ def test_symlink_created(host): assert target.is_symlink def test_unpacked(host): - verfile = host.file('/opt/atlassian/crowd/current/bin/start-crowd.sh') + verfile = host.file('/opt/atlassian/crowd/current/start_crowd.sh') assert verfile.exists def test_version_file_is_latest(host): From 0b129bce1d2a60044c4279bbc047699692559ff3 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Mon, 4 Nov 2019 11:14:11 +1100 Subject: [PATCH 070/109] DCD-431: Fix incorrect service start-up target and make configurable. --- roles/product_startup/defaults/main.yml | 3 ++- roles/product_startup/templates/product.service.j2 | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/product_startup/defaults/main.yml b/roles/product_startup/defaults/main.yml index ca1eda5..45d6f8a 100644 --- a/roles/product_startup/defaults/main.yml +++ b/roles/product_startup/defaults/main.yml @@ -14,5 +14,6 @@ atl_startup_exec_path: "{{ atl_product_installation_current }}/{{ atl_startup_sc atl_startup_exec_options: ["-fg"] atl_startup_systemd_params: [] - atl_systemd_service_name: "{{ atl_product_edition }}.service" + +atl_systemd_service_target: "multi-user.target" diff --git a/roles/product_startup/templates/product.service.j2 b/roles/product_startup/templates/product.service.j2 index 6b5077f..8310e88 100644 --- a/roles/product_startup/templates/product.service.j2 +++ b/roles/product_startup/templates/product.service.j2 @@ -15,4 +15,4 @@ ExecStart={{ atl_startup_exec_path }}{% for c in atl_startup_exec_options %} {{ Restart=on-failure [Install] -WantedBy=multi-target.target +WantedBy={{ atl_systemd_service_target }} From acfd10d2ae475347868ac6008052097aecbdf124 Mon Sep 17 00:00:00 2001 From: Glenn Stewart Date: Mon, 4 Nov 2019 11:39:08 +1100 Subject: [PATCH 071/109] ITOPS-2158 Added crowd test - testing --- roles/product_install/molecule/crowd_latest/playbook.yml | 1 + .../molecule/crowd_latest/tests/test_default.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/product_install/molecule/crowd_latest/playbook.yml b/roles/product_install/molecule/crowd_latest/playbook.yml index 3373f8a..490514e 100644 --- a/roles/product_install/molecule/crowd_latest/playbook.yml +++ b/roles/product_install/molecule/crowd_latest/playbook.yml @@ -5,6 +5,7 @@ atl_product_family: "crowd" atl_product_edition: "crowd" atl_product_user: "crowd" + atl_download_format: "tarball" roles: - role: linux_common - role: product_common diff --git a/roles/product_install/molecule/crowd_latest/tests/test_default.py b/roles/product_install/molecule/crowd_latest/tests/test_default.py index eb186dc..b75a0b5 100644 --- a/roles/product_install/molecule/crowd_latest/tests/test_default.py +++ b/roles/product_install/molecule/crowd_latest/tests/test_default.py @@ -37,7 +37,7 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/media/atl/crowd/shared/downloads/crowd.' + upstream + '-x64.bin') + installer = host.file('/media/atl/crowd/shared/downloads/crowd.' + upstream + '.tar.gz') assert installer.exists assert installer.user == 'root' @@ -47,6 +47,6 @@ def test_completed_lockfile(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - lockfile = host.file('/media/atl/crowd/shared/downloads/crowd.' + upstream + '-x64.bin_completed') + lockfile = host.file('/media/atl/crowd/shared/downloads/crowd.' + upstream + '.tar.gz_completed') assert lockfile.exists assert lockfile.user == 'root' From 353480916bcc821371f9d4071ac764dd2f82ff91 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 8 Nov 2019 11:28:45 +1100 Subject: [PATCH 072/109] ITOPSENG-101 add the locking logic for storing in shared_home --- .../tests/test_default.py | 6 +- .../tasks/jira-servicedesk_as_obr.yml | 139 ++++++++++++++++-- 2 files changed, 128 insertions(+), 17 deletions(-) diff --git a/roles/product_install/molecule/jira_version_from_file/tests/test_default.py b/roles/product_install/molecule/jira_version_from_file/tests/test_default.py index b8a1966..8b75761 100644 --- a/roles/product_install/molecule/jira_version_from_file/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_from_file/tests/test_default.py @@ -11,15 +11,15 @@ def test_version_is_correct(host): verfile = host.file('/media/atl/jira/shared/jira-core.version') assert verfile.exists - assert verfile.content.decode("UTF-8").strip() == "7.9.0" + assert verfile.content.decode("UTF-8").strip() == "7.13.1" def test_is_downloaded(host): - installer = host.file('/media/atl/jira/shared/downloads/jira-core.7.9.0-x64.bin') + installer = host.file('/media/atl/jira/shared/downloads/jira-core.7.13.1-x64.bin') assert installer.exists assert installer.user == 'root' def test_completed_lockfile(host): - lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.7.9.0-x64.bin_completed') + lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.7.13.1-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index ca013e6..f6b62db 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -21,21 +21,132 @@ debug: msg="obr_ref={{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}" -- name: Fetch the jsd obr - get_url: - url: "{{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}" - dest: "{{ atl_installer_temp }}" - register: jsdobr - -- name: check the name of the downloaded file +- name: how about getting the obr filename debug: - msg="{{ jsdobr.dest }}" + msg="obr_name=jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr" -- name: Copy the obr to shared_home - copy: - src: "{{ jsdobr.dest }}" - dest: "{{ atl_product_version_cache_dir }}" - remote_src: true +- name: is shared_home set ? + debug: + msg="atl_product_home_shared_download_dir={{ atl_product_home_shared_download_dir }}" +# For the first run a temp obr should be downloaded but moved to +# shared home to ensure all subsequent nodes have access +# to the same specific version binary. +# To prevent a race condition with multiple downloads at the same time +# a directory is used as a lockfile (atomic operation) when moving obr. + +- name: Set assumptions to avoid race condition + set_fact: + download_obr: true + move_obr: false + atl_obr_download_href: "{{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}" + atl_obr_filename: "jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr" + atl_obr_temp_download: "{{ atl_installer_temp }}/{{ atl_obr_filename }}" + atl_obr_shared_download: "{{ atl_product_home_shared_download_dir }}/{{ atl_obr_filename }}" + atl_obr_moving_lock: "{{{ atl_product_home_shared_download_dir }}/{{ atl_obr_filename }}_moving" + atl_obr_completed_lock: "{{ atl_product_home_shared_download_dir }}/{{ atl_obr_filename }}_completed" + +# Check for pre-downloaded obr on shared_home and completed lock dir. +- name: Check for completed lock directory + stat: + path: "{{ atl_obr_completed_lock }}" + register: completed_lock + +- name: Check for obr in home_shared + stat: + path: "{{ atl_obr_shared_download }}" + register: home_shared_download + +# If obr exists and lockdir exists use this obr instead +- name: Check lock directory and obr exists on shared_home + set_fact: + download_obr: false + atl_obr_download: "{{ atl_obr_shared_download }}" + when: + - home_shared_download.stat.exists + - completed_lock.stat.isdir is defined + - completed_lock.stat.isdir + +# Fetch obr if required +- name: download_obr is true so fetch and do all the things + block: + + # Fetch obr and copy to temp + - name: Fetch obr + get_url: + url: "{{ atl_obr_download_href }}" + dest: "{{ atl_obr_temp_download }}" + mode: 0755 + force: false + register: atl_obr_completed + + # If obr was fetched make the lock directory + - name: Create moving_lock. + file: + path: "{{ atl_obr_moving_lock }}" + state: directory + when: + - atl_obr_completed is succeeded + register: moving_lock_created + + # Directory lock was created by this run? + # If so, then set a fact intending to move obr + - name: Move obr Scenario - lock created by this run + set_fact: + move_obr: true + when: + - moving_lock_created is succeeded + - moving_lock_created.changed + # Otherwise directory lock was either already created or + # could not be created. Fall back is to continue and install from temp + + when: download_obr + +# If the intention is to move obr to home_shared +- name: Move obr to home_shared + block: + + - name: Copy temp installer to home_shared + copy: + src: "{{ atl_obr_temp_download }}" + dest: "{{ atl_obr_shared_download }}" + remote_src: true + when: + - moving_lock_created is succeeded + - moving_lock_created.changed + register: copied + + - name: Create completed_lock once obr downloaded and copied + file: + path: "{{ atl_obr_shared_completed_lock }}" + state: directory + when: copied is succeeded + register: completed_lock_created + + - name: Remove moving_lock to show that obr is completed + file: + path: "{{ atl_obr_shared_moving_lock }}" + state: absent + when: + - completed_lock_created is succeeded + - copied is succeeded + register: moving_lock_removed + + - name: Delete old temp installer + file: + path: "{{ atl_obr_temp_download }}" + state: absent + when: moving_lock_removed is succeeded + register: temp_deleted + + - name: Set install to home_shared location + set_fact: + atl_obr_download: "{{ atl_obr_home_shared_download }}" + when: temp_deleted is succeeded + + when: move_obr + +# At this point the binary is in {{ atl_obr_download }} +# (which is either on home_shared or temp) - name: Ensure instaled-plugins dir exists file: @@ -48,7 +159,7 @@ - name: Unpack the obr into the installed-plugins dir unarchive: remote_src: true - src: "{{ jsdobr.dest }}" + src: "{{ atl_obr_download }}" dest: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins" owner: "{{ atl_product_user }}" group: "{{ atl_product_user }}" From 4686fd3b33908d3901aed543b103122ef1b00bb8 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 8 Nov 2019 11:32:56 +1100 Subject: [PATCH 073/109] ITOPSENG-101 move downloads dir directly under shared mountpoint so that it is not backed up by backmac --- roles/product_install/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/product_install/defaults/main.yml b/roles/product_install/defaults/main.yml index 67b01a6..6a9940e 100644 --- a/roles/product_install/defaults/main.yml +++ b/roles/product_install/defaults/main.yml @@ -22,7 +22,7 @@ atl_product_download_filename: "{{ atl_download_edition | default(atl_product_ed atl_product_temp_download: "{{ atl_installer_temp }}/{{ atl_product_download_filename }}" atl_product_varfile: "{{ atl_installer_temp }}/{{ atl_product_family }}.varfile" -atl_product_home_shared_download_dir: "{{ atl_product_home_shared }}/downloads" +atl_product_home_shared_download_dir: "{{ atl_shared_mountpoint }}/downloads" atl_product_home_shared_download: "{{ atl_product_home_shared_download_dir }}/{{ atl_product_download_filename }}" atl_product_home_shared_moving_lock: "{{ atl_product_home_shared_download }}_moving" atl_product_home_shared_completed_lock: "{{ atl_product_home_shared_download }}_completed" From 6f040584d7eebf4de1c2ed22897441b98cf8c1f5 Mon Sep 17 00:00:00 2001 From: bmeehan Date: Fri, 8 Nov 2019 01:18:26 +0000 Subject: [PATCH 074/109] ITOPSENG-101 working obr storage logic --- .../tasks/jira-servicedesk_as_obr.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index f6b62db..6e7d0f1 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -40,10 +40,10 @@ move_obr: false atl_obr_download_href: "{{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}" atl_obr_filename: "jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr" - atl_obr_temp_download: "{{ atl_installer_temp }}/{{ atl_obr_filename }}" - atl_obr_shared_download: "{{ atl_product_home_shared_download_dir }}/{{ atl_obr_filename }}" - atl_obr_moving_lock: "{{{ atl_product_home_shared_download_dir }}/{{ atl_obr_filename }}_moving" - atl_obr_completed_lock: "{{ atl_product_home_shared_download_dir }}/{{ atl_obr_filename }}_completed" + atl_obr_download: "{{ atl_installer_temp }}/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr" + atl_obr_shared_download: "{{ atl_product_home_shared_download_dir }}/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr" + atl_obr_moving_lock: "{{ atl_product_home_shared_download_dir }}/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr_moving" + atl_obr_completed_lock: "{{ atl_product_home_shared_download_dir }}/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr_completed" # Check for pre-downloaded obr on shared_home and completed lock dir. - name: Check for completed lock directory @@ -74,7 +74,7 @@ - name: Fetch obr get_url: url: "{{ atl_obr_download_href }}" - dest: "{{ atl_obr_temp_download }}" + dest: "{{ atl_obr_download }}" mode: 0755 force: false register: atl_obr_completed @@ -107,7 +107,7 @@ - name: Copy temp installer to home_shared copy: - src: "{{ atl_obr_temp_download }}" + src: "{{ atl_obr_download }}" dest: "{{ atl_obr_shared_download }}" remote_src: true when: @@ -117,14 +117,14 @@ - name: Create completed_lock once obr downloaded and copied file: - path: "{{ atl_obr_shared_completed_lock }}" + path: "{{ atl_obr_completed_lock }}" state: directory when: copied is succeeded register: completed_lock_created - name: Remove moving_lock to show that obr is completed file: - path: "{{ atl_obr_shared_moving_lock }}" + path: "{{ atl_obr_moving_lock }}" state: absent when: - completed_lock_created is succeeded @@ -133,7 +133,7 @@ - name: Delete old temp installer file: - path: "{{ atl_obr_temp_download }}" + path: "{{ atl_obr_download }}" state: absent when: moving_lock_removed is succeeded register: temp_deleted From 6304b10cd453c4a4d380d05c60cda6ff3effe8fc Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Fri, 8 Nov 2019 12:29:21 +1100 Subject: [PATCH 075/109] DCD-727: Update for new manifest format. --- roles/restore_backups/tasks/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index 4d6865f..c1587bd 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -56,8 +56,8 @@ # FIXME: The manifest format is still undecided so the # following usages will need to be updated once it settles.. atl_backup_id: "{{ atl_backup_manifest.name }}" - atl_backup_db_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.artifacts.db.location.value | basename }}" - atl_backup_home_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.artifacts.sharedHome.location.value | basename }}" + atl_backup_db_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.artifacts.db.location.location | basename }}" + atl_backup_home_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.artifacts.sharedHome.location.location | basename }}" # FIXME: Here we fetch the backups. However we may wish to stream # these directly from S3 to the target DB/FS to avoid requiring @@ -66,16 +66,16 @@ aws_s3: mode: get overwrite: different - bucket: "{{ atl_backup_manifest.artifacts.db.location.value | urlsplit('hostname') }}" - object: "{{ atl_backup_manifest.artifacts.db.location.value | urlsplit('path') }}" + bucket: "{{ atl_backup_manifest.artifacts.db.location.location | urlsplit('hostname') }}" + object: "{{ atl_backup_manifest.artifacts.db.location.location | urlsplit('path') }}" dest: "{{ atl_backup_db_dest }}" - name: Fetch Home backup from S3 aws_s3: mode: get overwrite: different - bucket: "{{ atl_backup_manifest.artifacts.sharedHome.location.value | urlsplit('hostname') }}" - object: "{{ atl_backup_manifest.artifacts.sharedHome.location.value | urlsplit('path') }}" + bucket: "{{ atl_backup_manifest.artifacts.sharedHome.location.location | urlsplit('hostname') }}" + object: "{{ atl_backup_manifest.artifacts.sharedHome.location.location | urlsplit('path') }}" dest: "{{ atl_backup_home_dest }}" - name: Install distro-specific restore support packages From ced8608b02341e389be88a4108d335548e5979de Mon Sep 17 00:00:00 2001 From: bmeehan Date: Fri, 8 Nov 2019 02:36:20 +0000 Subject: [PATCH 076/109] ITOPSENG-101 corrected obr shared_home location var --- roles/product_install/tasks/jira-servicedesk_as_obr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index 6e7d0f1..448bb3d 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -140,7 +140,7 @@ - name: Set install to home_shared location set_fact: - atl_obr_download: "{{ atl_obr_home_shared_download }}" + atl_obr_download: "{{ atl_obr_shared_download }}" when: temp_deleted is succeeded when: move_obr From 48904ca0845f8ebe607f841c20b0ca2ce5022c8a Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 8 Nov 2019 14:45:29 +1100 Subject: [PATCH 077/109] ITOPSENG-101 noop the extra_tasks for jira-all --- roles/product_install/tasks/jira-all_extra_tasks.yml | 1 + 1 file changed, 1 insertion(+) create mode 120000 roles/product_install/tasks/jira-all_extra_tasks.yml diff --git a/roles/product_install/tasks/jira-all_extra_tasks.yml b/roles/product_install/tasks/jira-all_extra_tasks.yml new file mode 120000 index 0000000..55832eb --- /dev/null +++ b/roles/product_install/tasks/jira-all_extra_tasks.yml @@ -0,0 +1 @@ +no_op.yml \ No newline at end of file From 9d7626a14a3695a84e0f66ef45cdbf6c17c69408 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 8 Nov 2019 15:13:34 +1100 Subject: [PATCH 078/109] ITOPSENG-101 initial molecule tests commit --- roles/product_install/defaults/main.yml | 2 +- .../molecule/jira_all/Dockerfile.j2 | 14 ++++++ .../molecule/jira_all/molecule.yml | 32 ++++++++++++ .../molecule/jira_all/playbook.yml | 28 +++++++++++ .../molecule/jira_all/tests/test_default.py | 49 +++++++++++++++++++ .../jira_version_latest/tests/test_default.py | 4 +- 6 files changed, 126 insertions(+), 3 deletions(-) create mode 100644 roles/product_install/molecule/jira_all/Dockerfile.j2 create mode 100644 roles/product_install/molecule/jira_all/molecule.yml create mode 100644 roles/product_install/molecule/jira_all/playbook.yml create mode 100644 roles/product_install/molecule/jira_all/tests/test_default.py diff --git a/roles/product_install/defaults/main.yml b/roles/product_install/defaults/main.yml index 6a9940e..7875e1c 100644 --- a/roles/product_install/defaults/main.yml +++ b/roles/product_install/defaults/main.yml @@ -15,7 +15,7 @@ atl_download_format_suffix_map: atl_download_suffix: "{{ atl_download_format_suffix_map[atl_download_format] }}" atl_release_base_url: "https://product-downloads.atlassian.com/software" -atl_product_base_url: "{{ atl_release_base_url }}/{{ atl_product_family }}/downloads" +atl_product_base_url: "{{ atl_release_base_url }}/{{ atl_product_family }}/downloads/binary" atl_product_download_url: "{{ atl_product_base_url }}/atlassian-{{ atl_download_edition | default(atl_product_edition) }}-{{ atl_product_version }}{{ atl_download_suffix }}" atl_product_download_filename: "{{ atl_download_edition | default(atl_product_edition) }}.{{ atl_product_version }}{{ atl_download_suffix }}" diff --git a/roles/product_install/molecule/jira_all/Dockerfile.j2 b/roles/product_install/molecule/jira_all/Dockerfile.j2 new file mode 100644 index 0000000..e6aa95d --- /dev/null +++ b/roles/product_install/molecule/jira_all/Dockerfile.j2 @@ -0,0 +1,14 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi diff --git a/roles/product_install/molecule/jira_all/molecule.yml b/roles/product_install/molecule/jira_all/molecule.yml new file mode 100644 index 0000000..46049f8 --- /dev/null +++ b/roles/product_install/molecule/jira_all/molecule.yml @@ -0,0 +1,32 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: amazon_linux2 + image: amazonlinux:2 + groups: + - aws_node_local + - name: ubuntu_lts + image: ubuntu:bionic + groups: + - aws_node_local +provisioner: + name: ansible + options: + skip-tags: runtime_pkg + lint: + name: ansible-lint + inventory: + links: + group_vars: ../../../../group_vars/ +verifier: + name: testinfra + additional_files_or_dirs: + - ../../resources/tests/test_*.py + lint: + name: flake8 + enabled: false diff --git a/roles/product_install/molecule/jira_all/playbook.yml b/roles/product_install/molecule/jira_all/playbook.yml new file mode 100644 index 0000000..4da7090 --- /dev/null +++ b/roles/product_install/molecule/jira_all/playbook.yml @@ -0,0 +1,28 @@ +--- +- name: Converge + hosts: all + vars: + atl_product_family: "jira" + atl_product_edition: "jira-{{ lookup('env', 'ATL_PRODUCT_EDITION') | lower }}" + atl_product_user: "jira" + atl_product_version: "7.13.1" + atl_install_jsd_as_obr: true + atl_systemd_service_name: "jira.service" + atl_jdbc_encoding: 'UNICODE' + atl_jdbc_collation: 'C' + atl_jdbc_ctype: 'C' + atl_jdbc_template: 'template0' + pre_tasks: + - name: Create cache dir + file: + path: '/media/atl/jira/shared/' + state: directory + - name: Seed version + copy: + dest: '/media/atl/jira/shared/jira-core.version' + content: "7.13.1" + force: false # For idempotency check + roles: + - role: linux_common + - role: product_common + - role: product_install diff --git a/roles/product_install/molecule/jira_all/tests/test_default.py b/roles/product_install/molecule/jira_all/tests/test_default.py new file mode 100644 index 0000000..dd315e4 --- /dev/null +++ b/roles/product_install/molecule/jira_all/tests/test_default.py @@ -0,0 +1,49 @@ +import os +from six.moves import urllib + +import testinfra.utils.ansible_runner +import json + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +def test_version_is_correct(host): + verfile = host.file('/media/atl/jira/shared/jira-core.version') + assert verfile.exists + + assert verfile.content.decode("UTF-8").strip() == "7.13.1" + +def test_is_downloaded(host): + installer = host.file('/media/atl/downloads/jira-core.7.13.1-x64.bin') + assert installer.exists + assert installer.user == 'root' + +def test_completed_lockfile(host): + lockfile = host.file('/media/atl/downloads/jira-core.7.13.1-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' + +def test_is_unpacked(host): + installer = host.file('/opt/atlassian/jira-core/7.13.1/atlassian-jira/') + assert installer.exists + assert installer.is_directory + assert installer.user == 'jira' + assert installer.mode == 0o0755 + +def test_obr_is_downloaded(host): + + installer = host.file('/media/atl/downloads/jira-servicedesk-application-3.16.1.obr') + assert installer.exists + assert installer.user == 'root' + +def test_obr_completed_lockfile(host): + lockfile = host.file('/media/atl/downloads/jira-servicedesk-application-3.16.1.obr_completed') + assert lockfile.exists + assert lockfile.user == 'root' + +def test_obr_is_unpacked(host): + jsdjar = host.file('/media/atl/jira/shared/plugins/installed-plugins/jira-servicedesk-application-3.16.1.jar') + assert jsdjar.exists + assert jsdjar.user == 'jira' + assert jsdjar.mode == 0o0750 \ No newline at end of file diff --git a/roles/product_install/molecule/jira_version_latest/tests/test_default.py b/roles/product_install/molecule/jira_version_latest/tests/test_default.py index 930ab59..8e20d76 100644 --- a/roles/product_install/molecule/jira_version_latest/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_latest/tests/test_default.py @@ -23,7 +23,7 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'-x64.bin') + installer = host.file('/media/atl/downloads/jira-core.'+upstream+'-x64.bin') assert installer.exists assert installer.user == 'root' @@ -32,6 +32,6 @@ def test_completed_lockfile(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/jira-core.'+upstream+'-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' \ No newline at end of file From 3b7623faa17e2f0713800316d07b1728c3f15b23 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 8 Nov 2019 15:20:34 +1100 Subject: [PATCH 079/109] ITOPSENG-101 corrected edition --- roles/product_install/molecule/jira_all/playbook.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/product_install/molecule/jira_all/playbook.yml b/roles/product_install/molecule/jira_all/playbook.yml index 4da7090..9bddc9a 100644 --- a/roles/product_install/molecule/jira_all/playbook.yml +++ b/roles/product_install/molecule/jira_all/playbook.yml @@ -3,7 +3,7 @@ hosts: all vars: atl_product_family: "jira" - atl_product_edition: "jira-{{ lookup('env', 'ATL_PRODUCT_EDITION') | lower }}" + atl_product_edition: "jira-software" atl_product_user: "jira" atl_product_version: "7.13.1" atl_install_jsd_as_obr: true From 51907590ced44c4098b7a613e0eef1cd8ec96a87 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 8 Nov 2019 15:34:45 +1100 Subject: [PATCH 080/109] ITOPSENG-101 reverting the binary suffix, as its only for wac content not product-downloads --- roles/product_install/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/product_install/defaults/main.yml b/roles/product_install/defaults/main.yml index 7875e1c..6a9940e 100644 --- a/roles/product_install/defaults/main.yml +++ b/roles/product_install/defaults/main.yml @@ -15,7 +15,7 @@ atl_download_format_suffix_map: atl_download_suffix: "{{ atl_download_format_suffix_map[atl_download_format] }}" atl_release_base_url: "https://product-downloads.atlassian.com/software" -atl_product_base_url: "{{ atl_release_base_url }}/{{ atl_product_family }}/downloads/binary" +atl_product_base_url: "{{ atl_release_base_url }}/{{ atl_product_family }}/downloads" atl_product_download_url: "{{ atl_product_base_url }}/atlassian-{{ atl_download_edition | default(atl_product_edition) }}-{{ atl_product_version }}{{ atl_download_suffix }}" atl_product_download_filename: "{{ atl_download_edition | default(atl_product_edition) }}.{{ atl_product_version }}{{ atl_download_suffix }}" From c25d8da1d587772a38ab388f085ba0d85619b369 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 8 Nov 2019 15:56:46 +1100 Subject: [PATCH 081/109] updated pipelines.yml for new test --- bitbucket-pipelines.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 408b078..c9f6057 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -14,7 +14,7 @@ pipelines: - step: name: Pre Parallelization stage script: - - echo "Running tests in 30 batches" + - echo "Running tests in 31 batches" - step: name: Check if number of batches match actual number of scenarios script: @@ -267,4 +267,12 @@ pipelines: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 30 + - step: + name: Molecule Test Batch - 31 + services: + - docker + script: + - apt-get update && ./bin/install-ansible --dev + - ./bin/run-tests-in-batches --batch 31 + From 1cf02e721fabfa5154b2dbff9582003236ff3941 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 8 Nov 2019 15:59:48 +1100 Subject: [PATCH 082/109] ITOPSENG-101 moved core tests to software --- .../molecule/jira_all/tests/test_default.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/product_install/molecule/jira_all/tests/test_default.py b/roles/product_install/molecule/jira_all/tests/test_default.py index dd315e4..d03878b 100644 --- a/roles/product_install/molecule/jira_all/tests/test_default.py +++ b/roles/product_install/molecule/jira_all/tests/test_default.py @@ -9,23 +9,23 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( def test_version_is_correct(host): - verfile = host.file('/media/atl/jira/shared/jira-core.version') + verfile = host.file('/media/atl/jira/shared/jira-software.version') assert verfile.exists assert verfile.content.decode("UTF-8").strip() == "7.13.1" def test_is_downloaded(host): - installer = host.file('/media/atl/downloads/jira-core.7.13.1-x64.bin') + installer = host.file('/media/atl/downloads/jira-software.7.13.1-x64.bin') assert installer.exists assert installer.user == 'root' def test_completed_lockfile(host): - lockfile = host.file('/media/atl/downloads/jira-core.7.13.1-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/jira-software.7.13.1-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' def test_is_unpacked(host): - installer = host.file('/opt/atlassian/jira-core/7.13.1/atlassian-jira/') + installer = host.file('/opt/atlassian/jira-software/7.13.1/atlassian-jira/') assert installer.exists assert installer.is_directory assert installer.user == 'jira' From 8a8b2a889ddc3f4b86f2848e499e7b004ffb51e1 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Mon, 11 Nov 2019 13:17:15 +1100 Subject: [PATCH 083/109] ITOPSENG-101 simpler work around junk paths in unarchive --- .../molecule/jira_all/tests/test_default.py | 1 - .../tasks/jira-servicedesk_as_obr.yml | 13 ++++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/roles/product_install/molecule/jira_all/tests/test_default.py b/roles/product_install/molecule/jira_all/tests/test_default.py index d03878b..b9522ec 100644 --- a/roles/product_install/molecule/jira_all/tests/test_default.py +++ b/roles/product_install/molecule/jira_all/tests/test_default.py @@ -32,7 +32,6 @@ def test_is_unpacked(host): assert installer.mode == 0o0755 def test_obr_is_downloaded(host): - installer = host.file('/media/atl/downloads/jira-servicedesk-application-3.16.1.obr') assert installer.exists assert installer.user == 'root' diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index 448bb3d..bf974ea 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -156,18 +156,17 @@ owner: "{{ atl_product_user }}" group: "{{ atl_product_user }}" +# Note as ansible unarchive cant handle "-j junk paths" we need to ignore errors to bypass the path verify - name: Unpack the obr into the installed-plugins dir unarchive: remote_src: true src: "{{ atl_obr_download }}" dest: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins" + creates: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.jar" + extra_opts: + - -j + - -n owner: "{{ atl_product_user }}" group: "{{ atl_product_user }}" mode: 0750 - -- name: Copy JSD dependency jars into the installed-plugins dir - copy: - src: "{{ item }}" - dest: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins" - remote_src: true - with_fileglob: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins/dependencies/*.jar" + ignore_errors: yes \ No newline at end of file From 49797ab64557cac17106496194208daa918b0dbf Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Mon, 11 Nov 2019 13:28:51 +1100 Subject: [PATCH 084/109] ITOPSENG-101 newline at end of file --- roles/product_install/tasks/jira-servicedesk_as_obr.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index bf974ea..4406f2c 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -169,4 +169,5 @@ owner: "{{ atl_product_user }}" group: "{{ atl_product_user }}" mode: 0750 - ignore_errors: yes \ No newline at end of file + ignore_errors: yes + \ No newline at end of file From c307d247274449f3d453cbe295ead1816033d64e Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Mon, 11 Nov 2019 13:39:09 +1100 Subject: [PATCH 085/109] ITOPSENG-101 fix trailing whitespace --- roles/product_install/tasks/jira-servicedesk_as_obr.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index 4406f2c..ce940cb 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -170,4 +170,3 @@ group: "{{ atl_product_user }}" mode: 0750 ignore_errors: yes - \ No newline at end of file From 96e006cbb31b798fa846c51c08ddf2d7844953dc Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Wed, 13 Nov 2019 11:55:56 +1100 Subject: [PATCH 086/109] Sets python environment for amazon-linux-extras to system python --- roles/restore_backups/tasks/amazon.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/restore_backups/tasks/amazon.yml b/roles/restore_backups/tasks/amazon.yml index bf32125..2444212 100644 --- a/roles/restore_backups/tasks/amazon.yml +++ b/roles/restore_backups/tasks/amazon.yml @@ -5,3 +5,5 @@ command: amazon-linux-extras install -y "postgresql{{ postgres_version }}" args: creates: /usr/bin/psql + environment: + PYTHON: /bin/python From f3ced1534a717debb25aeb482e43c2988188472d Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Wed, 13 Nov 2019 12:03:48 +1100 Subject: [PATCH 087/109] DCD-796: Adds test for amazon-linux-extras install postgres --- .../molecule/restore/Dockerfile.j2 | 14 +++++++++ .../molecule/restore/molecule.yml | 30 +++++++++++++++++++ .../molecule/restore/playbook.yml | 11 +++++++ .../molecule/restore/tests/test_default.py | 16 ++++++++++ 4 files changed, 71 insertions(+) create mode 100644 roles/restore_backups/molecule/restore/Dockerfile.j2 create mode 100644 roles/restore_backups/molecule/restore/molecule.yml create mode 100644 roles/restore_backups/molecule/restore/playbook.yml create mode 100644 roles/restore_backups/molecule/restore/tests/test_default.py diff --git a/roles/restore_backups/molecule/restore/Dockerfile.j2 b/roles/restore_backups/molecule/restore/Dockerfile.j2 new file mode 100644 index 0000000..e6aa95d --- /dev/null +++ b/roles/restore_backups/molecule/restore/Dockerfile.j2 @@ -0,0 +1,14 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi diff --git a/roles/restore_backups/molecule/restore/molecule.yml b/roles/restore_backups/molecule/restore/molecule.yml new file mode 100644 index 0000000..04c0973 --- /dev/null +++ b/roles/restore_backups/molecule/restore/molecule.yml @@ -0,0 +1,30 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: amazon_linux2 + image: amazonlinux:2 + groups: + - aws_node_local + ulimits: + - nofile:262144:262144 +provisioner: + name: ansible + options: + skip-tags: runtime_pkg + lint: + name: ansible-lint + options: + x: ["701"] + inventory: + links: + group_vars: ../../../../group_vars/ +verifier: + name: testinfra + lint: + name: flake8 + enabled: false diff --git a/roles/restore_backups/molecule/restore/playbook.yml b/roles/restore_backups/molecule/restore/playbook.yml new file mode 100644 index 0000000..b3c60b0 --- /dev/null +++ b/roles/restore_backups/molecule/restore/playbook.yml @@ -0,0 +1,11 @@ +--- +- name: Converge + hosts: all + vars: + atl_backup_manifest_url: 's3://dcd-slingshot-test/dummy_manifest.json' + atl_product_user: 'jira' + atl_backup_home_restore_canary_path: '/tmp/canary.tmp' + + tasks: + - name: Install distro-specific restore support packages + include_tasks: "../../tasks/{{ ansible_distribution|lower }}.yml" diff --git a/roles/restore_backups/molecule/restore/tests/test_default.py b/roles/restore_backups/molecule/restore/tests/test_default.py new file mode 100644 index 0000000..018ae24 --- /dev/null +++ b/roles/restore_backups/molecule/restore/tests/test_default.py @@ -0,0 +1,16 @@ +import os +import pytest + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +@pytest.mark.parametrize('exe', [ + '/usr/bin/pg_dump', + '/usr/bin/pg_restore', + '/usr/bin/psql' +]) +def test_postgresql_amazon_linux_extras_exes(host, exe): + assert host.file(exe).exists From b2b18e2cff13c4042f642a7222fe52b0ad869c80 Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Wed, 13 Nov 2019 13:10:43 +1100 Subject: [PATCH 088/109] DCD-796: Fix incorrect number of pipelines batches --- bitbucket-pipelines.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 408b078..0df8a14 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -14,7 +14,7 @@ pipelines: - step: name: Pre Parallelization stage script: - - echo "Running tests in 30 batches" + - echo "Running tests in 31 batches" - step: name: Check if number of batches match actual number of scenarios script: @@ -267,4 +267,11 @@ pipelines: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 30 - + - step: + name: Molecule Test Batch - 31 + services: + - docker + script: + - apt-get update && ./bin/install-ansible --dev + - ./bin/run-tests-in-batches --batch 31 + From f4badd4667d58654707c7aa55b0627660c8c1dae Mon Sep 17 00:00:00 2001 From: Adam Brokes Date: Wed, 13 Nov 2019 14:52:44 +1100 Subject: [PATCH 089/109] DCD-803: Ignore COMMENT ON EXTENSION errors when restoring to RDS --- roles/restore_backups/tasks/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index c1587bd..6b28bb1 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -97,6 +97,10 @@ # Depends on fetch_backup roles state: restore target: "{{ atl_backup_db_dest }}" + register: result + failed_when: + - result.rc != 0 + - "COMMENT ON EXTENSION" not in result.msg when: db_created.changed and atl_backup_db_dest is defined From a0496644d4c45b9c030e68a2bcb8a17826827f3f Mon Sep 17 00:00:00 2001 From: Adam Brokes Date: Wed, 13 Nov 2019 15:28:47 +1100 Subject: [PATCH 090/109] DCD-803: Quote the statement --- roles/restore_backups/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index 6b28bb1..5b752ff 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -100,7 +100,7 @@ register: result failed_when: - result.rc != 0 - - "COMMENT ON EXTENSION" not in result.msg + - '"COMMENT ON EXTENSION" not in result.msg' when: db_created.changed and atl_backup_db_dest is defined From 1e3208d49e854234261f6f11657c8ca8b0fecfbe Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Wed, 13 Nov 2019 15:42:51 +1100 Subject: [PATCH 091/109] DCD-796: Implement test that pg_dump is the correct version --- roles/restore_backups/molecule/restore/tests/test_default.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/restore_backups/molecule/restore/tests/test_default.py b/roles/restore_backups/molecule/restore/tests/test_default.py index 018ae24..6f88a4f 100644 --- a/roles/restore_backups/molecule/restore/tests/test_default.py +++ b/roles/restore_backups/molecule/restore/tests/test_default.py @@ -14,3 +14,7 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( ]) def test_postgresql_amazon_linux_extras_exes(host, exe): assert host.file(exe).exists + +def test_postgresql_version(host): + pg_dump_version_output = host.check_output('pg_dump --version') + assert '(PostgreSQL) 9.6' in pg_dump_version_output From c1486e14ecc9a13907a56c64f168027c159c04ed Mon Sep 17 00:00:00 2001 From: Adam Brokes Date: Wed, 13 Nov 2019 15:49:00 +1100 Subject: [PATCH 092/109] DCD-803: Add explaining comment --- roles/restore_backups/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index 5b752ff..1f9ebdb 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -98,6 +98,7 @@ state: restore target: "{{ atl_backup_db_dest }}" register: result + # managed DBs in cloud providers are not allowing full root access to the DB engine, we can safely ignore the COMMENT ON EXTENSION error failed_when: - result.rc != 0 - '"COMMENT ON EXTENSION" not in result.msg' From 6f596c47dcfde2701d0038826f39fadf1172ddc5 Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Wed, 13 Nov 2019 16:23:51 +1100 Subject: [PATCH 093/109] DCD-798: name dump as tar file and specify custom format in db_restore to workaround module not supporting custom pg_dump format --- roles/restore_backups/tasks/main.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index 1f9ebdb..57cd47f 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -76,7 +76,9 @@ overwrite: different bucket: "{{ atl_backup_manifest.artifacts.sharedHome.location.location | urlsplit('hostname') }}" object: "{{ atl_backup_manifest.artifacts.sharedHome.location.location | urlsplit('path') }}" - dest: "{{ atl_backup_home_dest }}" + # We save the backup as a .tar file so that the postgresql_db module uses pg_restore instead of psql to do restore + # This can be removed when ansible 2.10 is released + dest: "{{ atl_backup_home_dest }}.tar" - name: Install distro-specific restore support packages include_tasks: "{{ ansible_distribution|lower }}.yml" @@ -96,7 +98,8 @@ template: "{{ atl_jdbc_template }}" # Depends on fetch_backup roles state: restore - target: "{{ atl_backup_db_dest }}" + target: "{{ atl_backup_db_dest }}.tar" + target_opts: "-Fc" register: result # managed DBs in cloud providers are not allowing full root access to the DB engine, we can safely ignore the COMMENT ON EXTENSION error failed_when: From b1643eccee7ce138a0e2340b088702de926476da Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Wed, 13 Nov 2019 17:31:06 +1100 Subject: [PATCH 094/109] DCD-802: become product user when unarchiving shared home. Implement test that shared home is owned by correct user --- roles/restore_backups/molecule/restore/playbook.yml | 5 ++--- .../restore_backups/molecule/restore/tests/test_default.py | 6 ++++++ roles/restore_backups/tasks/main.yml | 3 +-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/roles/restore_backups/molecule/restore/playbook.yml b/roles/restore_backups/molecule/restore/playbook.yml index b3c60b0..e2f8157 100644 --- a/roles/restore_backups/molecule/restore/playbook.yml +++ b/roles/restore_backups/molecule/restore/playbook.yml @@ -6,6 +6,5 @@ atl_product_user: 'jira' atl_backup_home_restore_canary_path: '/tmp/canary.tmp' - tasks: - - name: Install distro-specific restore support packages - include_tasks: "../../tasks/{{ ansible_distribution|lower }}.yml" + roles: + - restore_backups diff --git a/roles/restore_backups/molecule/restore/tests/test_default.py b/roles/restore_backups/molecule/restore/tests/test_default.py index 6f88a4f..d8ec2b2 100644 --- a/roles/restore_backups/molecule/restore/tests/test_default.py +++ b/roles/restore_backups/molecule/restore/tests/test_default.py @@ -18,3 +18,9 @@ def test_postgresql_amazon_linux_extras_exes(host, exe): def test_postgresql_version(host): pg_dump_version_output = host.check_output('pg_dump --version') assert '(PostgreSQL) 9.6' in pg_dump_version_output + +def test_shared_home_owner(host): + for root, dirs, files in os.walk('/media/atl/jira/shared'): + for fileName in files + dirs: + assert host.file(fileName).user == 'jira' + assert host.file(fileName).group == 'jira' \ No newline at end of file diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index 1f9ebdb..825517b 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -124,8 +124,7 @@ unarchive: src: "{{ atl_backup_home_dest }}" dest: "{{ atl_product_home_shared }}" - owner: "{{ atl_product_user }}" - group: "{{ atl_product_user }}" + become: "{{ atl_product_user }}" - name: Create restore-canary if necessary copy: From 39326a207ba9ac2ddc8c8713c378e33c93ea8626 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Thu, 14 Nov 2019 14:19:51 +1100 Subject: [PATCH 095/109] ITOPSENG-101 fixed pipelines batch count --- bitbucket-pipelines.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index a58d52c..b51f0f9 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -14,7 +14,7 @@ pipelines: - step: name: Pre Parallelization stage script: - - echo "Running tests in 31 batches" + - echo "Running tests in 32 batches" - step: name: Check if number of batches match actual number of scenarios script: @@ -275,3 +275,12 @@ pipelines: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 31 + - step: + name: Molecule Test Batch - 32 + services: + - docker + script: + - apt-get update && ./bin/install-ansible --dev + - ./bin/run-tests-in-batches --batch 32 + + From aea07909b03de466e93a96b32d0f6a0ced8733f2 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Thu, 14 Nov 2019 14:34:11 +1100 Subject: [PATCH 096/109] ITOPSENG-101 add hardcoded download path changes for now till testaid is fixed --- .../molecule/bitbucket_latest/tests/test_default.py | 4 ++-- .../molecule/confluence_latest/tests/test_default.py | 4 ++-- .../molecule/crowd_latest/tests/test_default.py | 4 ++-- roles/product_install/molecule/default/tests/test_default.py | 4 ++-- .../molecule/jira_cached_with_downgrade/tests/test_default.py | 4 ++-- .../molecule/jira_cached_with_upgrade/tests/test_default.py | 4 ++-- .../molecule/jira_software_latest/tests/test_default.py | 4 ++-- .../molecule/jira_tarball/tests/test_default.py | 4 ++-- .../molecule/jira_version_from_file/tests/test_default.py | 4 ++-- .../molecule/jira_version_override/tests/test_default.py | 4 ++-- .../molecule/servicedesk3/tests/test_default.py | 4 ++-- .../molecule/servicedesk4/tests/test_default.py | 4 ++-- .../molecule/servicedesk_latest/tests/test_default.py | 4 ++-- 13 files changed, 26 insertions(+), 26 deletions(-) diff --git a/roles/product_install/molecule/bitbucket_latest/tests/test_default.py b/roles/product_install/molecule/bitbucket_latest/tests/test_default.py index bbd851d..0592cbf 100644 --- a/roles/product_install/molecule/bitbucket_latest/tests/test_default.py +++ b/roles/product_install/molecule/bitbucket_latest/tests/test_default.py @@ -37,7 +37,7 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/media/atl/bitbucket/shared/downloads/bitbucket.' + upstream + '-x64.bin') + installer = host.file('/media/atl/downloads/bitbucket.' + upstream + '-x64.bin') assert installer.exists assert installer.user == 'root' @@ -47,6 +47,6 @@ def test_completed_lockfile(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - lockfile = host.file('/media/atl/bitbucket/shared/downloads/bitbucket.' + upstream + '-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/bitbucket.' + upstream + '-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' diff --git a/roles/product_install/molecule/confluence_latest/tests/test_default.py b/roles/product_install/molecule/confluence_latest/tests/test_default.py index 7ec072b..01044a5 100644 --- a/roles/product_install/molecule/confluence_latest/tests/test_default.py +++ b/roles/product_install/molecule/confluence_latest/tests/test_default.py @@ -35,7 +35,7 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/media/atl/confluence/shared-home/downloads/confluence.'+upstream+'-x64.bin') + installer = host.file('/media/atl/downloads/confluence.'+upstream+'-x64.bin') assert installer.exists assert installer.user == 'root' @@ -44,6 +44,6 @@ def test_completed_lockfile(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - lockfile = host.file('/media/atl/confluence/shared-home/downloads/confluence.'+upstream+'-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/confluence.'+upstream+'-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' diff --git a/roles/product_install/molecule/crowd_latest/tests/test_default.py b/roles/product_install/molecule/crowd_latest/tests/test_default.py index b75a0b5..6e5dc7c 100644 --- a/roles/product_install/molecule/crowd_latest/tests/test_default.py +++ b/roles/product_install/molecule/crowd_latest/tests/test_default.py @@ -37,7 +37,7 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/media/atl/crowd/shared/downloads/crowd.' + upstream + '.tar.gz') + installer = host.file('/media/atl/downloads/crowd.' + upstream + '.tar.gz') assert installer.exists assert installer.user == 'root' @@ -47,6 +47,6 @@ def test_completed_lockfile(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - lockfile = host.file('/media/atl/crowd/shared/downloads/crowd.' + upstream + '.tar.gz_completed') + lockfile = host.file('/media/atl/downloads/crowd.' + upstream + '.tar.gz_completed') assert lockfile.exists assert lockfile.user == 'root' diff --git a/roles/product_install/molecule/default/tests/test_default.py b/roles/product_install/molecule/default/tests/test_default.py index 930ab59..8e20d76 100644 --- a/roles/product_install/molecule/default/tests/test_default.py +++ b/roles/product_install/molecule/default/tests/test_default.py @@ -23,7 +23,7 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'-x64.bin') + installer = host.file('/media/atl/downloads/jira-core.'+upstream+'-x64.bin') assert installer.exists assert installer.user == 'root' @@ -32,6 +32,6 @@ def test_completed_lockfile(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/jira-core.'+upstream+'-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' \ No newline at end of file diff --git a/roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py b/roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py index 788c3de..725ace4 100644 --- a/roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py +++ b/roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py @@ -14,12 +14,12 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "7.10.2" def test_is_downloaded(host): - installer = host.file('/media/atl/jira/shared/downloads/jira-core.7.10.2-x64.bin') + installer = host.file('/media/atl/downloads/jira-core.7.10.2-x64.bin') assert installer.exists assert installer.user == 'root' def test_completed_lockfile(host): - lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.7.10.2-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/jira-core.7.10.2-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' diff --git a/roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py b/roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py index 0818e1b..6b215f2 100644 --- a/roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py +++ b/roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py @@ -14,12 +14,12 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "7.10.1" def test_is_downloaded(host): - installer = host.file('/media/atl/jira/shared/downloads/jira-core.7.10.1-x64.bin') + installer = host.file('/media/atl/downloads/jira-core.7.10.1-x64.bin') assert installer.exists assert installer.user == 'root' def test_completed_lockfile(host): - lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.7.10.1-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/jira-core.7.10.1-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' diff --git a/roles/product_install/molecule/jira_software_latest/tests/test_default.py b/roles/product_install/molecule/jira_software_latest/tests/test_default.py index de1dca3..80d53b8 100644 --- a/roles/product_install/molecule/jira_software_latest/tests/test_default.py +++ b/roles/product_install/molecule/jira_software_latest/tests/test_default.py @@ -35,7 +35,7 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/media/atl/jira/shared/downloads/jira-software.'+upstream+'-x64.bin') + installer = host.file('/media/atl/downloads/jira-software.'+upstream+'-x64.bin') assert installer.exists assert installer.user == 'root' @@ -44,6 +44,6 @@ def test_completed_lockfile(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - lockfile = host.file('/media/atl/jira/shared/downloads/jira-software.'+upstream+'-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/jira-software.'+upstream+'-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' \ No newline at end of file diff --git a/roles/product_install/molecule/jira_tarball/tests/test_default.py b/roles/product_install/molecule/jira_tarball/tests/test_default.py index 11a7438..7155f41 100644 --- a/roles/product_install/molecule/jira_tarball/tests/test_default.py +++ b/roles/product_install/molecule/jira_tarball/tests/test_default.py @@ -23,7 +23,7 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'.tar.gz') + installer = host.file('/media/atl/downloads/jira-core.'+upstream+'.tar.gz') assert installer.exists assert installer.user == 'root' @@ -32,6 +32,6 @@ def test_completed_lockfile(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'.tar.gz_completed') + lockfile = host.file('/media/atl/downloads/jira-core.'+upstream+'.tar.gz_completed') assert lockfile.exists assert lockfile.user == 'root' \ No newline at end of file diff --git a/roles/product_install/molecule/jira_version_from_file/tests/test_default.py b/roles/product_install/molecule/jira_version_from_file/tests/test_default.py index 8b75761..07098b7 100644 --- a/roles/product_install/molecule/jira_version_from_file/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_from_file/tests/test_default.py @@ -14,12 +14,12 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "7.13.1" def test_is_downloaded(host): - installer = host.file('/media/atl/jira/shared/downloads/jira-core.7.13.1-x64.bin') + installer = host.file('/media/atl/downloads/jira-core.7.13.1-x64.bin') assert installer.exists assert installer.user == 'root' def test_completed_lockfile(host): - lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.7.13.1-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/jira-core.7.13.1-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' diff --git a/roles/product_install/molecule/jira_version_override/tests/test_default.py b/roles/product_install/molecule/jira_version_override/tests/test_default.py index 3f16801..e4fa67d 100644 --- a/roles/product_install/molecule/jira_version_override/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_override/tests/test_default.py @@ -14,12 +14,12 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "7.13.2" def test_is_downloaded(host): - installer = host.file('/media/atl/jira/shared/downloads/jira-core.7.13.2-x64.bin') + installer = host.file('/media/atl/downloads/jira-core.7.13.2-x64.bin') assert installer.exists assert installer.user == 'root' def test_completed_lockfile(host): - lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.7.13.2-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/jira-core.7.13.2-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' diff --git a/roles/product_install/molecule/servicedesk3/tests/test_default.py b/roles/product_install/molecule/servicedesk3/tests/test_default.py index 5f50b6e..ae78b3d 100644 --- a/roles/product_install/molecule/servicedesk3/tests/test_default.py +++ b/roles/product_install/molecule/servicedesk3/tests/test_default.py @@ -14,12 +14,12 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "3.9.0" def test_is_downloaded(host): - installer = host.file('/media/atl/jira/shared/downloads/servicedesk.3.9.0-x64.bin') + installer = host.file('/media/atl/downloads/servicedesk.3.9.0-x64.bin') assert installer.exists assert installer.user == 'root' def test_completed_lockfile(host): - lockfile = host.file('/media/atl/jira/shared/downloads/servicedesk.3.9.0-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/servicedesk.3.9.0-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' diff --git a/roles/product_install/molecule/servicedesk4/tests/test_default.py b/roles/product_install/molecule/servicedesk4/tests/test_default.py index 5a22e8c..bc4814e 100644 --- a/roles/product_install/molecule/servicedesk4/tests/test_default.py +++ b/roles/product_install/molecule/servicedesk4/tests/test_default.py @@ -14,12 +14,12 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "4.1.0" def test_is_downloaded(host): - installer = host.file('/media/atl/jira/shared/downloads/servicedesk.4.1.0-x64.bin') + installer = host.file('/media/atl/downloads/servicedesk.4.1.0-x64.bin') assert installer.exists assert installer.user == 'root' def test_completed_lockfile(host): - lockfile = host.file('/media/atl/jira/shared/downloads/servicedesk.4.1.0-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/servicedesk.4.1.0-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' diff --git a/roles/product_install/molecule/servicedesk_latest/tests/test_default.py b/roles/product_install/molecule/servicedesk_latest/tests/test_default.py index cd975f3..bd4eea8 100644 --- a/roles/product_install/molecule/servicedesk_latest/tests/test_default.py +++ b/roles/product_install/molecule/servicedesk_latest/tests/test_default.py @@ -23,12 +23,12 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == sd def test_is_downloaded(host): - installer = host.file('/media/atl/jira/shared/downloads/servicedesk.'+sd+'-x64.bin') + installer = host.file('/media/atl/downloads/servicedesk.'+sd+'-x64.bin') assert installer.exists assert installer.user == 'root' def test_completed_lockfile(host): - lockfile = host.file('/media/atl/jira/shared/downloads/servicedesk.'+sd+'-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/servicedesk.'+sd+'-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' From a0080b5b079c511f4fb2ac34c2900e2a5395ed87 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 15 Nov 2019 12:15:32 +1100 Subject: [PATCH 097/109] ITOPSENG-101 fixed version in version_from_file and added better obr unpacking --- .../jira_version_from_file/tests/test_default.py | 4 ++-- .../tasks/jira-servicedesk_as_obr.yml | 12 +++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/roles/product_install/molecule/jira_version_from_file/tests/test_default.py b/roles/product_install/molecule/jira_version_from_file/tests/test_default.py index 07098b7..1b725b8 100644 --- a/roles/product_install/molecule/jira_version_from_file/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_from_file/tests/test_default.py @@ -14,12 +14,12 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "7.13.1" def test_is_downloaded(host): - installer = host.file('/media/atl/downloads/jira-core.7.13.1-x64.bin') + installer = host.file('/media/atl/downloads/jira-core.7.9.0-x64.bin') assert installer.exists assert installer.user == 'root' def test_completed_lockfile(host): - lockfile = host.file('/media/atl/downloads/jira-core.7.13.1-x64.bin_completed') + lockfile = host.file('/media/atl/downloads/jira-core.7.9.0-x64.bin_completed') assert lockfile.exists assert lockfile.user == 'root' diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index ce940cb..e236f44 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -163,10 +163,16 @@ src: "{{ atl_obr_download }}" dest: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins" creates: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.jar" - extra_opts: - - -j - - -n + list_files: no + exclude: + - M* owner: "{{ atl_product_user }}" group: "{{ atl_product_user }}" mode: 0750 ignore_errors: yes + +- name: Move JSD dependency jars into the installed-plugins dir + shell: + cmd: "mv {{ atl_product_version_cache_dir }}/plugins/installed-plugins/dependencies/*.jar {{ atl_product_version_cache_dir }}/plugins/installed-plugins" + creates: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins/servicedesk-core-ui-plugin-{{ atl_jsd_build_info.json.name }}-REL-0022.jar" + From 587b647cbaa5662786db09805ef9913143511753 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 15 Nov 2019 12:21:51 +1100 Subject: [PATCH 098/109] ITOPSENG-101 remove extra blank line at eof --- roles/product_install/tasks/jira-servicedesk_as_obr.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index e236f44..bce18c1 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -175,4 +175,3 @@ shell: cmd: "mv {{ atl_product_version_cache_dir }}/plugins/installed-plugins/dependencies/*.jar {{ atl_product_version_cache_dir }}/plugins/installed-plugins" creates: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins/servicedesk-core-ui-plugin-{{ atl_jsd_build_info.json.name }}-REL-0022.jar" - From 2f8790324a45caca62e03f79f88002f4e0300d22 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Fri, 15 Nov 2019 16:01:02 +1100 Subject: [PATCH 099/109] ITOPSENG-101 so over this --- roles/product_install/molecule/jira_all/molecule.yml | 2 -- .../molecule/jira_version_from_file/tests/test_default.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/roles/product_install/molecule/jira_all/molecule.yml b/roles/product_install/molecule/jira_all/molecule.yml index 46049f8..7fd3163 100644 --- a/roles/product_install/molecule/jira_all/molecule.yml +++ b/roles/product_install/molecule/jira_all/molecule.yml @@ -25,8 +25,6 @@ provisioner: group_vars: ../../../../group_vars/ verifier: name: testinfra - additional_files_or_dirs: - - ../../resources/tests/test_*.py lint: name: flake8 enabled: false diff --git a/roles/product_install/molecule/jira_version_from_file/tests/test_default.py b/roles/product_install/molecule/jira_version_from_file/tests/test_default.py index 1b725b8..589e7ce 100644 --- a/roles/product_install/molecule/jira_version_from_file/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_from_file/tests/test_default.py @@ -11,7 +11,7 @@ def test_version_is_correct(host): verfile = host.file('/media/atl/jira/shared/jira-core.version') assert verfile.exists - assert verfile.content.decode("UTF-8").strip() == "7.13.1" + assert verfile.content.decode("UTF-8").strip() == "7.9.0" def test_is_downloaded(host): installer = host.file('/media/atl/downloads/jira-core.7.9.0-x64.bin') From e648c9722131b1b4238296c524c5dd0b2bc39382 Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Mon, 18 Nov 2019 10:39:28 +1100 Subject: [PATCH 100/109] DCD-798: Fix saving shared home archive as tar instead of db dump when fetchin from S3 --- roles/restore_backups/tasks/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index 57cd47f..f743e6b 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -68,7 +68,9 @@ overwrite: different bucket: "{{ atl_backup_manifest.artifacts.db.location.location | urlsplit('hostname') }}" object: "{{ atl_backup_manifest.artifacts.db.location.location | urlsplit('path') }}" - dest: "{{ atl_backup_db_dest }}" + # We save the backup as a .tar file so that the postgresql_db module uses pg_restore instead of psql to do restore + # This can be removed when ansible 2.10 is released + dest: "{{ atl_backup_db_dest }}.tar" - name: Fetch Home backup from S3 aws_s3: @@ -76,9 +78,7 @@ overwrite: different bucket: "{{ atl_backup_manifest.artifacts.sharedHome.location.location | urlsplit('hostname') }}" object: "{{ atl_backup_manifest.artifacts.sharedHome.location.location | urlsplit('path') }}" - # We save the backup as a .tar file so that the postgresql_db module uses pg_restore instead of psql to do restore - # This can be removed when ansible 2.10 is released - dest: "{{ atl_backup_home_dest }}.tar" + dest: "{{ atl_backup_home_dest }}" - name: Install distro-specific restore support packages include_tasks: "{{ ansible_distribution|lower }}.yml" From 3fa354ff0e0591b0be929813201e1d75843232de Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Mon, 18 Nov 2019 12:17:43 +1100 Subject: [PATCH 101/109] Add note to README about raising issues. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 864b060..72136e1 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,10 @@ them in the `Custom command-line parameters for Ansible` field: -e atl_product_download_url=http://s3.amazon.com/atlassian/jira-9.0.0-PRE-TEST.tar.gz -e atl_use_system_jdk=true -e atl_download_format=tarball +## Reporting issues + +If you find any bugs in this repository, or have feature requests or use cases +for us, please raise them in our [public Jira project](https://jira.atlassian.com/projects/SCALE/summary). ## Development From 4827ae84236049d1edb491c5fd090cca1b89e058 Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Mon, 18 Nov 2019 16:37:26 +1100 Subject: [PATCH 102/109] DCD-802: Fix test and implementation for setting shared home owner and group to application user --- .../molecule/restore/playbook.yml | 55 ++++++++++++++++++- .../molecule/restore/tests/test_default.py | 14 +++-- roles/restore_backups/tasks/home_restore.yml | 37 +++++++++++++ roles/restore_backups/tasks/main.yml | 29 +--------- 4 files changed, 101 insertions(+), 34 deletions(-) create mode 100644 roles/restore_backups/tasks/home_restore.yml diff --git a/roles/restore_backups/molecule/restore/playbook.yml b/roles/restore_backups/molecule/restore/playbook.yml index e2f8157..d174e48 100644 --- a/roles/restore_backups/molecule/restore/playbook.yml +++ b/roles/restore_backups/molecule/restore/playbook.yml @@ -4,7 +4,58 @@ vars: atl_backup_manifest_url: 's3://dcd-slingshot-test/dummy_manifest.json' atl_product_user: 'jira' + atl_product_user_uid: '2001' atl_backup_home_restore_canary_path: '/tmp/canary.tmp' + atl_product_home_shared: '/media/atl/jira/shared' + atl_backup_id: 'test-backup' + atl_backup_home_dest: "{{ test_archive }}" - roles: - - restore_backups + test_archive_source: '/tmp/hello' + test_archive_file: 'hello.txt' + test_archive: '/tmp/hello.tar.gz' + test_pre_step_prefix: '[PRE-TEST]' + + pre_tasks: + - name: "{{ test_pre_step_prefix }} Install tar" + package: + state: present + name: tar + + - name: "{{ test_pre_step_prefix }} Install useradd and groupadd binaries" + package: + state: present + name: shadow-utils + + - name: "{{ test_pre_step_prefix }} Create application group" + group: + name: "{{ atl_product_user }}" + gid: "{{ atl_product_user_uid }}" + + - name: "{{ test_pre_step_prefix }} Create application user" + user: + name: "{{ atl_product_user }}" + uid: "{{ atl_product_user_uid }}" + group: "{{ atl_product_user }}" + + - block: + - name: "{{ test_pre_step_prefix }} Create a directory for the shared home archive" + file: + path: "{{ test_archive_source }}" + state: directory + - name: "{{ test_pre_step_prefix }} Create a file in the shared home" + lineinfile: + create: yes + line: 'Hello, world!' + path: "{{ test_archive_source }}/{{ test_archive_file }}" + - name: "{{ test_pre_step_prefix }} Archive the shared home" + archive: + path: "{{ test_archive_source }}" + dest: "{{ test_archive }}" + owner: "{{ atl_product_user }}" + + tasks: + - name: Install distro-specific restore support packages + include_tasks: "../../tasks/{{ ansible_distribution|lower }}.yml" + + - name: Restore shared home + include_tasks: "../../tasks/home_restore.yml" diff --git a/roles/restore_backups/molecule/restore/tests/test_default.py b/roles/restore_backups/molecule/restore/tests/test_default.py index d8ec2b2..f8ab676 100644 --- a/roles/restore_backups/molecule/restore/tests/test_default.py +++ b/roles/restore_backups/molecule/restore/tests/test_default.py @@ -19,8 +19,12 @@ def test_postgresql_version(host): pg_dump_version_output = host.check_output('pg_dump --version') assert '(PostgreSQL) 9.6' in pg_dump_version_output -def test_shared_home_owner(host): - for root, dirs, files in os.walk('/media/atl/jira/shared'): - for fileName in files + dirs: - assert host.file(fileName).user == 'jira' - assert host.file(fileName).group == 'jira' \ No newline at end of file +@pytest.mark.parametrize('file', [ + '/media/atl/jira/shared', + '/media/atl/jira/shared/hello', + '/media/atl/jira/shared/hello/hello.txt' +]) +def test_shared_home_owner(host, file): + assert host.file(file).exists + assert host.file(file).user == 'jira' + assert host.file(file).group == 'jira' \ No newline at end of file diff --git a/roles/restore_backups/tasks/home_restore.yml b/roles/restore_backups/tasks/home_restore.yml new file mode 100644 index 0000000..2a9fa1f --- /dev/null +++ b/roles/restore_backups/tasks/home_restore.yml @@ -0,0 +1,37 @@ +--- +- name: Check for the restore canary file + stat: + path: "{{ atl_backup_home_restore_canary_path }}" + register: restore_canary + +- block: + - name: Create shared home if necessary + file: + path: "{{ atl_product_home_shared }}" + state: directory + mode: 0750 + owner: "{{ atl_product_user }}" + group: "{{ atl_product_user }}" + + - name: Restore the shared-home backup + unarchive: + src: "{{ atl_backup_home_dest }}" + remote_src: yes + dest: "{{ atl_product_home_shared }}" + mode: 0640 + + - name: Set shared home file owner to application user + file: + path: "{{ atl_product_home_shared }}" + recurse: yes + group: "{{ atl_product_user }}" + state: directory + mode: 0640 + owner: "{{ atl_product_user }}" + + - name: Create restore-canary if necessary + copy: + dest: "{{ atl_backup_home_restore_canary_path }}" + content: "{{ atl_backup_id }}" + + when: not restore_canary.stat.exists diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index 3071044..c5b26ed 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -108,33 +108,8 @@ when: db_created.changed and atl_backup_db_dest is defined - - name: Check for the restore canary file - stat: - path: "{{ atl_backup_home_restore_canary_path }}" - register: restore_canary - - - block: - - - name: Create shared home if necessary - file: - path: "{{ atl_product_home_shared }}" - state: directory - mode: 0750 - owner: "{{ atl_product_user }}" - group: "{{ atl_product_user }}" - - - name: Restore the shared-home backup - unarchive: - src: "{{ atl_backup_home_dest }}" - dest: "{{ atl_product_home_shared }}" - become: "{{ atl_product_user }}" - - - name: Create restore-canary if necessary - copy: - dest: "{{ atl_backup_home_restore_canary_path }}" - content: "{{ atl_backup_id }}" - - when: not restore_canary.stat.exists + - name: Restore shared home + include_tasks: "{{ home_restore.yml }}" when: atl_restore_required From 688be1f5d79f2ff75b51d3aadbf71055f7b482fe Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Mon, 18 Nov 2019 16:59:56 +1100 Subject: [PATCH 103/109] DCD-802: Add test and logic for setting version file owner to root in shared home --- roles/restore_backups/molecule/restore/playbook.yml | 12 +++++++++++- .../molecule/restore/tests/test_default.py | 7 ++++++- roles/restore_backups/tasks/home_restore.yml | 13 +++++++++++-- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/roles/restore_backups/molecule/restore/playbook.yml b/roles/restore_backups/molecule/restore/playbook.yml index d174e48..3e4272c 100644 --- a/roles/restore_backups/molecule/restore/playbook.yml +++ b/roles/restore_backups/molecule/restore/playbook.yml @@ -4,6 +4,8 @@ vars: atl_backup_manifest_url: 's3://dcd-slingshot-test/dummy_manifest.json' atl_product_user: 'jira' + atl_product_edition: 'jira-software' + atl_product_version_cache: "{{ atl_product_home_shared }}/{{ atl_product_edition }}.version" atl_product_user_uid: '2001' atl_backup_home_restore_canary_path: '/tmp/canary.tmp' atl_product_home_shared: '/media/atl/jira/shared' @@ -12,6 +14,7 @@ test_archive_source: '/tmp/hello' test_archive_file: 'hello.txt' + test_product_version_file: "/tmp/{{ atl_product_edition }}.version" test_archive: '/tmp/hello.tar.gz' test_pre_step_prefix: '[PRE-TEST]' @@ -47,9 +50,16 @@ create: yes line: 'Hello, world!' path: "{{ test_archive_source }}/{{ test_archive_file }}" + - name: "{{ test_pre_step_prefix }} Create the version file in the shared home" + lineinfile: + create: yes + line: '8.5' + path: "{{ test_product_version_file }}" - name: "{{ test_pre_step_prefix }} Archive the shared home" archive: - path: "{{ test_archive_source }}" + path: + - "{{ test_archive_source }}" + - "{{ test_product_version_file }}" dest: "{{ test_archive }}" owner: "{{ atl_product_user }}" diff --git a/roles/restore_backups/molecule/restore/tests/test_default.py b/roles/restore_backups/molecule/restore/tests/test_default.py index f8ab676..98af566 100644 --- a/roles/restore_backups/molecule/restore/tests/test_default.py +++ b/roles/restore_backups/molecule/restore/tests/test_default.py @@ -27,4 +27,9 @@ def test_postgresql_version(host): def test_shared_home_owner(host, file): assert host.file(file).exists assert host.file(file).user == 'jira' - assert host.file(file).group == 'jira' \ No newline at end of file + assert host.file(file).group == 'jira' + +def test_version_file_owned_by_root(host): + assert host.file('/media/atl/jira/shared/jira-software.version').exists + assert host.file('/media/atl/jira/shared/jira-software.version').user == 'root' + assert host.file('/media/atl/jira/shared/jira-software.version').group == 'root' \ No newline at end of file diff --git a/roles/restore_backups/tasks/home_restore.yml b/roles/restore_backups/tasks/home_restore.yml index 2a9fa1f..c3cb081 100644 --- a/roles/restore_backups/tasks/home_restore.yml +++ b/roles/restore_backups/tasks/home_restore.yml @@ -20,14 +20,23 @@ dest: "{{ atl_product_home_shared }}" mode: 0640 - - name: Set shared home file owner to application user + - name: Set shared home owner and group to application user file: path: "{{ atl_product_home_shared }}" recurse: yes group: "{{ atl_product_user }}" + owner: "{{ atl_product_user }}" state: directory mode: 0640 - owner: "{{ atl_product_user }}" + + - name: Set version file owner and group to root + file: + path: "{{ atl_product_version_cache }}" + group: root + owner: root + state: file + # Ignore the error in case there is no product version file in the backup + ignore_errors: yes - name: Create restore-canary if necessary copy: From cbe62d02fd4c49a2bc795b5b886fd99f73bf0523 Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Mon, 18 Nov 2019 17:02:08 +1100 Subject: [PATCH 104/109] DCD-802: Organise vars in restore molecule playbook --- .../molecule/restore/playbook.yml | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/roles/restore_backups/molecule/restore/playbook.yml b/roles/restore_backups/molecule/restore/playbook.yml index 3e4272c..a2b36ea 100644 --- a/roles/restore_backups/molecule/restore/playbook.yml +++ b/roles/restore_backups/molecule/restore/playbook.yml @@ -2,21 +2,22 @@ - name: Converge hosts: all vars: - atl_backup_manifest_url: 's3://dcd-slingshot-test/dummy_manifest.json' - atl_product_user: 'jira' - atl_product_edition: 'jira-software' - atl_product_version_cache: "{{ atl_product_home_shared }}/{{ atl_product_edition }}.version" - atl_product_user_uid: '2001' - atl_backup_home_restore_canary_path: '/tmp/canary.tmp' - atl_product_home_shared: '/media/atl/jira/shared' - atl_backup_id: 'test-backup' atl_backup_home_dest: "{{ test_archive }}" + atl_backup_home_restore_canary_path: '/tmp/canary.tmp' + atl_backup_id: 'test-backup' + atl_backup_manifest_url: 'fake_manifest' + + atl_product_edition: 'jira-software' + atl_product_home_shared: '/media/atl/jira/shared' + atl_product_user: 'jira' + atl_product_user_uid: '2001' + atl_product_version_cache: "{{ atl_product_home_shared }}/{{ atl_product_edition }}.version" - test_archive_source: '/tmp/hello' - test_archive_file: 'hello.txt' - test_product_version_file: "/tmp/{{ atl_product_edition }}.version" test_archive: '/tmp/hello.tar.gz' + test_archive_file: 'hello.txt' + test_archive_source: '/tmp/hello' test_pre_step_prefix: '[PRE-TEST]' + test_product_version_file: "/tmp/{{ atl_product_edition }}.version" pre_tasks: - name: "{{ test_pre_step_prefix }} Install tar" From b12d26c282e77c2bc51bb4ac389a5fc2974909ac Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Tue, 19 Nov 2019 16:01:23 +1100 Subject: [PATCH 105/109] DCD-802: Fix reference to home_restore task in include task in restore_backups main.yml --- roles/restore_backups/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index c5b26ed..1c81430 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -109,7 +109,7 @@ - name: Restore shared home - include_tasks: "{{ home_restore.yml }}" + include_tasks: "home_restore.yml" when: atl_restore_required From 56d4be5c04b52bcf7168df3e20a6cb440e441969 Mon Sep 17 00:00:00 2001 From: Brett Meehan Date: Wed, 20 Nov 2019 10:08:07 +1100 Subject: [PATCH 106/109] ITOPSENG-101 addresses PR comments --- .../tasks/jira-servicedesk_as_obr.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index bce18c1..e602765 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -110,9 +110,6 @@ src: "{{ atl_obr_download }}" dest: "{{ atl_obr_shared_download }}" remote_src: true - when: - - moving_lock_created is succeeded - - moving_lock_created.changed register: copied - name: Create completed_lock once obr downloaded and copied @@ -150,7 +147,7 @@ - name: Ensure instaled-plugins dir exists file: - path: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins" + path: "{{ atl_product_home_shared }}/plugins/installed-plugins" state: directory mode: 0750 owner: "{{ atl_product_user }}" @@ -161,8 +158,8 @@ unarchive: remote_src: true src: "{{ atl_obr_download }}" - dest: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins" - creates: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.jar" + dest: "{{ atl_product_home_shared }}/plugins/installed-plugins" + creates: "{{ atl_product_home_shared }}/plugins/installed-plugins/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.jar" list_files: no exclude: - M* @@ -173,5 +170,5 @@ - name: Move JSD dependency jars into the installed-plugins dir shell: - cmd: "mv {{ atl_product_version_cache_dir }}/plugins/installed-plugins/dependencies/*.jar {{ atl_product_version_cache_dir }}/plugins/installed-plugins" - creates: "{{ atl_product_version_cache_dir }}/plugins/installed-plugins/servicedesk-core-ui-plugin-{{ atl_jsd_build_info.json.name }}-REL-0022.jar" + cmd: "mv {{ atl_product_home_shared }}/plugins/installed-plugins/dependencies/*.jar {{ atl_product_home_shared }}/plugins/installed-plugins" + creates: "{{ atl_product_home_shared }}/plugins/installed-plugins/servicedesk-core-ui-plugin-{{ atl_jsd_build_info.json.name }}-*.jar" From 096549edadb1006effb3989782249b3a87ae1f16 Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Wed, 20 Nov 2019 13:55:59 +1100 Subject: [PATCH 107/109] Preserve file permissions when restoring the shared home backup --- roles/restore_backups/tasks/home_restore.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/restore_backups/tasks/home_restore.yml b/roles/restore_backups/tasks/home_restore.yml index c3cb081..bdeb866 100644 --- a/roles/restore_backups/tasks/home_restore.yml +++ b/roles/restore_backups/tasks/home_restore.yml @@ -18,7 +18,7 @@ src: "{{ atl_backup_home_dest }}" remote_src: yes dest: "{{ atl_product_home_shared }}" - mode: 0640 + mode: preserve - name: Set shared home owner and group to application user file: @@ -27,7 +27,6 @@ group: "{{ atl_product_user }}" owner: "{{ atl_product_user }}" state: directory - mode: 0640 - name: Set version file owner and group to root file: From cb6647ba8e70108d5bf4400c617a13497b5aac7a Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Wed, 20 Nov 2019 15:53:16 +1100 Subject: [PATCH 108/109] DCD-802: Fix file mode preservation in restore and add test for file mode --- roles/restore_backups/molecule/restore/playbook.yml | 3 +++ roles/restore_backups/molecule/restore/tests/test_default.py | 5 +++++ roles/restore_backups/tasks/home_restore.yml | 1 - 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/roles/restore_backups/molecule/restore/playbook.yml b/roles/restore_backups/molecule/restore/playbook.yml index a2b36ea..073d24b 100644 --- a/roles/restore_backups/molecule/restore/playbook.yml +++ b/roles/restore_backups/molecule/restore/playbook.yml @@ -46,16 +46,19 @@ file: path: "{{ test_archive_source }}" state: directory + mode: 0755 - name: "{{ test_pre_step_prefix }} Create a file in the shared home" lineinfile: create: yes line: 'Hello, world!' path: "{{ test_archive_source }}/{{ test_archive_file }}" + mode: 0640 - name: "{{ test_pre_step_prefix }} Create the version file in the shared home" lineinfile: create: yes line: '8.5' path: "{{ test_product_version_file }}" + mode: 0640 - name: "{{ test_pre_step_prefix }} Archive the shared home" archive: path: diff --git a/roles/restore_backups/molecule/restore/tests/test_default.py b/roles/restore_backups/molecule/restore/tests/test_default.py index 98af566..c9882bc 100644 --- a/roles/restore_backups/molecule/restore/tests/test_default.py +++ b/roles/restore_backups/molecule/restore/tests/test_default.py @@ -1,5 +1,6 @@ import os import pytest +from stat import * import testinfra.utils.ansible_runner @@ -29,6 +30,10 @@ def test_shared_home_owner(host, file): assert host.file(file).user == 'jira' assert host.file(file).group == 'jira' +def test_file_modes(host): + assert host.file('/media/atl/jira/shared/hello').mode == 0o755 + assert host.file('/media/atl/jira/shared/hello/hello.txt').mode == 0o640 + def test_version_file_owned_by_root(host): assert host.file('/media/atl/jira/shared/jira-software.version').exists assert host.file('/media/atl/jira/shared/jira-software.version').user == 'root' diff --git a/roles/restore_backups/tasks/home_restore.yml b/roles/restore_backups/tasks/home_restore.yml index bdeb866..4c5f2ad 100644 --- a/roles/restore_backups/tasks/home_restore.yml +++ b/roles/restore_backups/tasks/home_restore.yml @@ -18,7 +18,6 @@ src: "{{ atl_backup_home_dest }}" remote_src: yes dest: "{{ atl_product_home_shared }}" - mode: preserve - name: Set shared home owner and group to application user file: From 0cbd11c7020144ec863533c8e84117c0c8407e1c Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Wed, 20 Nov 2019 15:54:08 +1100 Subject: [PATCH 109/109] DCD-802: Fix unused import in restore test --- roles/restore_backups/molecule/restore/tests/test_default.py | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/restore_backups/molecule/restore/tests/test_default.py b/roles/restore_backups/molecule/restore/tests/test_default.py index c9882bc..bcfa53d 100644 --- a/roles/restore_backups/molecule/restore/tests/test_default.py +++ b/roles/restore_backups/molecule/restore/tests/test_default.py @@ -1,6 +1,5 @@ import os import pytest -from stat import * import testinfra.utils.ansible_runner