From c724b0432af4cc37bff83f07cfee7f3ab712ea3a Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 8 Oct 2019 17:19:44 +1100 Subject: [PATCH 01/18] DCD-686: Initial implementation of DB restore from dump. --- roles/database_init/tasks/main.yml | 85 ++++++++++++++++++------------ 1 file changed, 51 insertions(+), 34 deletions(-) diff --git a/roles/database_init/tasks/main.yml b/roles/database_init/tasks/main.yml index 99638f7..003f7c8 100644 --- a/roles/database_init/tasks/main.yml +++ b/roles/database_init/tasks/main.yml @@ -1,40 +1,57 @@ --- -- name: Create application DB user - postgresql_user: - login_host: "{{ atl_db_host }}" - login_user: "{{ atl_db_root_user }}" - login_password: "{{ atl_db_root_password }}" - port: "{{ atl_db_port }}" - name: "{{ atl_jdbc_user }}" - password: "{{ atl_jdbc_password }}" - expires: 'infinity' - tags: - - new_only +- block: -- name: Update root privs for new user - postgresql_privs: - login_host: "{{ atl_db_host }}" - login_user: "{{ atl_db_root_user }}" - login_password: "{{ atl_db_root_password }}" - database: postgres - roles: "{{ atl_db_root_user }}" - objs: "{{ atl_jdbc_user }}" - type: group - tags: - - new_only + - name: Create application DB user + postgresql_user: + login_host: "{{ atl_db_host }}" + login_user: "{{ atl_db_root_user }}" + login_password: "{{ atl_db_root_password }}" + port: "{{ atl_db_port }}" + name: "{{ atl_jdbc_user }}" + password: "{{ atl_jdbc_password }}" + expires: 'infinity' + + - name: Update root privs for new user + postgresql_privs: + login_host: "{{ atl_db_host }}" + login_user: "{{ atl_db_root_user }}" + login_password: "{{ atl_db_root_password }}" + database: postgres + roles: "{{ atl_db_root_user }}" + objs: "{{ atl_jdbc_user }}" + type: group + + - name: Create new application database + postgresql_db: + login_host: "{{ atl_db_host }}" + login_user: "{{ atl_db_root_user }}" + login_password: "{{ atl_db_root_password }}" + port: "{{ atl_db_port }}" + name: "{{ atl_jdbc_db_name }}" + owner: "{{ atl_jdbc_user }}" + encoding: "{{ atl_jdbc_encoding }}" + lc_collate: "{{ atl_jdbc_collation }}" + lc_ctype: "{{ atl_jdbc_ctype }}" + template: "{{ atl_jdbc_template }}" + when: atl_backup_db_dest is not defined + + - name: Restore application database + postgresql_db: + login_host: "{{ atl_db_host }}" + login_user: "{{ atl_db_root_user }}" + login_password: "{{ atl_db_root_password }}" + port: "{{ atl_db_port }}" + name: "{{ atl_jdbc_db_name }}" + owner: "{{ atl_jdbc_user }}" + encoding: "{{ atl_jdbc_encoding }}" + lc_collate: "{{ atl_jdbc_collation }}" + lc_ctype: "{{ atl_jdbc_ctype }}" + template: "{{ atl_jdbc_template }}" + # Depends on fetch_backup roles + state: restore + target: "{{ atl_backup_db_dest }}" + when: atl_backup_db_dest is defined -- name: Create application database - postgresql_db: - login_host: "{{ atl_db_host }}" - login_user: "{{ atl_db_root_user }}" - login_password: "{{ atl_db_root_password }}" - port: "{{ atl_db_port }}" - name: "{{ atl_jdbc_db_name }}" - owner: "{{ atl_jdbc_user }}" - encoding: "{{ atl_jdbc_encoding }}" - lc_collate: "{{ atl_jdbc_collation }}" - lc_ctype: "{{ atl_jdbc_ctype }}" - template: "{{ atl_jdbc_template }}" tags: - new_only From 32d3640bbb90412df5eebda7aa00ac8045682709 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 9 Oct 2019 12:43:41 +1100 Subject: [PATCH 02/18] DCD-686: Add fetching of backups to Jira for testing. --- aws_jira_dc_node.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/aws_jira_dc_node.yml b/aws_jira_dc_node.yml index 41b7be6..6ef0765 100644 --- a/aws_jira_dc_node.yml +++ b/aws_jira_dc_node.yml @@ -21,6 +21,7 @@ - role: linux_common - role: aws_common - role: aws_shared_fs_config + - role: fetch_backups - role: product_common - role: product_install - role: database_init From cb691d25562630d088bd193edb6efee743ab1c76 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 9 Oct 2019 13:14:04 +1100 Subject: [PATCH 03/18] DCD-686: We still need some python2 packages. --- bin/install-ansible | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/install-ansible b/bin/install-ansible index b515cdf..196562e 100755 --- a/bin/install-ansible +++ b/bin/install-ansible @@ -6,8 +6,12 @@ source /etc/os-release if [[ $ID = "amzn" ]]; then yum install -y \ python3-devel \ - python3-pip + python3-pip \ + python2-boto3 \ + python2-botocore + else + # FIXME: Currently assumes Debian-based apt-get update && \ apt-get install -y \ python3-dev \ From 5996a176d5f54f9b221208582848b2331fd3729d Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 9 Oct 2019 13:25:32 +1100 Subject: [PATCH 04/18] DCD-686: We need postgres installed for client utils. --- bin/install-ansible | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/install-ansible b/bin/install-ansible index 196562e..98189a5 100755 --- a/bin/install-ansible +++ b/bin/install-ansible @@ -4,7 +4,9 @@ set -e source /etc/os-release if [[ $ID = "amzn" ]]; then + amazon-linux-extras enable postgresql9.6 yum install -y \ + postgresql \ python3-devel \ python3-pip \ python2-boto3 \ From adf9c270932fd6df095c6f3d762356cf901fe262 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 9 Oct 2019 13:44:02 +1100 Subject: [PATCH 05/18] DCD-686: Ignore any roles specified in the DB dump during restore. --- roles/database_init/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/database_init/tasks/main.yml b/roles/database_init/tasks/main.yml index 003f7c8..ec5ce38 100644 --- a/roles/database_init/tasks/main.yml +++ b/roles/database_init/tasks/main.yml @@ -51,6 +51,7 @@ # Depends on fetch_backup roles state: restore target: "{{ atl_backup_db_dest }}" + target_opts: "--no-owner --role={{ atl_jdbc_user }}" when: atl_backup_db_dest is defined tags: From 9d8b47ba9ea7f8c5f2aa82352b53138c528efda0 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 9 Oct 2019 13:48:50 +1100 Subject: [PATCH 06/18] DCD-686: Only download backups if changed. --- roles/fetch_backups/tasks/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/fetch_backups/tasks/main.yml b/roles/fetch_backups/tasks/main.yml index 45d7e1f..16b18b8 100644 --- a/roles/fetch_backups/tasks/main.yml +++ b/roles/fetch_backups/tasks/main.yml @@ -28,6 +28,7 @@ - name: Fetch the manifest from S3 aws_s3: mode: get + overwrite: different bucket: "{{ atl_backup_manifest_bucket }}" object: "{{ atl_backup_manifest_path }}" dest: "{{ atl_backup_manifest_dest }}" @@ -55,6 +56,7 @@ - name: Fetch DB backup from S3 aws_s3: mode: get + overwrite: different bucket: "{{ atl_backup_manifest.db_dump | urlsplit('hostname') }}" object: "{{ atl_backup_manifest.db_dump | urlsplit('path') }}" dest: "{{ atl_backup_db_dest }}" @@ -62,6 +64,7 @@ - name: Fetch Home backup from S3 aws_s3: mode: get + overwrite: different bucket: "{{ atl_backup_manifest.shared_home_dump | urlsplit('hostname') }}" object: "{{ atl_backup_manifest.shared_home_dump | urlsplit('path') }}" dest: "{{ atl_backup_home_dest }}" From 0400d8943d557f9f366dedde768d2e4afc47c275 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 9 Oct 2019 15:29:58 +1100 Subject: [PATCH 07/18] DCD-686: Only restore DB when it is a new one. --- roles/database_init/tasks/main.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/roles/database_init/tasks/main.yml b/roles/database_init/tasks/main.yml index ec5ce38..aa8da99 100644 --- a/roles/database_init/tasks/main.yml +++ b/roles/database_init/tasks/main.yml @@ -34,7 +34,7 @@ lc_collate: "{{ atl_jdbc_collation }}" lc_ctype: "{{ atl_jdbc_ctype }}" template: "{{ atl_jdbc_template }}" - when: atl_backup_db_dest is not defined + register: db_create - name: Restore application database postgresql_db: @@ -51,8 +51,7 @@ # Depends on fetch_backup roles state: restore target: "{{ atl_backup_db_dest }}" - target_opts: "--no-owner --role={{ atl_jdbc_user }}" - when: atl_backup_db_dest is defined + when: db_create.changed and atl_backup_db_dest is defined tags: - new_only From 72659de9ad8de6238eaaefa2d65c1cb1cc7ba564 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 10 Oct 2019 09:30:45 +1100 Subject: [PATCH 08/18] DCD-686: Install Postgres only if restoration is required. --- bin/install-ansible | 2 -- roles/fetch_backups/tasks/amazon.yml | 7 +++++++ roles/fetch_backups/tasks/main.yml | 3 +++ 3 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 roles/fetch_backups/tasks/amazon.yml diff --git a/bin/install-ansible b/bin/install-ansible index 98189a5..196562e 100755 --- a/bin/install-ansible +++ b/bin/install-ansible @@ -4,9 +4,7 @@ set -e source /etc/os-release if [[ $ID = "amzn" ]]; then - amazon-linux-extras enable postgresql9.6 yum install -y \ - postgresql \ python3-devel \ python3-pip \ python2-boto3 \ diff --git a/roles/fetch_backups/tasks/amazon.yml b/roles/fetch_backups/tasks/amazon.yml new file mode 100644 index 0000000..bf32125 --- /dev/null +++ b/roles/fetch_backups/tasks/amazon.yml @@ -0,0 +1,7 @@ +--- + +# Amazon Linux 2 supplies extra packages via a special command. +- name: Enable Postgresql from 'extras' + command: amazon-linux-extras install -y "postgresql{{ postgres_version }}" + args: + creates: /usr/bin/psql diff --git a/roles/fetch_backups/tasks/main.yml b/roles/fetch_backups/tasks/main.yml index 16b18b8..27c5efe 100644 --- a/roles/fetch_backups/tasks/main.yml +++ b/roles/fetch_backups/tasks/main.yml @@ -69,4 +69,7 @@ object: "{{ atl_backup_manifest.shared_home_dump | urlsplit('path') }}" dest: "{{ atl_backup_home_dest }}" + - name: Install distro-specific restore support packages + include_tasks: "{{ ansible_distribution|lower }}.yml" + when: atl_backup_manifest_url is defined and atl_backup_manifest_url != '' From dddf3a86ec4320dd432778cf76a47f1ccca137db Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 10 Oct 2019 09:39:50 +1100 Subject: [PATCH 09/18] DCD-686: Add comment about manifest format. --- roles/fetch_backups/tasks/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/fetch_backups/tasks/main.yml b/roles/fetch_backups/tasks/main.yml index 27c5efe..a5a87de 100644 --- a/roles/fetch_backups/tasks/main.yml +++ b/roles/fetch_backups/tasks/main.yml @@ -40,6 +40,8 @@ dest: "{{ atl_backup_manifest_dest }}" when: atl_backup_manifest_url.scheme != 's3' + # FIXME: The manifest format is still undecided; everything + # referencing this variable should be considered a placeholder. - name: Load parameters from manifest include_vars: file: "{{ atl_backup_manifest_dest }}" From 6f56925fa12c8646e733574a32e37e1eaaafd9a3 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 10 Oct 2019 11:05:20 +1100 Subject: [PATCH 10/18] DCD-686: Move restore operations into the fetch role for the time being. --- roles/database_init/tasks/main.yml | 19 +-------- roles/fetch_backups/defaults/main.yml | 4 ++ roles/fetch_backups/tasks/main.yml | 56 ++++++++++++++++++++++++++- 3 files changed, 59 insertions(+), 20 deletions(-) create mode 100644 roles/fetch_backups/defaults/main.yml diff --git a/roles/database_init/tasks/main.yml b/roles/database_init/tasks/main.yml index aa8da99..8827f99 100644 --- a/roles/database_init/tasks/main.yml +++ b/roles/database_init/tasks/main.yml @@ -34,24 +34,7 @@ lc_collate: "{{ atl_jdbc_collation }}" lc_ctype: "{{ atl_jdbc_ctype }}" template: "{{ atl_jdbc_template }}" - register: db_create - - - name: Restore application database - postgresql_db: - login_host: "{{ atl_db_host }}" - login_user: "{{ atl_db_root_user }}" - login_password: "{{ atl_db_root_password }}" - port: "{{ atl_db_port }}" - name: "{{ atl_jdbc_db_name }}" - owner: "{{ atl_jdbc_user }}" - encoding: "{{ atl_jdbc_encoding }}" - lc_collate: "{{ atl_jdbc_collation }}" - lc_ctype: "{{ atl_jdbc_ctype }}" - template: "{{ atl_jdbc_template }}" - # Depends on fetch_backup roles - state: restore - target: "{{ atl_backup_db_dest }}" - when: db_create.changed and atl_backup_db_dest is defined + register: db_created tags: - new_only diff --git a/roles/fetch_backups/defaults/main.yml b/roles/fetch_backups/defaults/main.yml new file mode 100644 index 0000000..6561c7e --- /dev/null +++ b/roles/fetch_backups/defaults/main.yml @@ -0,0 +1,4 @@ +--- + +atl_backup_home_restore_canary_filename: ".slingshot_home_restore" +atl_backup_home_restore_canary_path: "{{ atl_product_home_shared }}/{{ atl_backup_home_restore_canary_filename }}" diff --git a/roles/fetch_backups/tasks/main.yml b/roles/fetch_backups/tasks/main.yml index a5a87de..5dcdf01 100644 --- a/roles/fetch_backups/tasks/main.yml +++ b/roles/fetch_backups/tasks/main.yml @@ -3,6 +3,15 @@ # This role will attempt to fetch and load the backup manifest from a # remote HTTP or S3 URL. On successful completion the contents of JSON # or YAML document will be in the var `atl_backup_manifest`. +# +# PREREQUISITES: +# * `atl_backup_manifest_url` points at the manifest. +# * The shared home filesystem is mounted if necessary (e.g. NFS/EFS). +# * The database has been created and the variable `db_created` is +# registered with the result (i.e: `register: db_created`). +# +# NOTE: The actual DB/FS restore operations could potentially be split +# out into discrete roles, but currently that is not required. - block: @@ -40,8 +49,6 @@ dest: "{{ atl_backup_manifest_dest }}" when: atl_backup_manifest_url.scheme != 's3' - # FIXME: The manifest format is still undecided; everything - # referencing this variable should be considered a placeholder. - name: Load parameters from manifest include_vars: file: "{{ atl_backup_manifest_dest }}" @@ -49,6 +56,9 @@ - name: Define the DB and home dump destinations set_fact: + # FIXME: The manifest format is still undecided so the + # following usages will need to be updated once it settles.. + atl_backup_id: "{{ atl_backup_manifest.name }}" atl_backup_db_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.db_dump | basename }}" atl_backup_home_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.shared_home_dump | basename }}" @@ -74,4 +84,46 @@ - name: Install distro-specific restore support packages include_tasks: "{{ ansible_distribution|lower }}.yml" + + - name: Restore application database + postgresql_db: + login_host: "{{ atl_db_host }}" + login_user: "{{ atl_db_root_user }}" + login_password: "{{ atl_db_root_password }}" + port: "{{ atl_db_port }}" + name: "{{ atl_jdbc_db_name }}" + owner: "{{ atl_jdbc_user }}" + encoding: "{{ atl_jdbc_encoding }}" + lc_collate: "{{ atl_jdbc_collation }}" + lc_ctype: "{{ atl_jdbc_ctype }}" + template: "{{ atl_jdbc_template }}" + # Depends on fetch_backup roles + state: restore + target: "{{ atl_backup_db_dest }}" + when: db_created.changed and atl_backup_db_dest is defined + + + - name: Check for the restore canary file + stat: + path: "{{ atl_backup_home_restore_canary_path }}" + register: restore_canary + + - name: Create shared home if necessary + file: + path: "{{ atl_product_home_shared }}" + state: directory + mode: 0750 + owner: "{{ atl_product_user }}" + group: "{{ atl_product_user }}" + when: restore_canary.stat.exists + + - name: Restore the shared-home backup + unarchive: + path: "{{ atl_backup_home_restore_canary_path }}" + dest: "{{ atl_product_home_shared }}" + owner: "{{ atl_product_user }}" + group: "{{ atl_product_user }}" + when: restore_canary.stat.exists + + when: atl_backup_manifest_url is defined and atl_backup_manifest_url != '' From 93c359d0295b84854c8a4124d5578fde829cd916 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 10 Oct 2019 11:12:18 +1100 Subject: [PATCH 11/18] DCD-686: Rename role to reflect updated functionality and exand restore functionality. --- aws_jira_dc_node.yml | 2 +- .../defaults/main.yml | 0 .../tasks/amazon.yml | 0 .../tasks/main.yml | 37 +++++++++++-------- 4 files changed, 23 insertions(+), 16 deletions(-) rename roles/{fetch_backups => restore_backups}/defaults/main.yml (100%) rename roles/{fetch_backups => restore_backups}/tasks/amazon.yml (100%) rename roles/{fetch_backups => restore_backups}/tasks/main.yml (84%) diff --git a/aws_jira_dc_node.yml b/aws_jira_dc_node.yml index 6ef0765..b0470fb 100644 --- a/aws_jira_dc_node.yml +++ b/aws_jira_dc_node.yml @@ -21,9 +21,9 @@ - role: linux_common - role: aws_common - role: aws_shared_fs_config - - role: fetch_backups - role: product_common - role: product_install - role: database_init + - role: restore_backups - role: jira_config - role: product_startup diff --git a/roles/fetch_backups/defaults/main.yml b/roles/restore_backups/defaults/main.yml similarity index 100% rename from roles/fetch_backups/defaults/main.yml rename to roles/restore_backups/defaults/main.yml diff --git a/roles/fetch_backups/tasks/amazon.yml b/roles/restore_backups/tasks/amazon.yml similarity index 100% rename from roles/fetch_backups/tasks/amazon.yml rename to roles/restore_backups/tasks/amazon.yml diff --git a/roles/fetch_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml similarity index 84% rename from roles/fetch_backups/tasks/main.yml rename to roles/restore_backups/tasks/main.yml index 5dcdf01..a0023cf 100644 --- a/roles/fetch_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -108,22 +108,29 @@ path: "{{ atl_backup_home_restore_canary_path }}" register: restore_canary - - name: Create shared home if necessary - file: - path: "{{ atl_product_home_shared }}" - state: directory - mode: 0750 - owner: "{{ atl_product_user }}" - group: "{{ atl_product_user }}" - when: restore_canary.stat.exists + - block: - - name: Restore the shared-home backup - unarchive: - path: "{{ atl_backup_home_restore_canary_path }}" - dest: "{{ atl_product_home_shared }}" - owner: "{{ atl_product_user }}" - group: "{{ atl_product_user }}" - when: restore_canary.stat.exists + - name: Create shared home if necessary + file: + path: "{{ atl_product_home_shared }}" + state: directory + mode: 0750 + owner: "{{ atl_product_user }}" + group: "{{ atl_product_user }}" + + - name: Restore the shared-home backup + unarchive: + path: "{{ atl_backup_home_restore_canary_path }}" + dest: "{{ atl_product_home_shared }}" + owner: "{{ atl_product_user }}" + group: "{{ atl_product_user }}" + + - name: Create restore-canary if necessary + copy: + dest: "{{ atl_backup_home_restore_canary_path }}" + content: "{{ atl_backup_id }}" + + when: not restore_canary.stat.exists when: atl_backup_manifest_url is defined and atl_backup_manifest_url != '' From 8a343f7e8c66e4b4fd5b3b519495b76108c716d2 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 10 Oct 2019 11:26:47 +1100 Subject: [PATCH 12/18] DCD-686: Fix unarchive operation. --- roles/restore_backups/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index a0023cf..019c5d5 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -120,7 +120,7 @@ - name: Restore the shared-home backup unarchive: - path: "{{ atl_backup_home_restore_canary_path }}" + src: "{{ atl_backup_home_restore_canary_path }}" dest: "{{ atl_product_home_shared }}" owner: "{{ atl_product_user }}" group: "{{ atl_product_user }}" From 8254482761e5b73b1b420fd91c59495cf639a95d Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 10 Oct 2019 11:28:56 +1100 Subject: [PATCH 13/18] DCD-686: Fix tarball location. --- roles/restore_backups/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index 019c5d5..f53b685 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -120,7 +120,7 @@ - name: Restore the shared-home backup unarchive: - src: "{{ atl_backup_home_restore_canary_path }}" + src: "{{ atl_backup_home_dest }}" dest: "{{ atl_product_home_shared }}" owner: "{{ atl_product_user }}" group: "{{ atl_product_user }}" From 46a2bad4b8ce01ead1c4ba77f1532836fc6288e8 Mon Sep 17 00:00:00 2001 From: Varun Arbatti <1063972+theghostwhoforks@users.noreply.github.com> Date: Thu, 10 Oct 2019 14:51:06 +1100 Subject: [PATCH 14/18] DCD-742: Updates manifest structure in ansible --- roles/restore_backups/tasks/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index f53b685..d12d073 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -59,8 +59,8 @@ # FIXME: The manifest format is still undecided so the # following usages will need to be updated once it settles.. atl_backup_id: "{{ atl_backup_manifest.name }}" - atl_backup_db_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.db_dump | basename }}" - atl_backup_home_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.shared_home_dump | basename }}" + atl_backup_db_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.artifacts.db.location.value | basename }}" + atl_backup_home_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.artifacts.shared_home.location.value | basename }}" # FIXME: Here we fetch the backups. However we may wish to stream # these directly from S3 to the target DB/FS to avoid requiring @@ -69,16 +69,16 @@ aws_s3: mode: get overwrite: different - bucket: "{{ atl_backup_manifest.db_dump | urlsplit('hostname') }}" - object: "{{ atl_backup_manifest.db_dump | urlsplit('path') }}" + bucket: "{{ atl_backup_manifest.artifacts.db.location.value | urlsplit('hostname') }}" + object: "{{ atl_backup_manifest.artifacts.db.location.value | urlsplit('path') }}" dest: "{{ atl_backup_db_dest }}" - name: Fetch Home backup from S3 aws_s3: mode: get overwrite: different - bucket: "{{ atl_backup_manifest.shared_home_dump | urlsplit('hostname') }}" - object: "{{ atl_backup_manifest.shared_home_dump | urlsplit('path') }}" + bucket: "{{ atl_backup_manifest.artifacts.shared_home.location.value | urlsplit('hostname') }}" + object: "{{ atl_backup_manifest.artifacts.shared_home.location.value | urlsplit('path') }}" dest: "{{ atl_backup_home_dest }}" - name: Install distro-specific restore support packages From 7e4d1301e10ad2c133341b0275913d85c7bf9f67 Mon Sep 17 00:00:00 2001 From: Varun Arbatti <1063972+theghostwhoforks@users.noreply.github.com> Date: Mon, 21 Oct 2019 11:08:28 +1100 Subject: [PATCH 15/18] DCD-686: Changes case of shared home from snakeCase to lower camel case --- roles/restore_backups/tasks/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index d12d073..18f2169 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -60,7 +60,7 @@ # following usages will need to be updated once it settles.. atl_backup_id: "{{ atl_backup_manifest.name }}" atl_backup_db_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.artifacts.db.location.value | basename }}" - atl_backup_home_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.artifacts.shared_home.location.value | basename }}" + atl_backup_home_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.artifacts.sharedHome.location.value | basename }}" # FIXME: Here we fetch the backups. However we may wish to stream # these directly from S3 to the target DB/FS to avoid requiring @@ -77,8 +77,8 @@ aws_s3: mode: get overwrite: different - bucket: "{{ atl_backup_manifest.artifacts.shared_home.location.value | urlsplit('hostname') }}" - object: "{{ atl_backup_manifest.artifacts.shared_home.location.value | urlsplit('path') }}" + bucket: "{{ atl_backup_manifest.artifacts.sharedHome.location.value | urlsplit('hostname') }}" + object: "{{ atl_backup_manifest.artifacts.sharedHome.location.value | urlsplit('path') }}" dest: "{{ atl_backup_home_dest }}" - name: Install distro-specific restore support packages From 4c0cec450909f8c6382cb6689c166facb7c70504 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 30 Oct 2019 11:01:18 +1100 Subject: [PATCH 16/18] DCD-686: Check for no-op when no manifest URL specified. --- group_vars/aws_node_local.yml | 1 + roles/restore_backups/.yamllint | 12 +++++++ .../molecule/default/Dockerfile.j2 | 14 ++++++++ .../molecule/default/molecule.yml | 36 +++++++++++++++++++ .../molecule/default/playbook.yml | 10 ++++++ .../molecule/default/tests/test_default.py | 10 ++++++ roles/restore_backups/tasks/main.yml | 2 +- 7 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 roles/restore_backups/.yamllint create mode 100644 roles/restore_backups/molecule/default/Dockerfile.j2 create mode 100644 roles/restore_backups/molecule/default/molecule.yml create mode 100644 roles/restore_backups/molecule/default/playbook.yml create mode 100644 roles/restore_backups/molecule/default/tests/test_default.py diff --git a/group_vars/aws_node_local.yml b/group_vars/aws_node_local.yml index 69285f9..1675ba8 100644 --- a/group_vars/aws_node_local.yml +++ b/group_vars/aws_node_local.yml @@ -128,3 +128,4 @@ atl_rds_subnet_group_name: "{{ lookup('env', 'ATL_RDS_SUBNET_GROUP_NAME') }}" atl_rds_security_group: "{{ lookup('env', 'ATL_RDS_SECURITY_GROUP') }}" atl_backup_manifest_url: "{{ lookup('env', 'ATL_BACKUP_MANIFEST_URL') }}" +atl_restore_required: "{{ atl_backup_manifest_url is defined and atl_backup_manifest_url != '' }}" diff --git a/roles/restore_backups/.yamllint b/roles/restore_backups/.yamllint new file mode 100644 index 0000000..a87f8ff --- /dev/null +++ b/roles/restore_backups/.yamllint @@ -0,0 +1,12 @@ +extends: default + +rules: + braces: + max-spaces-inside: 1 + level: error + brackets: + max-spaces-inside: 1 + level: error + line-length: disable + truthy: disable + trailing-spaces: false diff --git a/roles/restore_backups/molecule/default/Dockerfile.j2 b/roles/restore_backups/molecule/default/Dockerfile.j2 new file mode 100644 index 0000000..e6aa95d --- /dev/null +++ b/roles/restore_backups/molecule/default/Dockerfile.j2 @@ -0,0 +1,14 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi diff --git a/roles/restore_backups/molecule/default/molecule.yml b/roles/restore_backups/molecule/default/molecule.yml new file mode 100644 index 0000000..7f082f6 --- /dev/null +++ b/roles/restore_backups/molecule/default/molecule.yml @@ -0,0 +1,36 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: amazon_linux2 + image: amazonlinux:2 + groups: + - aws_node_local + ulimits: + - nofile:262144:262144 + - name: ubuntu_lts + image: ubuntu:bionic + groups: + - aws_node_local + ulimits: + - nofile:262144:262144 +provisioner: + name: ansible + options: + skip-tags: runtime_pkg + lint: + name: ansible-lint + options: + x: ["701"] + inventory: + links: + group_vars: ../../../../group_vars/ +verifier: + name: testinfra + lint: + name: flake8 + enabled: false diff --git a/roles/restore_backups/molecule/default/playbook.yml b/roles/restore_backups/molecule/default/playbook.yml new file mode 100644 index 0000000..ffd0c12 --- /dev/null +++ b/roles/restore_backups/molecule/default/playbook.yml @@ -0,0 +1,10 @@ +--- +- name: Converge + hosts: all + vars: + atl_backup_manifest_url: '' + atl_backup_home_restore_canary_path: '/tmp/canary.tmp' + + roles: + # Should be no-op + - role: restore_backups diff --git a/roles/restore_backups/molecule/default/tests/test_default.py b/roles/restore_backups/molecule/default/tests/test_default.py new file mode 100644 index 0000000..0a7276f --- /dev/null +++ b/roles/restore_backups/molecule/default/tests/test_default.py @@ -0,0 +1,10 @@ +import os + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +def test_no_canary_file(host): + assert not host.file('atl_backup_home_restore_canary_path').exists diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index 18f2169..8767b4d 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -133,4 +133,4 @@ when: not restore_canary.stat.exists - when: atl_backup_manifest_url is defined and atl_backup_manifest_url != '' + when: atl_restore_required From e440daa1a5de40f9dae6d0fd6e10d4c09e61f676 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 30 Oct 2019 11:03:19 +1100 Subject: [PATCH 17/18] DCD-686: Update pipeline tests. --- bitbucket-pipelines.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index ec993c6..d28b140 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -14,7 +14,7 @@ pipelines: - step: name: Pre Parallelization stage script: - - echo "Running tests in 28 batches" + - echo "Running tests in 29 batches" - step: name: Check if number of batches match actual number of scenarios script: @@ -251,4 +251,12 @@ pipelines: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 28 + - step: + name: Molecule Test Batch - 29 + services: + - docker + script: + - apt-get update && ./bin/install-ansible --dev + - ./bin/run-tests-in-batches --batch 29 + From 1ea30531b6806ae4753cbb1c20efdcecd314a987 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 31 Oct 2019 10:30:17 +1100 Subject: [PATCH 18/18] DCD-686: Remove HTTP manifest download for now. --- roles/restore_backups/tasks/main.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index 8767b4d..4d6865f 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -1,8 +1,8 @@ --- # This role will attempt to fetch and load the backup manifest from a -# remote HTTP or S3 URL. On successful completion the contents of JSON -# or YAML document will be in the var `atl_backup_manifest`. +# remote S3 URL. On successful completion the contents of JSON or YAML +# document will be in the var `atl_backup_manifest`. # # PREREQUISITES: # * `atl_backup_manifest_url` points at the manifest. @@ -12,6 +12,9 @@ # # NOTE: The actual DB/FS restore operations could potentially be split # out into discrete roles, but currently that is not required. +# +# TODO: Support HTTPS with authentication. Deferred until after the +# initial testing release. - block: @@ -43,12 +46,6 @@ dest: "{{ atl_backup_manifest_dest }}" when: atl_backup_manifest_url.scheme == 's3' - - name: Fetch the manifest from remote host - get_url: - url: "{{ atl_backup_manifest_url }}" - dest: "{{ atl_backup_manifest_dest }}" - when: atl_backup_manifest_url.scheme != 's3' - - name: Load parameters from manifest include_vars: file: "{{ atl_backup_manifest_dest }}"