From e59d56cfe07c8c28e48bda8861d108f253814268 Mon Sep 17 00:00:00 2001 From: Yifei Zhang Date: Tue, 5 Jul 2022 15:03:00 +1000 Subject: [PATCH 1/3] CLIP-1561: Pinned pipenv version to resolve 'NoneType' object is not callable error. --- bin/install-ansible | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/install-ansible b/bin/install-ansible index 7b994b8..42eeb01 100755 --- a/bin/install-ansible +++ b/bin/install-ansible @@ -23,7 +23,7 @@ fi export PATH=/usr/local/bin:$PATH export PIP_DEFAULT_TIMEOUT=60 -pip3 install pipenv +pip3 install pipenv==2021.5.29 echo "Installing ansible and dependencies..." PIPENV_NOSPIN=1 PIPENV_HIDE_EMOJIS=1 pipenv sync 2>&1 | iconv -c -f utf-8 -t ascii From 5f8bbaa4a10a08e67918e2352ca52070f7ca6187 Mon Sep 17 00:00:00 2001 From: Yifei Zhang Date: Tue, 5 Jul 2022 15:19:20 +1000 Subject: [PATCH 2/3] CLIP-1561: Removed restore_backsups role to fix failed builds. --- aws_bitbucket_dc_node.yml | 1 - aws_confluence_dc_node.yml | 1 - aws_jira_dc_node.yml | 1 - roles/restore_backups/.yamllint | 12 -- roles/restore_backups/defaults/main.yml | 4 - .../molecule/default/Dockerfile.j2 | 14 --- .../molecule/default/molecule.yml | 36 ------ .../molecule/default/playbook.yml | 10 -- .../molecule/default/tests/test_default.py | 10 -- .../restore_conf_server/Dockerfile.j2 | 14 --- .../molecule/restore_conf_server/molecule.yml | 30 ----- .../molecule/restore_conf_server/playbook.yml | 74 ----------- .../restore_conf_server/tests/test_default.py | 15 --- .../restore_jira_clustered/Dockerfile.j2 | 14 --- .../restore_jira_clustered/molecule.yml | 30 ----- .../restore_jira_clustered/playbook.yml | 76 ----------- .../tests/test_default.py | 39 ------ roles/restore_backups/tasks/amazon.yml | 9 -- roles/restore_backups/tasks/home_restore.yml | 75 ----------- roles/restore_backups/tasks/main.yml | 118 ------------------ 20 files changed, 583 deletions(-) delete mode 100644 roles/restore_backups/.yamllint delete mode 100644 roles/restore_backups/defaults/main.yml delete mode 100644 roles/restore_backups/molecule/default/Dockerfile.j2 delete mode 100644 roles/restore_backups/molecule/default/molecule.yml delete mode 100644 roles/restore_backups/molecule/default/playbook.yml delete mode 100644 roles/restore_backups/molecule/default/tests/test_default.py delete mode 100644 roles/restore_backups/molecule/restore_conf_server/Dockerfile.j2 delete mode 100644 roles/restore_backups/molecule/restore_conf_server/molecule.yml delete mode 100644 roles/restore_backups/molecule/restore_conf_server/playbook.yml delete mode 100644 roles/restore_backups/molecule/restore_conf_server/tests/test_default.py delete mode 100644 roles/restore_backups/molecule/restore_jira_clustered/Dockerfile.j2 delete mode 100644 roles/restore_backups/molecule/restore_jira_clustered/molecule.yml delete mode 100644 roles/restore_backups/molecule/restore_jira_clustered/playbook.yml delete mode 100644 roles/restore_backups/molecule/restore_jira_clustered/tests/test_default.py delete mode 100644 roles/restore_backups/tasks/amazon.yml delete mode 100644 roles/restore_backups/tasks/home_restore.yml delete mode 100644 roles/restore_backups/tasks/main.yml diff --git a/aws_bitbucket_dc_node.yml b/aws_bitbucket_dc_node.yml index 3d8348e..0d93318 100644 --- a/aws_bitbucket_dc_node.yml +++ b/aws_bitbucket_dc_node.yml @@ -32,7 +32,6 @@ - role: product_common - role: product_install - role: database_init - - role: restore_backups - role: bitbucket_config - role: product_startup - role: bitbucket_dataset_restore diff --git a/aws_confluence_dc_node.yml b/aws_confluence_dc_node.yml index 046657c..b3bf5b8 100644 --- a/aws_confluence_dc_node.yml +++ b/aws_confluence_dc_node.yml @@ -20,7 +20,6 @@ tags: [skip_on_stack_update] - role: database_init tags: [skip_on_stack_update] - - role: restore_backups - role: confluence_common - role: confluence_config - role: product_startup diff --git a/aws_jira_dc_node.yml b/aws_jira_dc_node.yml index a994279..572dd30 100644 --- a/aws_jira_dc_node.yml +++ b/aws_jira_dc_node.yml @@ -27,6 +27,5 @@ tags: [skip_on_stack_update] - role: database_init tags: [skip_on_stack_update] - - role: restore_backups - role: jira_config - role: product_startup diff --git a/roles/restore_backups/.yamllint b/roles/restore_backups/.yamllint deleted file mode 100644 index a87f8ff..0000000 --- a/roles/restore_backups/.yamllint +++ /dev/null @@ -1,12 +0,0 @@ -extends: default - -rules: - braces: - max-spaces-inside: 1 - level: error - brackets: - max-spaces-inside: 1 - level: error - line-length: disable - truthy: disable - trailing-spaces: false diff --git a/roles/restore_backups/defaults/main.yml b/roles/restore_backups/defaults/main.yml deleted file mode 100644 index 6561c7e..0000000 --- a/roles/restore_backups/defaults/main.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- - -atl_backup_home_restore_canary_filename: ".slingshot_home_restore" -atl_backup_home_restore_canary_path: "{{ atl_product_home_shared }}/{{ atl_backup_home_restore_canary_filename }}" diff --git a/roles/restore_backups/molecule/default/Dockerfile.j2 b/roles/restore_backups/molecule/default/Dockerfile.j2 deleted file mode 100644 index e6aa95d..0000000 --- a/roles/restore_backups/molecule/default/Dockerfile.j2 +++ /dev/null @@ -1,14 +0,0 @@ -# Molecule managed - -{% if item.registry is defined %} -FROM {{ item.registry.url }}/{{ item.image }} -{% else %} -FROM {{ item.image }} -{% endif %} - -RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ - elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \ - elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ - elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \ - elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ - elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi diff --git a/roles/restore_backups/molecule/default/molecule.yml b/roles/restore_backups/molecule/default/molecule.yml deleted file mode 100644 index 7f082f6..0000000 --- a/roles/restore_backups/molecule/default/molecule.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -dependency: - name: galaxy -driver: - name: docker -lint: - name: yamllint -platforms: - - name: amazon_linux2 - image: amazonlinux:2 - groups: - - aws_node_local - ulimits: - - nofile:262144:262144 - - name: ubuntu_lts - image: ubuntu:bionic - groups: - - aws_node_local - ulimits: - - nofile:262144:262144 -provisioner: - name: ansible - options: - skip-tags: runtime_pkg - lint: - name: ansible-lint - options: - x: ["701"] - inventory: - links: - group_vars: ../../../../group_vars/ -verifier: - name: testinfra - lint: - name: flake8 - enabled: false diff --git a/roles/restore_backups/molecule/default/playbook.yml b/roles/restore_backups/molecule/default/playbook.yml deleted file mode 100644 index ffd0c12..0000000 --- a/roles/restore_backups/molecule/default/playbook.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- name: Converge - hosts: all - vars: - atl_backup_manifest_url: '' - atl_backup_home_restore_canary_path: '/tmp/canary.tmp' - - roles: - # Should be no-op - - role: restore_backups diff --git a/roles/restore_backups/molecule/default/tests/test_default.py b/roles/restore_backups/molecule/default/tests/test_default.py deleted file mode 100644 index 0a7276f..0000000 --- a/roles/restore_backups/molecule/default/tests/test_default.py +++ /dev/null @@ -1,10 +0,0 @@ -import os - -import testinfra.utils.ansible_runner - -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') - - -def test_no_canary_file(host): - assert not host.file('atl_backup_home_restore_canary_path').exists diff --git a/roles/restore_backups/molecule/restore_conf_server/Dockerfile.j2 b/roles/restore_backups/molecule/restore_conf_server/Dockerfile.j2 deleted file mode 100644 index e6aa95d..0000000 --- a/roles/restore_backups/molecule/restore_conf_server/Dockerfile.j2 +++ /dev/null @@ -1,14 +0,0 @@ -# Molecule managed - -{% if item.registry is defined %} -FROM {{ item.registry.url }}/{{ item.image }} -{% else %} -FROM {{ item.image }} -{% endif %} - -RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ - elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \ - elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ - elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \ - elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ - elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi diff --git a/roles/restore_backups/molecule/restore_conf_server/molecule.yml b/roles/restore_backups/molecule/restore_conf_server/molecule.yml deleted file mode 100644 index 04c0973..0000000 --- a/roles/restore_backups/molecule/restore_conf_server/molecule.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- -dependency: - name: galaxy -driver: - name: docker -lint: - name: yamllint -platforms: - - name: amazon_linux2 - image: amazonlinux:2 - groups: - - aws_node_local - ulimits: - - nofile:262144:262144 -provisioner: - name: ansible - options: - skip-tags: runtime_pkg - lint: - name: ansible-lint - options: - x: ["701"] - inventory: - links: - group_vars: ../../../../group_vars/ -verifier: - name: testinfra - lint: - name: flake8 - enabled: false diff --git a/roles/restore_backups/molecule/restore_conf_server/playbook.yml b/roles/restore_backups/molecule/restore_conf_server/playbook.yml deleted file mode 100644 index d526a64..0000000 --- a/roles/restore_backups/molecule/restore_conf_server/playbook.yml +++ /dev/null @@ -1,74 +0,0 @@ ---- -- name: Converge - hosts: all - vars: - atl_backup_home_dest: "{{ test_archive }}" - atl_backup_id: 'test-backup' - atl_backup_manifest_url: 'fake_manifest' - atl_backup_home_is_server: 'true' - - atl_product_home_shared: '/media/atl/confluence/shared-home' - atl_backup_home_restore_canary_path: "{{ atl_product_home_shared }}/canary.tmp" - atl_product_edition: 'confluence' - atl_product_user: 'confluence' - atl_product_user_uid: '2001' - atl_product_version_cache: "{{ atl_product_home_shared }}/{{ atl_product_edition }}.version" - - test_archive: '/tmp/hello.tar.gz' - test_archive_file: 'hello.txt' - test_archive_source: '/tmp/hello' - - test_pre_step_prefix: '[PRE-TEST]' - test_product_version_file: "/tmp/{{ atl_product_edition }}.version" - - pre_tasks: - - name: "{{ test_pre_step_prefix }} Install tar and useradd/groupadd binaries" - package: - state: present - name: - - tar - - shadow-utils - - - name: "{{ test_pre_step_prefix }} Create application group" - group: - name: "{{ atl_product_user }}" - gid: "{{ atl_product_user_uid }}" - - - name: "{{ test_pre_step_prefix }} Create application user" - user: - name: "{{ atl_product_user }}" - uid: "{{ atl_product_user_uid }}" - group: "{{ atl_product_user }}" - - - name: "{{ test_pre_step_prefix }} Create a Conf server home directory structure" - file: - path: "{{ item }}" - state: directory - mode: 0755 - with_items: - - "{{ test_archive_source }}" - - "{{ test_archive_source }}/attachments" - - "{{ test_archive_source }}/shared-home" - - - name: "{{ test_pre_step_prefix }} Create files" - copy: - dest: "{{ item }}" - content: "content" - with_items: - - "{{ test_archive_source }}/unwanted.txt" - - "{{ test_archive_source }}/attachments/image.jpg" - - "{{ test_archive_source }}/shared-home/shared-content.txt" - - - name: "{{ test_pre_step_prefix }} Archive the shared home" - archive: - path: - - "{{ test_archive_source }}/*" - dest: "{{ test_archive }}" - owner: "{{ atl_product_user }}" - - tasks: - - name: Install distro-specific restore support packages - include_tasks: "../../tasks/{{ ansible_distribution|lower }}.yml" - - - name: Restore shared home - include_tasks: "../../tasks/home_restore.yml" diff --git a/roles/restore_backups/molecule/restore_conf_server/tests/test_default.py b/roles/restore_backups/molecule/restore_conf_server/tests/test_default.py deleted file mode 100644 index ced7af8..0000000 --- a/roles/restore_backups/molecule/restore_conf_server/tests/test_default.py +++ /dev/null @@ -1,15 +0,0 @@ -import os -import pytest - -import testinfra.utils.ansible_runner - -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') - -def test_conf_server_converted(host): - assert host.file('/media/atl/confluence/shared-home').is_directory - assert host.file('/media/atl/confluence/shared-home/shared-content.txt').is_file - assert host.file('/media/atl/confluence/shared-home/attachments').is_directory - assert host.file('/media/atl/confluence/shared-home/attachments/image.jpg').is_file - - assert not host.file('/media/atl/confluence/shared-home/unwanted.txt').is_file diff --git a/roles/restore_backups/molecule/restore_jira_clustered/Dockerfile.j2 b/roles/restore_backups/molecule/restore_jira_clustered/Dockerfile.j2 deleted file mode 100644 index e6aa95d..0000000 --- a/roles/restore_backups/molecule/restore_jira_clustered/Dockerfile.j2 +++ /dev/null @@ -1,14 +0,0 @@ -# Molecule managed - -{% if item.registry is defined %} -FROM {{ item.registry.url }}/{{ item.image }} -{% else %} -FROM {{ item.image }} -{% endif %} - -RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ - elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \ - elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ - elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \ - elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ - elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi diff --git a/roles/restore_backups/molecule/restore_jira_clustered/molecule.yml b/roles/restore_backups/molecule/restore_jira_clustered/molecule.yml deleted file mode 100644 index 04c0973..0000000 --- a/roles/restore_backups/molecule/restore_jira_clustered/molecule.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- -dependency: - name: galaxy -driver: - name: docker -lint: - name: yamllint -platforms: - - name: amazon_linux2 - image: amazonlinux:2 - groups: - - aws_node_local - ulimits: - - nofile:262144:262144 -provisioner: - name: ansible - options: - skip-tags: runtime_pkg - lint: - name: ansible-lint - options: - x: ["701"] - inventory: - links: - group_vars: ../../../../group_vars/ -verifier: - name: testinfra - lint: - name: flake8 - enabled: false diff --git a/roles/restore_backups/molecule/restore_jira_clustered/playbook.yml b/roles/restore_backups/molecule/restore_jira_clustered/playbook.yml deleted file mode 100644 index a866f32..0000000 --- a/roles/restore_backups/molecule/restore_jira_clustered/playbook.yml +++ /dev/null @@ -1,76 +0,0 @@ ---- -- name: Converge - hosts: all - vars: - atl_backup_home_dest: "{{ test_archive }}" - atl_backup_home_restore_canary_path: '/tmp/canary.tmp' - atl_backup_id: 'test-backup' - atl_backup_manifest_url: 'fake_manifest' - atl_backup_home_is_server: 'false' - - atl_product_edition: 'jira-software' - atl_product_home_shared: '/media/atl/jira/shared' - atl_product_user: 'jira' - atl_product_user_uid: '2001' - atl_product_version_cache: "{{ atl_product_home_shared }}/{{ atl_product_edition }}.version" - - test_archive: '/tmp/hello.tar.gz' - test_archive_file: 'hello.txt' - test_archive_source: '/tmp/hello' - test_pre_step_prefix: '[PRE-TEST]' - test_product_version_file: "/tmp/{{ atl_product_edition }}.version" - - pre_tasks: - - name: "{{ test_pre_step_prefix }} Install tar" - package: - state: present - name: tar - - - name: "{{ test_pre_step_prefix }} Install useradd and groupadd binaries" - package: - state: present - name: shadow-utils - - - name: "{{ test_pre_step_prefix }} Create application group" - group: - name: "{{ atl_product_user }}" - gid: "{{ atl_product_user_uid }}" - - - name: "{{ test_pre_step_prefix }} Create application user" - user: - name: "{{ atl_product_user }}" - uid: "{{ atl_product_user_uid }}" - group: "{{ atl_product_user }}" - - - block: - - name: "{{ test_pre_step_prefix }} Create a directory for the shared home archive" - file: - path: "{{ test_archive_source }}" - state: directory - mode: 0755 - - name: "{{ test_pre_step_prefix }} Create a file in the shared home" - lineinfile: - create: yes - line: 'Hello, world!' - path: "{{ test_archive_source }}/{{ test_archive_file }}" - mode: 0640 - - name: "{{ test_pre_step_prefix }} Create the version file in the shared home" - lineinfile: - create: yes - line: '8.5' - path: "{{ test_product_version_file }}" - mode: 0640 - - name: "{{ test_pre_step_prefix }} Archive the shared home" - archive: - path: - - "{{ test_archive_source }}" - - "{{ test_product_version_file }}" - dest: "{{ test_archive }}" - owner: "{{ atl_product_user }}" - - tasks: - - name: Install distro-specific restore support packages - include_tasks: "../../tasks/{{ ansible_distribution|lower }}.yml" - - - name: Restore shared home - include_tasks: "../../tasks/home_restore.yml" diff --git a/roles/restore_backups/molecule/restore_jira_clustered/tests/test_default.py b/roles/restore_backups/molecule/restore_jira_clustered/tests/test_default.py deleted file mode 100644 index bcfa53d..0000000 --- a/roles/restore_backups/molecule/restore_jira_clustered/tests/test_default.py +++ /dev/null @@ -1,39 +0,0 @@ -import os -import pytest - -import testinfra.utils.ansible_runner - -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') - - -@pytest.mark.parametrize('exe', [ - '/usr/bin/pg_dump', - '/usr/bin/pg_restore', - '/usr/bin/psql' -]) -def test_postgresql_amazon_linux_extras_exes(host, exe): - assert host.file(exe).exists - -def test_postgresql_version(host): - pg_dump_version_output = host.check_output('pg_dump --version') - assert '(PostgreSQL) 9.6' in pg_dump_version_output - -@pytest.mark.parametrize('file', [ - '/media/atl/jira/shared', - '/media/atl/jira/shared/hello', - '/media/atl/jira/shared/hello/hello.txt' -]) -def test_shared_home_owner(host, file): - assert host.file(file).exists - assert host.file(file).user == 'jira' - assert host.file(file).group == 'jira' - -def test_file_modes(host): - assert host.file('/media/atl/jira/shared/hello').mode == 0o755 - assert host.file('/media/atl/jira/shared/hello/hello.txt').mode == 0o640 - -def test_version_file_owned_by_root(host): - assert host.file('/media/atl/jira/shared/jira-software.version').exists - assert host.file('/media/atl/jira/shared/jira-software.version').user == 'root' - assert host.file('/media/atl/jira/shared/jira-software.version').group == 'root' \ No newline at end of file diff --git a/roles/restore_backups/tasks/amazon.yml b/roles/restore_backups/tasks/amazon.yml deleted file mode 100644 index 2444212..0000000 --- a/roles/restore_backups/tasks/amazon.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- - -# Amazon Linux 2 supplies extra packages via a special command. -- name: Enable Postgresql from 'extras' - command: amazon-linux-extras install -y "postgresql{{ postgres_version }}" - args: - creates: /usr/bin/psql - environment: - PYTHON: /bin/python diff --git a/roles/restore_backups/tasks/home_restore.yml b/roles/restore_backups/tasks/home_restore.yml deleted file mode 100644 index 3415231..0000000 --- a/roles/restore_backups/tasks/home_restore.yml +++ /dev/null @@ -1,75 +0,0 @@ ---- -- name: Check for the restore canary file - stat: - path: "{{ atl_backup_home_restore_canary_path }}" - register: restore_canary - -- block: - - name: Create shared home if necessary - file: - path: "{{ atl_product_home_shared }}" - state: directory - mode: 0750 - owner: "{{ atl_product_user }}" - group: "{{ atl_product_user }}" - - # We also need to use `tar` here as `unarchive` runs `tar` three times doing - # idempotence checks, which we can skip. - - name: Restore the shared-home backup - command: - argv: - - "tar" - - "--extract" - - "--file" - - "{{ atl_backup_home_dest }}" - - "--directory" - - "{{ atl_product_home_shared }}" - warn: false - when: atl_backup_home_is_server is not defined or not atl_backup_home_is_server|bool - - # Use tar transform to convert the Confluence Server (unclustered) - # layout to shared-home version. What occurs is: - # - # * --transform runs first, moving attachments into the shared home. - # * --strip-components removes the top-level directory - # - # NOTE: Also see the `confluence_config` role, which uses - # symlinks to support server and clustered layouts - # concurrently. - - name: Restore a Confluence server home to share-home layout - command: - argv: - - "tar" - - "--extract" - - "--transform=s,^attachments,shared-home/attachments," - - "--strip-components=1" - - "--file" - - "{{ atl_backup_home_dest }}" - - "--directory" - - "{{ atl_product_home_shared }}" - warn: false - when: atl_backup_home_is_server is defined and atl_backup_home_is_server|bool - - - name: Set shared home owner and group to application user - file: - path: "{{ atl_product_home_shared }}" - recurse: yes - group: "{{ atl_product_user }}" - owner: "{{ atl_product_user }}" - state: directory - - - name: Set version file owner and group to root - file: - path: "{{ atl_product_version_cache }}" - group: root - owner: root - state: file - # Ignore the error in case there is no product version file in the backup - ignore_errors: yes - - - name: Create restore-canary if necessary - copy: - dest: "{{ atl_backup_home_restore_canary_path }}" - content: "{{ atl_backup_id }}" - - when: not restore_canary.stat.exists diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml deleted file mode 100644 index f8162e0..0000000 --- a/roles/restore_backups/tasks/main.yml +++ /dev/null @@ -1,118 +0,0 @@ ---- - -# This role will attempt to fetch and load the backup manifest from a -# remote S3 URL. On successful completion the contents of JSON or YAML -# document will be in the var `atl_backup_manifest`. -# -# PREREQUISITES: -# * `atl_backup_manifest_url` points at the manifest. -# * The shared home filesystem is mounted if necessary (e.g. NFS/EFS). -# * The database has been created and the variable `db_created` is -# registered with the result (i.e: `register: db_created`). -# -# NOTE: The actual DB/FS restore operations could potentially be split -# out into discrete roles, but currently that is not required. -# -# TODO: Support HTTPS with authentication. Deferred until after the -# initial testing release. - -- block: - - - name: Ensure temp directory is present - file: - path: "{{ atl_installer_temp }}" - state: directory - mode: 0750 - owner: "{{ atl_product_user }}" - group: "{{ atl_product_user }}" - changed_when: false # For Molecule idempotence check - - - name: Parse the manifest URL - set_fact: - atl_backup_manifest_url: "{{ atl_backup_manifest_url | urlsplit }}" - - - name: Extract manifest file information - set_fact: - atl_backup_manifest_bucket: "{{ atl_backup_manifest_url.hostname }}" - atl_backup_manifest_path: "{{ atl_backup_manifest_url.path }}" - atl_backup_manifest_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest_url.path | basename }}" - - - name: Fetch the manifest from S3 - aws_s3: - mode: get - overwrite: different - bucket: "{{ atl_backup_manifest_bucket }}" - object: "{{ atl_backup_manifest_path }}" - dest: "{{ atl_backup_manifest_dest }}" - when: atl_backup_manifest_url.scheme == 's3' - - - name: Load parameters from manifest - include_vars: - file: "{{ atl_backup_manifest_dest }}" - name: atl_backup_manifest - - - name: Define the DB and home dump destinations - set_fact: - # FIXME: The manifest format is still undecided so the - # following usages will need to be updated once it settles.. - atl_backup_id: "{{ atl_backup_manifest.name }}" - atl_backup_db_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.artifacts.db.location.location | basename }}" - atl_backup_home_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.artifacts.sharedHome.location.location | basename }}" - atl_backup_home_is_server: "{{ atl_backup_manifest.artifacts.sharedHome.serverHome | default(false, true) | bool }}" - - # FIXME: Here we fetch the backups. However we may wish to stream - # these directly from S3 to the target DB/FS to avoid requiring - # disk-space for the intermediate files. - - name: Fetch DB backup from S3 - aws_s3: - mode: get - overwrite: different - bucket: "{{ atl_backup_manifest.artifacts.db.location.location | urlsplit('hostname') }}" - object: "{{ atl_backup_manifest.artifacts.db.location.location | urlsplit('path') }}" - # We save the backup as a .tar file so that the postgresql_db module uses pg_restore instead of psql to do restore - # This can be removed when ansible 2.10 is released - dest: "{{ atl_backup_db_dest }}.tar" - - - name: Fetch Home backup from S3 - aws_s3: - mode: get - overwrite: different - bucket: "{{ atl_backup_manifest.artifacts.sharedHome.location.location | urlsplit('hostname') }}" - object: "{{ atl_backup_manifest.artifacts.sharedHome.location.location | urlsplit('path') }}" - dest: "{{ atl_backup_home_dest }}" - - - name: Install distro-specific restore support packages - include_tasks: "{{ ansible_distribution|lower }}.yml" - - - # Restores the application database. If a var with name `atl_force_db_restore` is set to true, the database will be restored even when the database has not been created in the same playbook run. - # This is done to accommodate running the restore role independent of the database_init role. - - name: Restore application database - postgresql_db: - login_host: "{{ atl_db_host }}" - login_user: "{{ atl_db_root_user }}" - login_password: "{{ atl_db_root_password }}" - port: "{{ atl_db_port }}" - name: "{{ atl_jdbc_db_name }}" - owner: "{{ atl_jdbc_user }}" - encoding: "{{ atl_jdbc_encoding }}" - lc_collate: "{{ atl_jdbc_collation }}" - lc_ctype: "{{ atl_jdbc_ctype }}" - template: "{{ atl_jdbc_template }}" - # Depends on fetch_backup roles - state: restore - target: "{{ atl_backup_db_dest }}.tar" - target_opts: "-Fc" - register: result - # managed DBs in cloud providers are not allowing full root access to the DB engine, we can safely ignore the COMMENT ON EXTENSION error - failed_when: - - result.rc != 0 - - '"COMMENT ON EXTENSION" not in result.msg' - # default('false', true) filter makes the default filter return the specified default value for python False-y values (like an empty string) - when: atl_backup_db_dest is defined and (db_created.changed or (atl_force_db_restore | default('false', true) | bool)) - - - name: Restore shared home - include_tasks: "home_restore.yml" - - - when: atl_restore_required From 2afce27aa32b51e6f35f759d46065cc8ab47d641 Mon Sep 17 00:00:00 2001 From: Yifei Zhang Date: Tue, 5 Jul 2022 15:28:25 +1000 Subject: [PATCH 3/3] CLIP-1561: Updated pipeline to match number of scenarios. --- bitbucket-pipelines.yml | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index def6d71..ca1af07 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -18,7 +18,7 @@ pipelines: - step: name: Pre Parallelization stage script: - - echo "Running tests in 40 batches" + - echo "Running tests in 37 batches" - step: name: Check if number of batches match actual number of scenarios script: @@ -327,28 +327,4 @@ pipelines: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 37 - - step: - name: Molecule Test Batch - 38 - services: - - docker - script: - - apt-get update && ./bin/install-ansible --dev - - ./bin/run-tests-in-batches --batch 38 - - - step: - name: Molecule Test Batch - 39 - services: - - docker - script: - - apt-get update && ./bin/install-ansible --dev - - ./bin/run-tests-in-batches --batch 39 - - - step: - name: Molecule Test Batch - 40 - services: - - docker - script: - - apt-get update && ./bin/install-ansible --dev - - ./bin/run-tests-in-batches --batch 38 - - +