diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index c9f6057..a58d52c 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -34,7 +34,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 1 - + - step: name: Molecule Test Batch - 2 services: @@ -42,7 +42,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 2 - + - step: name: Molecule Test Batch - 3 services: @@ -50,7 +50,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 3 - + - step: name: Molecule Test Batch - 4 services: @@ -58,7 +58,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 4 - + - step: name: Molecule Test Batch - 5 services: @@ -66,7 +66,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 5 - + - step: name: Molecule Test Batch - 6 services: @@ -74,7 +74,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 6 - + - step: name: Molecule Test Batch - 7 services: @@ -82,7 +82,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 7 - + - step: name: Molecule Test Batch - 8 services: @@ -90,7 +90,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 8 - + - step: name: Molecule Test Batch - 9 services: @@ -98,7 +98,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 9 - + - step: name: Molecule Test Batch - 10 services: @@ -106,7 +106,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 10 - + - step: name: Molecule Test Batch - 11 services: @@ -114,7 +114,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 11 - + - step: name: Molecule Test Batch - 12 services: @@ -122,7 +122,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 12 - + - step: name: Molecule Test Batch - 13 services: @@ -130,7 +130,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 13 - + - step: name: Molecule Test Batch - 14 services: @@ -138,7 +138,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 14 - + - step: name: Molecule Test Batch - 15 services: @@ -146,7 +146,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 15 - + - step: name: Molecule Test Batch - 16 services: @@ -154,7 +154,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 16 - + - step: name: Molecule Test Batch - 17 services: @@ -162,7 +162,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 17 - + - step: name: Molecule Test Batch - 18 services: @@ -170,7 +170,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 18 - + - step: name: Molecule Test Batch - 19 services: @@ -178,7 +178,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 19 - + - step: name: Molecule Test Batch - 20 services: @@ -186,7 +186,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 20 - + - step: name: Molecule Test Batch - 21 services: @@ -194,7 +194,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 21 - + - step: name: Molecule Test Batch - 22 services: @@ -202,7 +202,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 22 - + - step: name: Molecule Test Batch - 23 services: @@ -210,7 +210,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 23 - + - step: name: Molecule Test Batch - 24 services: @@ -218,7 +218,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 24 - + - step: name: Molecule Test Batch - 25 services: @@ -226,7 +226,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 25 - + - step: name: Molecule Test Batch - 26 services: @@ -234,7 +234,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 26 - + - step: name: Molecule Test Batch - 27 services: @@ -242,7 +242,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 27 - + - step: name: Molecule Test Batch - 28 services: @@ -250,7 +250,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 28 - + - step: name: Molecule Test Batch - 29 services: @@ -258,7 +258,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 29 - + - step: name: Molecule Test Batch - 30 services: @@ -266,7 +266,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 30 - + - step: name: Molecule Test Batch - 31 services: @@ -274,5 +274,4 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 31 - - + diff --git a/roles/restore_backups/molecule/restore/Dockerfile.j2 b/roles/restore_backups/molecule/restore/Dockerfile.j2 new file mode 100644 index 0000000..e6aa95d --- /dev/null +++ b/roles/restore_backups/molecule/restore/Dockerfile.j2 @@ -0,0 +1,14 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi diff --git a/roles/restore_backups/molecule/restore/molecule.yml b/roles/restore_backups/molecule/restore/molecule.yml new file mode 100644 index 0000000..04c0973 --- /dev/null +++ b/roles/restore_backups/molecule/restore/molecule.yml @@ -0,0 +1,30 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: amazon_linux2 + image: amazonlinux:2 + groups: + - aws_node_local + ulimits: + - nofile:262144:262144 +provisioner: + name: ansible + options: + skip-tags: runtime_pkg + lint: + name: ansible-lint + options: + x: ["701"] + inventory: + links: + group_vars: ../../../../group_vars/ +verifier: + name: testinfra + lint: + name: flake8 + enabled: false diff --git a/roles/restore_backups/molecule/restore/playbook.yml b/roles/restore_backups/molecule/restore/playbook.yml new file mode 100644 index 0000000..b3c60b0 --- /dev/null +++ b/roles/restore_backups/molecule/restore/playbook.yml @@ -0,0 +1,11 @@ +--- +- name: Converge + hosts: all + vars: + atl_backup_manifest_url: 's3://dcd-slingshot-test/dummy_manifest.json' + atl_product_user: 'jira' + atl_backup_home_restore_canary_path: '/tmp/canary.tmp' + + tasks: + - name: Install distro-specific restore support packages + include_tasks: "../../tasks/{{ ansible_distribution|lower }}.yml" diff --git a/roles/restore_backups/molecule/restore/tests/test_default.py b/roles/restore_backups/molecule/restore/tests/test_default.py new file mode 100644 index 0000000..6f88a4f --- /dev/null +++ b/roles/restore_backups/molecule/restore/tests/test_default.py @@ -0,0 +1,20 @@ +import os +import pytest + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +@pytest.mark.parametrize('exe', [ + '/usr/bin/pg_dump', + '/usr/bin/pg_restore', + '/usr/bin/psql' +]) +def test_postgresql_amazon_linux_extras_exes(host, exe): + assert host.file(exe).exists + +def test_postgresql_version(host): + pg_dump_version_output = host.check_output('pg_dump --version') + assert '(PostgreSQL) 9.6' in pg_dump_version_output diff --git a/roles/restore_backups/tasks/amazon.yml b/roles/restore_backups/tasks/amazon.yml index bf32125..2444212 100644 --- a/roles/restore_backups/tasks/amazon.yml +++ b/roles/restore_backups/tasks/amazon.yml @@ -5,3 +5,5 @@ command: amazon-linux-extras install -y "postgresql{{ postgres_version }}" args: creates: /usr/bin/psql + environment: + PYTHON: /bin/python diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml index 4d6865f..1f9ebdb 100644 --- a/roles/restore_backups/tasks/main.yml +++ b/roles/restore_backups/tasks/main.yml @@ -56,8 +56,8 @@ # FIXME: The manifest format is still undecided so the # following usages will need to be updated once it settles.. atl_backup_id: "{{ atl_backup_manifest.name }}" - atl_backup_db_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.artifacts.db.location.value | basename }}" - atl_backup_home_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.artifacts.sharedHome.location.value | basename }}" + atl_backup_db_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.artifacts.db.location.location | basename }}" + atl_backup_home_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.artifacts.sharedHome.location.location | basename }}" # FIXME: Here we fetch the backups. However we may wish to stream # these directly from S3 to the target DB/FS to avoid requiring @@ -66,16 +66,16 @@ aws_s3: mode: get overwrite: different - bucket: "{{ atl_backup_manifest.artifacts.db.location.value | urlsplit('hostname') }}" - object: "{{ atl_backup_manifest.artifacts.db.location.value | urlsplit('path') }}" + bucket: "{{ atl_backup_manifest.artifacts.db.location.location | urlsplit('hostname') }}" + object: "{{ atl_backup_manifest.artifacts.db.location.location | urlsplit('path') }}" dest: "{{ atl_backup_db_dest }}" - name: Fetch Home backup from S3 aws_s3: mode: get overwrite: different - bucket: "{{ atl_backup_manifest.artifacts.sharedHome.location.value | urlsplit('hostname') }}" - object: "{{ atl_backup_manifest.artifacts.sharedHome.location.value | urlsplit('path') }}" + bucket: "{{ atl_backup_manifest.artifacts.sharedHome.location.location | urlsplit('hostname') }}" + object: "{{ atl_backup_manifest.artifacts.sharedHome.location.location | urlsplit('path') }}" dest: "{{ atl_backup_home_dest }}" - name: Install distro-specific restore support packages @@ -97,6 +97,11 @@ # Depends on fetch_backup roles state: restore target: "{{ atl_backup_db_dest }}" + register: result + # managed DBs in cloud providers are not allowing full root access to the DB engine, we can safely ignore the COMMENT ON EXTENSION error + failed_when: + - result.rc != 0 + - '"COMMENT ON EXTENSION" not in result.msg' when: db_created.changed and atl_backup_db_dest is defined