Merge remote-tracking branch 'origin/master' into DCD-798-fix-for-pg_dump-cusstom-format

roles/restore_backups/molecule/restore/Dockerfile.j2

@@ -0,0 +1,14 @@
+# Molecule managed
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi

roles/restore_backups/molecule/restore/molecule.yml

@@ -0,0 +1,30 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+lint:
+  name: yamllint
+platforms:
+  - name: amazon_linux2
+    image: amazonlinux:2
+    groups:
+      - aws_node_local
+    ulimits:
+      - nofile:262144:262144
+provisioner:
+  name: ansible
+  options:
+    skip-tags: runtime_pkg
+  lint:
+    name: ansible-lint
+    options:
+      x: ["701"]
+  inventory:
+    links:
+      group_vars: ../../../../group_vars/
+verifier:
+  name: testinfra
+  lint:
+    name: flake8
+    enabled: false

roles/restore_backups/molecule/restore/playbook.yml

@@ -0,0 +1,75 @@
+---
+- name: Converge
+  hosts: all
+  vars:
+    atl_backup_home_dest: "{{ test_archive }}"
+    atl_backup_home_restore_canary_path: '/tmp/canary.tmp'
+    atl_backup_id: 'test-backup'
+    atl_backup_manifest_url: 'fake_manifest'
+
+    atl_product_edition: 'jira-software'
+    atl_product_home_shared: '/media/atl/jira/shared'
+    atl_product_user: 'jira'
+    atl_product_user_uid: '2001'
+    atl_product_version_cache: "{{ atl_product_home_shared }}/{{ atl_product_edition }}.version"
+
+    test_archive: '/tmp/hello.tar.gz'
+    test_archive_file: 'hello.txt'
+    test_archive_source: '/tmp/hello'
+    test_pre_step_prefix: '[PRE-TEST]'
+    test_product_version_file: "/tmp/{{ atl_product_edition }}.version"
+
+  pre_tasks:
+    - name: "{{ test_pre_step_prefix }} Install tar"
+      package:
+        state: present
+        name: tar
+
+    - name: "{{ test_pre_step_prefix }} Install useradd and groupadd binaries"
+      package:
+        state: present
+        name: shadow-utils
+
+    - name: "{{ test_pre_step_prefix }} Create application group"
+      group:
+        name: "{{ atl_product_user }}"
+        gid: "{{ atl_product_user_uid }}"
+
+    - name: "{{ test_pre_step_prefix }} Create application user"
+      user:
+        name: "{{ atl_product_user }}"
+        uid: "{{ atl_product_user_uid }}"
+        group: "{{ atl_product_user }}"
+
+    - block:
+        - name: "{{ test_pre_step_prefix }} Create a directory for the shared home archive"
+          file:
+            path: "{{ test_archive_source }}"
+            state: directory
+            mode: 0755
+        - name: "{{ test_pre_step_prefix }} Create a file in the shared home"
+          lineinfile:
+            create: yes
+            line: 'Hello, world!'
+            path: "{{ test_archive_source }}/{{ test_archive_file }}"
+            mode: 0640
+        - name: "{{ test_pre_step_prefix }} Create the version file in the shared home"
+          lineinfile:
+            create: yes
+            line: '8.5'
+            path: "{{ test_product_version_file }}"
+            mode: 0640
+        - name: "{{ test_pre_step_prefix }} Archive the shared home"
+          archive:
+            path:
+              - "{{ test_archive_source }}"
+              - "{{ test_product_version_file }}"
+            dest: "{{ test_archive }}"
+            owner: "{{ atl_product_user }}"
+
+  tasks:
+    - name: Install distro-specific restore support packages
+      include_tasks: "../../tasks/{{ ansible_distribution|lower }}.yml"
+
+    - name: Restore shared home
+      include_tasks: "../../tasks/home_restore.yml"

roles/restore_backups/molecule/restore/tests/test_default.py

@@ -0,0 +1,39 @@
+import os
+import pytest
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+@pytest.mark.parametrize('exe', [
+    '/usr/bin/pg_dump',
+    '/usr/bin/pg_restore',
+    '/usr/bin/psql'
+])
+def test_postgresql_amazon_linux_extras_exes(host, exe):
+    assert host.file(exe).exists
+
+def test_postgresql_version(host):
+    pg_dump_version_output = host.check_output('pg_dump --version')
+    assert '(PostgreSQL) 9.6' in pg_dump_version_output
+
+@pytest.mark.parametrize('file', [
+    '/media/atl/jira/shared',
+    '/media/atl/jira/shared/hello',
+    '/media/atl/jira/shared/hello/hello.txt'
+])
+def test_shared_home_owner(host, file):
+    assert host.file(file).exists
+    assert host.file(file).user == 'jira'
+    assert host.file(file).group == 'jira'
+
+def test_file_modes(host):
+    assert host.file('/media/atl/jira/shared/hello').mode == 0o755
+    assert host.file('/media/atl/jira/shared/hello/hello.txt').mode == 0o640
+
+def test_version_file_owned_by_root(host):
+    assert host.file('/media/atl/jira/shared/jira-software.version').exists
+    assert host.file('/media/atl/jira/shared/jira-software.version').user == 'root'
+    assert host.file('/media/atl/jira/shared/jira-software.version').group == 'root'

roles/restore_backups/tasks/amazon.yml

@@ -8,3 +8,5 @@
     PYTHON: /bin/python
   args:
     creates: /usr/bin/psql
+  environment:
+    PYTHON: /bin/python

roles/restore_backups/tasks/home_restore.yml

@@ -0,0 +1,44 @@
+---
+- name: Check for the restore canary file
+  stat:
+    path: "{{ atl_backup_home_restore_canary_path }}"
+  register: restore_canary
+
+- block:
+    - name: Create shared home if necessary
+      file:
+        path: "{{ atl_product_home_shared }}"
+        state: directory
+        mode: 0750
+        owner: "{{ atl_product_user }}"
+        group: "{{ atl_product_user }}"
+
+    - name: Restore the shared-home backup
+      unarchive:
+        src: "{{ atl_backup_home_dest }}"
+        remote_src: yes
+        dest: "{{ atl_product_home_shared }}"
+
+    - name: Set shared home owner and group to application user
+      file:
+        path: "{{ atl_product_home_shared }}"
+        recurse: yes
+        group: "{{ atl_product_user }}"
+        owner: "{{ atl_product_user }}"
+        state: directory
+
+    - name: Set version file owner and group to root
+      file:
+        path: "{{ atl_product_version_cache }}"
+        group: root
+        owner: root
+        state: file
+      # Ignore the error in case there is no product version file in the backup
+      ignore_errors: yes
+
+    - name: Create restore-canary if necessary
+      copy:
+        dest: "{{ atl_backup_home_restore_canary_path }}"
+        content: "{{ atl_backup_id }}"
+
+  when: not restore_canary.stat.exists

roles/restore_backups/tasks/main.yml

@@ -68,7 +68,9 @@
         overwrite: different
         bucket: "{{ atl_backup_manifest.artifacts.db.location.location | urlsplit('hostname') }}"
         object: "{{ atl_backup_manifest.artifacts.db.location.location | urlsplit('path') }}"
-        dest: "{{ atl_backup_db_dest }}"
+        # We save the backup as a .tar file so that the postgresql_db module uses pg_restore instead of psql to do restore
+        # This can be removed when ansible 2.10 is released
+        dest: "{{ atl_backup_db_dest }}.tar"
 
     - name: Fetch Home backup from S3
       aws_s3:
@@ -96,42 +98,18 @@
         template: "{{ atl_jdbc_template }}"
         # Depends on fetch_backup roles
         state: restore
-        target: "{{ atl_backup_db_dest }}"
+        target: "{{ atl_backup_db_dest }}.tar"
+        target_opts: "-Fc"
       register: result
+      # managed DBs in cloud providers are not allowing full root access to the DB engine, we can safely ignore the COMMENT ON EXTENSION error
       failed_when:
         - result.rc != 0
-        - "COMMENT ON EXTENSION" not in result.msg
+        - '"COMMENT ON EXTENSION" not in result.msg'
       when: db_created.changed and atl_backup_db_dest is defined
 
 
-    - name: Check for the restore canary file
-      stat:
-        path: "{{ atl_backup_home_restore_canary_path }}"
-      register: restore_canary
-
-    - block:
-
-        - name: Create shared home if necessary
-          file:
-            path: "{{ atl_product_home_shared }}"
-            state: directory
-            mode: 0750
-            owner: "{{ atl_product_user }}"
-            group: "{{ atl_product_user }}"
-
-        - name: Restore the shared-home backup
-          unarchive:
-            src: "{{ atl_backup_home_dest }}"
-            dest: "{{ atl_product_home_shared }}"
-            owner: "{{ atl_product_user }}"
-            group: "{{ atl_product_user }}"
-
-        - name: Create restore-canary if necessary
-          copy:
-            dest: "{{ atl_backup_home_restore_canary_path }}"
-            content: "{{ atl_backup_id }}"
-
-      when: not restore_canary.stat.exists
+    - name: Restore shared home
+      include_tasks: "home_restore.yml"
 
 
   when: atl_restore_required