Merged in DCD-880-create-playbook-to-run-slingshot (pull request #65)

DCD-880 create playbook to run slingshot

Approved-by: Varun Arbatti

aws_slingshot_backup.yml

@@ -0,0 +1,14 @@
+# Runs atlassian-slingshot to backup your product instance. Should be run when there is only 1 app node (i.e server or single node cluster)
+# Params
+# Required
+# atl_product_edition - one of "confluence", "jira", or "bitbucket".
+# atl_product_family - should be the same as the "atl_product_edition"
+# atl_slingshot_backup_dir - working directory for the backup, should be large enough to fit database and shared home dumps
+# atl_s3_backup_bucket - s3 bucket to upload your backup to
+# atl_s3_backup_region - AWS region where the s3 bucket is deployed
+#
+# There are other variables which can be overriden. See the tasks for details. The will be defaulted by the role defaults or the inventory group_vars
+
+- hosts: aws_node_local
+  roles:
+    - role: create_backups

bin/ansible-with-atl-env

@@ -20,6 +20,7 @@ set +a
 pipenv run \
     ansible-playbook -v \
         $ATL_DEPLOYMENT_REPOSITORY_CUSTOM_PARAMS \
+        -e "${PLAYBOOK_INVOCATION_EXTRA_PARAMS}" \
         -i $INV \
         $PLAYBOOK \
         2>&1 | tee --append $LOG_FILE

bitbucket-pipelines.yml

@@ -18,7 +18,7 @@ pipelines:
     - step:
         name: Pre Parallelization stage
         script:
-          - echo "Running tests in 33 batches"
+          - echo "Running tests in 34 batches"
     - step:
         name: Check if number of batches match actual number of scenarios
         script:
@@ -38,7 +38,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 1
-
+           
        - step:
            name: Molecule Test Batch - 2
            services:
@@ -46,7 +46,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 2
-
+           
        - step:
            name: Molecule Test Batch - 3
            services:
@@ -54,7 +54,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 3
-
+           
        - step:
            name: Molecule Test Batch - 4
            services:
@@ -62,7 +62,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 4
-
+           
        - step:
            name: Molecule Test Batch - 5
            services:
@@ -70,7 +70,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 5
-
+           
        - step:
            name: Molecule Test Batch - 6
            services:
@@ -78,7 +78,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 6
-
+           
        - step:
            name: Molecule Test Batch - 7
            services:
@@ -86,7 +86,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 7
-
+           
        - step:
            name: Molecule Test Batch - 8
            services:
@@ -94,7 +94,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 8
-
+           
        - step:
            name: Molecule Test Batch - 9
            services:
@@ -102,7 +102,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 9
-
+           
        - step:
            name: Molecule Test Batch - 10
            services:
@@ -110,7 +110,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 10
-
+           
        - step:
            name: Molecule Test Batch - 11
            services:
@@ -118,7 +118,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 11
-
+           
        - step:
            name: Molecule Test Batch - 12
            services:
@@ -126,7 +126,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 12
-
+           
        - step:
            name: Molecule Test Batch - 13
            services:
@@ -134,7 +134,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 13
-
+           
        - step:
            name: Molecule Test Batch - 14
            services:
@@ -142,7 +142,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 14
-
+           
        - step:
            name: Molecule Test Batch - 15
            services:
@@ -150,7 +150,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 15
-
+           
        - step:
            name: Molecule Test Batch - 16
            services:
@@ -158,7 +158,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 16
-
+           
        - step:
            name: Molecule Test Batch - 17
            services:
@@ -166,7 +166,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 17
-
+           
        - step:
            name: Molecule Test Batch - 18
            services:
@@ -174,7 +174,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 18
-
+           
        - step:
            name: Molecule Test Batch - 19
            services:
@@ -182,7 +182,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 19
-
+           
        - step:
            name: Molecule Test Batch - 20
            services:
@@ -190,7 +190,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 20
-
+           
        - step:
            name: Molecule Test Batch - 21
            services:
@@ -198,7 +198,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 21
-
+           
        - step:
            name: Molecule Test Batch - 22
            services:
@@ -206,7 +206,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 22
-
+           
        - step:
            name: Molecule Test Batch - 23
            services:
@@ -214,7 +214,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 23
-
+           
        - step:
            name: Molecule Test Batch - 24
            services:
@@ -222,7 +222,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 24
-
+           
        - step:
            name: Molecule Test Batch - 25
            services:
@@ -230,7 +230,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 25
-
+           
        - step:
            name: Molecule Test Batch - 26
            services:
@@ -238,7 +238,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 26
-
+           
        - step:
            name: Molecule Test Batch - 27
            services:
@@ -246,7 +246,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 27
-
+           
        - step:
            name: Molecule Test Batch - 28
            services:
@@ -254,7 +254,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 28
-
+           
        - step:
            name: Molecule Test Batch - 29
            services:
@@ -262,7 +262,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 29
-
+           
        - step:
            name: Molecule Test Batch - 30
            services:
@@ -270,7 +270,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 30
-
+           
        - step:
            name: Molecule Test Batch - 31
            services:
@@ -278,7 +278,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 31
-
+           
        - step:
            name: Molecule Test Batch - 32
            services:
@@ -286,7 +286,7 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 32
-
+           
        - step:
            name: Molecule Test Batch - 33
            services:
@@ -294,3 +294,13 @@ pipelines:
            script:
            - apt-get update && ./bin/install-ansible --dev
            - ./bin/run-tests-in-batches --batch 33
+           
+       - step:
+           name: Molecule Test Batch - 34
+           services:
+             - docker
+           script:
+           - apt-get update && ./bin/install-ansible --dev
+           - ./bin/run-tests-in-batches --batch 34
+           
+       

roles/create_backups/.yamllint

@@ -0,0 +1,33 @@
+---
+# Based on ansible-lint config
+extends: default
+
+rules:
+  braces:
+    max-spaces-inside: 1
+    level: error
+  brackets:
+    max-spaces-inside: 1
+    level: error
+  colons:
+    max-spaces-after: -1
+    level: error
+  commas:
+    max-spaces-after: -1
+    level: error
+  comments: disable
+  comments-indentation: disable
+  document-start: disable
+  empty-lines:
+    max: 3
+    level: error
+  hyphens:
+    level: error
+  indentation: disable
+  key-duplicates: enable
+  line-length: disable
+  new-line-at-end-of-file: disable
+  new-lines:
+    type: unix
+  trailing-spaces: disable
+  truthy: disable

roles/create_backups/defaults/main.yml

@@ -0,0 +1,11 @@
+---
+
+atl_db_port: 5432
+atl_systemd_service_name: "{{ atl_product_edition }}.service"
+
+atl_slingshot_yum_repo_url: 'https://packages.atlassian.com/atlassian-slingshot-rpm'
+atl_amazon_extras_postgresql_version: '9.6'
+
+atl_slingshot_deb_pubkey_url: 'https://packages.atlassian.com/api/gpg/key/public'
+atl_slingshot_deb_repo_url: 'https://packages.atlassian.com/atlassian-slingshot-deb'
+atl_slingshot_deb_repo_components: 'stable main'

roles/create_backups/molecule/default/Dockerfile.j2

@@ -0,0 +1,22 @@
+# Molecule managed
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+{% if item.env is defined %}
+{% for var, value in item.env.items() %}
+{% if value %}
+ENV {{ var }} {{ value }}
+{% endif %}
+{% endfor %}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates iproute2 && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash iproute && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash iproute && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml iproute2 && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates iproute2 && xbps-remove -O; fi

roles/create_backups/molecule/default/INSTALL.rst

@@ -0,0 +1,22 @@
+*******
+Docker driver installation guide
+*******
+
+Requirements
+============
+
+* Docker Engine
+
+Install
+=======
+
+Please refer to the `Virtual environment`_ documentation for installation best
+practices. If not using a virtual environment, please consider passing the
+widely recommended `'--user' flag`_ when invoking ``pip``.
+
+.. _Virtual environment: https://virtualenv.pypa.io/en/latest/
+.. _'--user' flag: https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site
+
+.. code-block:: bash
+
+    $ pip install 'molecule[docker]'

roles/create_backups/molecule/default/molecule.yml

@@ -0,0 +1,20 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+lint:
+  name: yamllint
+platforms:
+  - name: amazon_linux2
+    image: amazonlinux:2
+  - name: ubuntu_lts
+    image: ubuntu:bionic
+provisioner:
+  name: ansible
+  lint:
+    name: ansible-lint
+verifier:
+  name: testinfra
+  lint:
+    name: flake8

roles/create_backups/molecule/default/playbook.yml

@@ -0,0 +1,11 @@
+---
+- name: Converge
+  hosts: all
+  tasks:
+    # Role default vars must be included because we aren't running the whole role, just a subset of tasks
+    - name: Include role vars
+      include_vars:
+        file: ../../defaults/main.yml
+
+    - name: Install distro-specific backup support packages
+      include_tasks: "../../tasks/{{ ansible_distribution|lower }}.yml"

roles/create_backups/molecule/default/tests/test_default.py

@@ -0,0 +1,11 @@
+import os
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_slingshot_installed(host):
+    assert host.file('/usr/bin/atlassian-slingshot').exists
+    assert host.file('/usr/bin/psql').exists

roles/create_backups/tasks/amazon.yml

@@ -0,0 +1,21 @@
+---
+
+- name: Add atlassian-slingshot yum repository
+  yum_repository:
+    name: AtlassianSlingshot
+    file: atlassian-slingshot
+    state: present
+    description: Atlassian Slingshot
+    baseurl: "{{ atl_slingshot_yum_repo_url }}"
+    gpgcheck: no
+
+- name: Install atlassian-slingshot
+  yum:
+    name: atlassian-slingshot
+
+- name: Enable Postgresql from 'extras'
+  command: amazon-linux-extras install -y "postgresql{{ atl_amazon_extras_postgresql_version }}"
+  args:
+    creates: /usr/bin/psql
+  environment:
+    PYTHON: /bin/python

roles/create_backups/tasks/main.yml

@@ -0,0 +1,38 @@
+---
+
+- name: Install distro-specific backup support packages
+  include_tasks: "{{ ansible_distribution|lower }}.yml"
+
+- name: Stop the {{ atl_product_edition }} service
+  service:
+    name: "{{ atl_systemd_service_name }}"
+    state: stopped
+
+- name: Run the slingshot backup
+  command:
+    argv:
+      - "atlassian-slingshot"
+      - "backup"
+      - "--backupDir"
+      - "{{ atl_slingshot_backup_dir }}"
+      - "--dbHost"
+      - "{{ atl_db_host }}"
+      - "--dbPort"
+      - "{{ atl_db_port }}"
+      - "--dbUser"
+      - "{{ atl_jdbc_user }}"
+      - "--dbName"
+      - "{{ atl_jdbc_db_name }}"
+      - "--s3BucketName"
+      - "{{ atl_s3_backup_bucket }}"
+      - "--awsRegion"
+      - "{{ atl_s3_backup_region }}"
+      - "--sharedHomeLocation"
+      - "{{ atl_product_home_shared }}"
+  environment:
+    ATL_DB_PASSWORD: "{{ atl_jdbc_password }}"
+
+- name: Restart the {{ atl_product_edition }} service
+  service:
+    name: "{{ atl_systemd_service_name }}"
+    state: restarted

roles/create_backups/tasks/ubuntu.yml

@@ -0,0 +1,27 @@
+---
+- name: Install gnupg
+  apt:
+    name: gnupg
+    state: present
+
+- name: Add atlassian-slingshot debian repo public key
+  apt_key:
+    url: "{{ atl_slingshot_deb_pubkey_url }}"
+    state: present
+
+- name: Add atlassian-slingshot debian repository
+  apt_repository:
+    repo: "deb {{ atl_slingshot_deb_repo_url }} {{ atl_slingshot_deb_repo_components }}"
+    state: present
+    filename: atlassian-slingshot
+
+- name: Install atlassian-slingshot
+  apt:
+    name: atlassian-slingshot
+    update_cache: yes
+    state: present
+
+- name: Install postgresql
+  apt:
+    name: postgresql
+    state: present