From e7de5baff78906610e28111964c564e777077c13 Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Fri, 3 Jan 2020 11:33:54 +1100 Subject: [PATCH 01/16] DCD-880: Create role to run slingshot and implement task to install slingshot on amazon linux --- roles/create_backups/.yamllint | 33 +++++++++++++++++++ .../molecule/default/Dockerfile.j2 | 22 +++++++++++++ .../molecule/default/INSTALL.rst | 22 +++++++++++++ .../molecule/default/molecule.yml | 24 ++++++++++++++ .../molecule/default/playbook.yml | 6 ++++ .../molecule/default/tests/test_default.py | 10 ++++++ roles/create_backups/tasks/amazon.yml | 14 ++++++++ 7 files changed, 131 insertions(+) create mode 100644 roles/create_backups/.yamllint create mode 100644 roles/create_backups/molecule/default/Dockerfile.j2 create mode 100644 roles/create_backups/molecule/default/INSTALL.rst create mode 100644 roles/create_backups/molecule/default/molecule.yml create mode 100644 roles/create_backups/molecule/default/playbook.yml create mode 100644 roles/create_backups/molecule/default/tests/test_default.py create mode 100644 roles/create_backups/tasks/amazon.yml diff --git a/roles/create_backups/.yamllint b/roles/create_backups/.yamllint new file mode 100644 index 0000000..8827676 --- /dev/null +++ b/roles/create_backups/.yamllint @@ -0,0 +1,33 @@ +--- +# Based on ansible-lint config +extends: default + +rules: + braces: + max-spaces-inside: 1 + level: error + brackets: + max-spaces-inside: 1 + level: error + colons: + max-spaces-after: -1 + level: error + commas: + max-spaces-after: -1 + level: error + comments: disable + comments-indentation: disable + document-start: disable + empty-lines: + max: 3 + level: error + hyphens: + level: error + indentation: disable + key-duplicates: enable + line-length: disable + new-line-at-end-of-file: disable + new-lines: + type: unix + trailing-spaces: disable + truthy: disable diff --git a/roles/create_backups/molecule/default/Dockerfile.j2 b/roles/create_backups/molecule/default/Dockerfile.j2 new file mode 100644 index 0000000..0de39e6 --- /dev/null +++ b/roles/create_backups/molecule/default/Dockerfile.j2 @@ -0,0 +1,22 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +{% if item.env is defined %} +{% for var, value in item.env.items() %} +{% if value %} +ENV {{ var }} {{ value }} +{% endif %} +{% endfor %} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates iproute2 && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash iproute && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash iproute && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml iproute2 && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates iproute2 && xbps-remove -O; fi diff --git a/roles/create_backups/molecule/default/INSTALL.rst b/roles/create_backups/molecule/default/INSTALL.rst new file mode 100644 index 0000000..6a44bde --- /dev/null +++ b/roles/create_backups/molecule/default/INSTALL.rst @@ -0,0 +1,22 @@ +******* +Docker driver installation guide +******* + +Requirements +============ + +* Docker Engine + +Install +======= + +Please refer to the `Virtual environment`_ documentation for installation best +practices. If not using a virtual environment, please consider passing the +widely recommended `'--user' flag`_ when invoking ``pip``. + +.. _Virtual environment: https://virtualenv.pypa.io/en/latest/ +.. _'--user' flag: https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site + +.. code-block:: bash + + $ pip install 'molecule[docker]' diff --git a/roles/create_backups/molecule/default/molecule.yml b/roles/create_backups/molecule/default/molecule.yml new file mode 100644 index 0000000..e1a8dce --- /dev/null +++ b/roles/create_backups/molecule/default/molecule.yml @@ -0,0 +1,24 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: amazon_linux2 + image: amazonlinux:2 + groups: + - aws_node_local + # - name: ubuntu_lts + # image: ubuntu:bionic + # groups: + # - aws_node_local +provisioner: + name: ansible + lint: + name: ansible-lint +verifier: + name: testinfra + lint: + name: flake8 diff --git a/roles/create_backups/molecule/default/playbook.yml b/roles/create_backups/molecule/default/playbook.yml new file mode 100644 index 0000000..c57f038 --- /dev/null +++ b/roles/create_backups/molecule/default/playbook.yml @@ -0,0 +1,6 @@ +--- +- name: Converge + hosts: all + tasks: + - name: Install distro-specific backup support packages + include_tasks: "../../tasks/{{ ansible_distribution|lower }}.yml" diff --git a/roles/create_backups/molecule/default/tests/test_default.py b/roles/create_backups/molecule/default/tests/test_default.py new file mode 100644 index 0000000..4b2be48 --- /dev/null +++ b/roles/create_backups/molecule/default/tests/test_default.py @@ -0,0 +1,10 @@ +import os + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +def test_slingshot_installed(host): + assert host.file('/usr/bin/atlassian-slingshot').exists diff --git a/roles/create_backups/tasks/amazon.yml b/roles/create_backups/tasks/amazon.yml new file mode 100644 index 0000000..d9a5e6d --- /dev/null +++ b/roles/create_backups/tasks/amazon.yml @@ -0,0 +1,14 @@ +--- + +- name: Add atlassian-slingshot yum repository + yum_repository: + name: AtlassianSlingshot + file: atlassian-slingshot + state: present + description: Atlassian Slingshot + baseurl: https://packages.atlassian.com/atlassian-slingshot-rpm + gpgcheck: no + +- name: Install atlassian-slingshot + yum: + name: atlassian-slingshot \ No newline at end of file From 7f715f409c2e61b7706f3dd9f17cc35a50faa6fc Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Fri, 3 Jan 2020 11:51:26 +1100 Subject: [PATCH 02/16] DCD-880: Add task to install atlassian-slingshot on ubuntu --- .../molecule/default/molecule.yml | 8 +++---- roles/create_backups/tasks/ubuntu.yml | 22 +++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 roles/create_backups/tasks/ubuntu.yml diff --git a/roles/create_backups/molecule/default/molecule.yml b/roles/create_backups/molecule/default/molecule.yml index e1a8dce..ecc65a7 100644 --- a/roles/create_backups/molecule/default/molecule.yml +++ b/roles/create_backups/molecule/default/molecule.yml @@ -10,10 +10,10 @@ platforms: image: amazonlinux:2 groups: - aws_node_local - # - name: ubuntu_lts - # image: ubuntu:bionic - # groups: - # - aws_node_local + - name: ubuntu_lts + image: ubuntu:bionic + groups: + - aws_node_local provisioner: name: ansible lint: diff --git a/roles/create_backups/tasks/ubuntu.yml b/roles/create_backups/tasks/ubuntu.yml new file mode 100644 index 0000000..1b050e2 --- /dev/null +++ b/roles/create_backups/tasks/ubuntu.yml @@ -0,0 +1,22 @@ +--- +- name: Install gnupg + apt: + name: gnupg + state: present + +- name: Add atlassian-slingshot debian repo public key + apt_key: + url: https://packages.atlassian.com/api/gpg/key/public + state: present + +- name: Add atlassian-slingshot debian repository + apt_repository: + repo: deb https://packages.atlassian.com/atlassian-slingshot-deb stable main + state: present + filename: atlassian-slingshot + +- name: Install atlassian-slingshot + apt: + name: atlassian-slingshot + update_cache: yes + state: present \ No newline at end of file From 58692b28431233c28f0e7255fa3a61874d9df220 Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Fri, 3 Jan 2020 12:04:40 +1100 Subject: [PATCH 03/16] DCD-880: Variablify create_backups tasks --- roles/create_backups/defaults/main.yml | 6 ++++++ roles/create_backups/molecule/default/molecule.yml | 4 ---- roles/create_backups/molecule/default/playbook.yml | 5 +++++ roles/create_backups/tasks/amazon.yml | 2 +- roles/create_backups/tasks/ubuntu.yml | 4 ++-- 5 files changed, 14 insertions(+), 7 deletions(-) create mode 100644 roles/create_backups/defaults/main.yml diff --git a/roles/create_backups/defaults/main.yml b/roles/create_backups/defaults/main.yml new file mode 100644 index 0000000..f687b28 --- /dev/null +++ b/roles/create_backups/defaults/main.yml @@ -0,0 +1,6 @@ +--- +atl_slingshot_yum_repo_url: 'https://packages.atlassian.com/atlassian-slingshot-rpm' + +atl_slingshot_deb_pubkey_url: 'https://packages.atlassian.com/api/gpg/key/public' +atl_slingshot_deb_repo_url: 'https://packages.atlassian.com/atlassian-slingshot-deb' +atl_slingshot_deb_repo_components: 'stable main' \ No newline at end of file diff --git a/roles/create_backups/molecule/default/molecule.yml b/roles/create_backups/molecule/default/molecule.yml index ecc65a7..c6a6d26 100644 --- a/roles/create_backups/molecule/default/molecule.yml +++ b/roles/create_backups/molecule/default/molecule.yml @@ -8,12 +8,8 @@ lint: platforms: - name: amazon_linux2 image: amazonlinux:2 - groups: - - aws_node_local - name: ubuntu_lts image: ubuntu:bionic - groups: - - aws_node_local provisioner: name: ansible lint: diff --git a/roles/create_backups/molecule/default/playbook.yml b/roles/create_backups/molecule/default/playbook.yml index c57f038..34a4c1b 100644 --- a/roles/create_backups/molecule/default/playbook.yml +++ b/roles/create_backups/molecule/default/playbook.yml @@ -2,5 +2,10 @@ - name: Converge hosts: all tasks: + # Role default vars must be included because we aren't running the whole role, just a subset of tasks + - name: Include role vars + include_vars: + file: ../../defaults/main.yml + - name: Install distro-specific backup support packages include_tasks: "../../tasks/{{ ansible_distribution|lower }}.yml" diff --git a/roles/create_backups/tasks/amazon.yml b/roles/create_backups/tasks/amazon.yml index d9a5e6d..e89b233 100644 --- a/roles/create_backups/tasks/amazon.yml +++ b/roles/create_backups/tasks/amazon.yml @@ -6,7 +6,7 @@ file: atlassian-slingshot state: present description: Atlassian Slingshot - baseurl: https://packages.atlassian.com/atlassian-slingshot-rpm + baseurl: "{{ atl_slingshot_yum_repo_url }}" gpgcheck: no - name: Install atlassian-slingshot diff --git a/roles/create_backups/tasks/ubuntu.yml b/roles/create_backups/tasks/ubuntu.yml index 1b050e2..218c579 100644 --- a/roles/create_backups/tasks/ubuntu.yml +++ b/roles/create_backups/tasks/ubuntu.yml @@ -6,12 +6,12 @@ - name: Add atlassian-slingshot debian repo public key apt_key: - url: https://packages.atlassian.com/api/gpg/key/public + url: "{{ atl_slingshot_deb_pubkey_url }}" state: present - name: Add atlassian-slingshot debian repository apt_repository: - repo: deb https://packages.atlassian.com/atlassian-slingshot-deb stable main + repo: "deb {{ atl_slingshot_deb_repo_url }} {{ atl_slingshot_deb_repo_components }}" state: present filename: atlassian-slingshot From 5d543b3f0c7e91439cddd089baded05d288db96a Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Fri, 3 Jan 2020 13:32:27 +1100 Subject: [PATCH 04/16] DCD-880: Install postgresql as part of slingshot backup --- roles/create_backups/defaults/main.yml | 1 + .../create_backups/molecule/default/tests/test_default.py | 1 + roles/create_backups/tasks/amazon.yml | 7 ++++++- roles/create_backups/tasks/main.yml | 5 +++++ roles/create_backups/tasks/ubuntu.yml | 5 +++++ 5 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 roles/create_backups/tasks/main.yml diff --git a/roles/create_backups/defaults/main.yml b/roles/create_backups/defaults/main.yml index f687b28..5db9b36 100644 --- a/roles/create_backups/defaults/main.yml +++ b/roles/create_backups/defaults/main.yml @@ -1,5 +1,6 @@ --- atl_slingshot_yum_repo_url: 'https://packages.atlassian.com/atlassian-slingshot-rpm' +atl_amazon_extras_postgresql_version: '9.6' atl_slingshot_deb_pubkey_url: 'https://packages.atlassian.com/api/gpg/key/public' atl_slingshot_deb_repo_url: 'https://packages.atlassian.com/atlassian-slingshot-deb' diff --git a/roles/create_backups/molecule/default/tests/test_default.py b/roles/create_backups/molecule/default/tests/test_default.py index 4b2be48..69b0d76 100644 --- a/roles/create_backups/molecule/default/tests/test_default.py +++ b/roles/create_backups/molecule/default/tests/test_default.py @@ -8,3 +8,4 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( def test_slingshot_installed(host): assert host.file('/usr/bin/atlassian-slingshot').exists + assert host.file('/usr/bin/psql').exists diff --git a/roles/create_backups/tasks/amazon.yml b/roles/create_backups/tasks/amazon.yml index e89b233..796ef30 100644 --- a/roles/create_backups/tasks/amazon.yml +++ b/roles/create_backups/tasks/amazon.yml @@ -11,4 +11,9 @@ - name: Install atlassian-slingshot yum: - name: atlassian-slingshot \ No newline at end of file + name: atlassian-slingshot + +- name: Enable Postgresql from 'extras' + command: amazon-linux-extras install -y "postgresql{{ atl_amazon_extras_postgresql_version }}" + args: + creates: /usr/bin/psql diff --git a/roles/create_backups/tasks/main.yml b/roles/create_backups/tasks/main.yml new file mode 100644 index 0000000..a20ae3f --- /dev/null +++ b/roles/create_backups/tasks/main.yml @@ -0,0 +1,5 @@ +--- +- name: Install distro-specific backup support packages + include_tasks: "../../tasks/{{ ansible_distribution|lower }}.yml" + + \ No newline at end of file diff --git a/roles/create_backups/tasks/ubuntu.yml b/roles/create_backups/tasks/ubuntu.yml index 218c579..94476a6 100644 --- a/roles/create_backups/tasks/ubuntu.yml +++ b/roles/create_backups/tasks/ubuntu.yml @@ -19,4 +19,9 @@ apt: name: atlassian-slingshot update_cache: yes + state: present + +- name: Install postgresql + apt: + name: postgresql state: present \ No newline at end of file From 60c51da160f26f377f9caeafdce2204a7c780190 Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Fri, 3 Jan 2020 13:58:26 +1100 Subject: [PATCH 05/16] DCD-880: stop product service and run slingshot in backup playbook --- roles/create_backups/defaults/main.yml | 6 +++++- roles/create_backups/tasks/main.yml | 26 +++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/roles/create_backups/defaults/main.yml b/roles/create_backups/defaults/main.yml index 5db9b36..e00f60f 100644 --- a/roles/create_backups/defaults/main.yml +++ b/roles/create_backups/defaults/main.yml @@ -1,7 +1,11 @@ --- + +atl_db_port: 5432 +atl_systemd_service_name: "{{ atl_product_edition }}.service" + atl_slingshot_yum_repo_url: 'https://packages.atlassian.com/atlassian-slingshot-rpm' atl_amazon_extras_postgresql_version: '9.6' atl_slingshot_deb_pubkey_url: 'https://packages.atlassian.com/api/gpg/key/public' atl_slingshot_deb_repo_url: 'https://packages.atlassian.com/atlassian-slingshot-deb' -atl_slingshot_deb_repo_components: 'stable main' \ No newline at end of file +atl_slingshot_deb_repo_components: 'stable main' diff --git a/roles/create_backups/tasks/main.yml b/roles/create_backups/tasks/main.yml index a20ae3f..45d1d34 100644 --- a/roles/create_backups/tasks/main.yml +++ b/roles/create_backups/tasks/main.yml @@ -1,5 +1,29 @@ --- + - name: Install distro-specific backup support packages include_tasks: "../../tasks/{{ ansible_distribution|lower }}.yml" - \ No newline at end of file +- name: Stop the {{ atl_product_edition }} service + service: + name: "{{ atl_systemd_service_name }}" + state: stopped + +- name: Run the slingshot backup + command: + argv: + - "atlassian-slingshot" + - "--backupDir {{ atl_slingshot_backup_dir }}" + - "--dbHost {{ atl_db_host }}" + - "--dbPort {{ atl_db_port }}" + - "--dbUser {{ atl_jdbc_user }}" + - "--dbName {{ atl_jdbc_db_name }}" + - "--s3BucketName {{ atl_s3_backup_bucket }}" + - "--awsRegion {{ atl_s3_backup_region }}" + - "--sharedHomeLocation {{ atl_product_home_shared }}" + environment: + ATL_DB_PASSWORD: "{{ atl_jdbc_password }}" + +- name: Restart the {{ atl_product_edition }} service + service: + name: "{{ atl_systemd_service_name }}" + state: restarted From bf2fcca44980802c851dd90bf5cb42fa9c7a4188 Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Fri, 3 Jan 2020 14:23:35 +1100 Subject: [PATCH 06/16] DCD-880: Add playbook to run the backup role --- aws_slingshot_backup.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 aws_slingshot_backup.yml diff --git a/aws_slingshot_backup.yml b/aws_slingshot_backup.yml new file mode 100644 index 0000000..d776712 --- /dev/null +++ b/aws_slingshot_backup.yml @@ -0,0 +1,3 @@ +- hosts: aws_node_local + roles: + - role: create_backups \ No newline at end of file From dc593f649bda4ca79f38b43059af00b3cf16f6de Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Fri, 3 Jan 2020 14:33:21 +1100 Subject: [PATCH 07/16] DCD-880: Fix path to distro specific tasks for slingshot backup role --- roles/create_backups/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/create_backups/tasks/main.yml b/roles/create_backups/tasks/main.yml index 45d1d34..bbac333 100644 --- a/roles/create_backups/tasks/main.yml +++ b/roles/create_backups/tasks/main.yml @@ -1,7 +1,7 @@ --- - name: Install distro-specific backup support packages - include_tasks: "../../tasks/{{ ansible_distribution|lower }}.yml" + include_tasks: "{{ ansible_distribution|lower }}.yml" - name: Stop the {{ atl_product_edition }} service service: From b6a8d70a7081ae21a9c1976834acffaf9b0b7cc0 Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Fri, 3 Jan 2020 14:47:03 +1100 Subject: [PATCH 08/16] DCD-880: Fix slingshot invocation --- roles/create_backups/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/create_backups/tasks/main.yml b/roles/create_backups/tasks/main.yml index bbac333..feffc88 100644 --- a/roles/create_backups/tasks/main.yml +++ b/roles/create_backups/tasks/main.yml @@ -12,6 +12,7 @@ command: argv: - "atlassian-slingshot" + - "backup" - "--backupDir {{ atl_slingshot_backup_dir }}" - "--dbHost {{ atl_db_host }}" - "--dbPort {{ atl_db_port }}" From a6b38cb21f5251076a58a1f5467df5a6c3bd0cb1 Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Fri, 3 Jan 2020 14:53:35 +1100 Subject: [PATCH 09/16] DCD-880: Fix arguments for slingshot backup --- roles/create_backups/tasks/main.yml | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/roles/create_backups/tasks/main.yml b/roles/create_backups/tasks/main.yml index feffc88..2d62295 100644 --- a/roles/create_backups/tasks/main.yml +++ b/roles/create_backups/tasks/main.yml @@ -13,14 +13,22 @@ argv: - "atlassian-slingshot" - "backup" - - "--backupDir {{ atl_slingshot_backup_dir }}" - - "--dbHost {{ atl_db_host }}" - - "--dbPort {{ atl_db_port }}" - - "--dbUser {{ atl_jdbc_user }}" - - "--dbName {{ atl_jdbc_db_name }}" - - "--s3BucketName {{ atl_s3_backup_bucket }}" - - "--awsRegion {{ atl_s3_backup_region }}" - - "--sharedHomeLocation {{ atl_product_home_shared }}" + - "--backupDir" + - "{{ atl_slingshot_backup_dir }}" + - "--dbHost" + - "{{ atl_db_host }}" + - "--dbPort" + - "{{ atl_db_port }}" + - "--dbUser" + - "{{ atl_jdbc_user }}" + - "--dbName" + - "{{ atl_jdbc_db_name }}" + - "--s3BucketName" + - "{{ atl_s3_backup_bucket }}" + - "--awsRegion" + - "{{ atl_s3_backup_region }}" + - "--sharedHomeLocation" + - "{{ atl_product_home_shared }}" environment: ATL_DB_PASSWORD: "{{ atl_jdbc_password }}" From f3d5e19674df36c888a83fad8616de052e519eb8 Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Fri, 3 Jan 2020 15:18:13 +1100 Subject: [PATCH 10/16] DCD-880: Add documentation to backup playbook --- aws_slingshot_backup.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/aws_slingshot_backup.yml b/aws_slingshot_backup.yml index d776712..57b9690 100644 --- a/aws_slingshot_backup.yml +++ b/aws_slingshot_backup.yml @@ -1,3 +1,14 @@ +# Runs atlassian-slingshot to backup your product instance. Should be run when there is only 1 app node (i.e server or single node cluster) +# Params +# Required +# atl_product_edition - one of "confluence", "jira", or "bitbucket". +# atl_product_family - should be the same as the "atl_product_edition" +# atl_slingshot_backup_dir - working directory for the backup, should be large enough to fit database and shared home dumps +# atl_s3_backup_bucket - s3 bucket to upload your backup to +# atl_s3_backup_region - AWS region where the s3 bucket is deployed +# +# There are other variables which can be overriden. See the tasks for details. The will be defaulted by the role defaults or the inventory group_vars + - hosts: aws_node_local roles: - role: create_backups \ No newline at end of file From 8ba60b6586a3c07ce06c9fbbfbb969fa13a78592 Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Fri, 3 Jan 2020 15:56:06 +1100 Subject: [PATCH 11/16] DCD-880: Add option to override ansible playbook params at invocation --- bin/ansible-with-atl-env | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/ansible-with-atl-env b/bin/ansible-with-atl-env index 3685381..e021bcb 100755 --- a/bin/ansible-with-atl-env +++ b/bin/ansible-with-atl-env @@ -20,6 +20,7 @@ set +a pipenv run \ ansible-playbook -v \ $ATL_DEPLOYMENT_REPOSITORY_CUSTOM_PARAMS \ + -e "${PLAYBOOK_INVOCATION_EXTRA_PARAMS}" \ -i $INV \ $PLAYBOOK \ 2>&1 | tee --append $LOG_FILE From 054e5eff59e4266c373752f46f93521ef26134ee Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Fri, 3 Jan 2020 16:12:18 +1100 Subject: [PATCH 12/16] DCD-880: Create SSM document to run slingshot --- ssm-documents/ATL-RunAutomationPlaybook.json | 30 ++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 ssm-documents/ATL-RunAutomationPlaybook.json diff --git a/ssm-documents/ATL-RunAutomationPlaybook.json b/ssm-documents/ATL-RunAutomationPlaybook.json new file mode 100644 index 0000000..11ea298 --- /dev/null +++ b/ssm-documents/ATL-RunAutomationPlaybook.json @@ -0,0 +1,30 @@ +{ + "schemaVersion": "2.2", + "description": "Use this document to run Ansible Playbooks from the atlassian deployment automation repo", + "parameters": { + "Playbook": { + "type": "String", + "description": "The Playbook file to run, relative to the repository root." + }, + "InvocationVariables": { + "type": "String", + "description": "Variables to apply to the playbook. See playbook documentation for details on required parameters.", + "displayType": "textarea", + "allowedPattern": "^$|^\\w+\\=[^\\s|:();&]+(\\s\\w+\\=[^\\s|:();&]+)*$" + } + }, + "mainSteps": [ + { + "action": "aws:runShellScript", + "name": "runShellScript", + "inputs": { + "runCommand": [ + "#!/bin/bash", + "cd /opt/atlassian/dc-deployments-automation", + "export PLAYBOOK_INVOCATION_EXTRA_PARAMS='{{InvocationVariables}}'", + "./bin/ansible-with-atl-env ./inv/aws_node_local aws_slingshot_backup.yml /var/log/ansible-slingshot.log" + ] + } + } + ] +} \ No newline at end of file From 4d83d113860119332fd5bea8c1816ecf19a72536 Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Mon, 6 Jan 2020 17:08:47 +1100 Subject: [PATCH 13/16] DCD-880: Fix python interpreter for amazon linux extras --- roles/create_backups/tasks/amazon.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/create_backups/tasks/amazon.yml b/roles/create_backups/tasks/amazon.yml index 796ef30..7292a8a 100644 --- a/roles/create_backups/tasks/amazon.yml +++ b/roles/create_backups/tasks/amazon.yml @@ -17,3 +17,5 @@ command: amazon-linux-extras install -y "postgresql{{ atl_amazon_extras_postgresql_version }}" args: creates: /usr/bin/psql + environment: + PYTHON: /bin/python From 01a0b289ed0b8c3fa5165c9a790d8d09b9ac653d Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Wed, 8 Jan 2020 11:06:29 +1100 Subject: [PATCH 14/16] DCD-880: Add execution timeout to backup ssm playbook --- ssm-documents/ATL-RunAutomationPlaybook.json | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ssm-documents/ATL-RunAutomationPlaybook.json b/ssm-documents/ATL-RunAutomationPlaybook.json index 11ea298..6e4e2c5 100644 --- a/ssm-documents/ATL-RunAutomationPlaybook.json +++ b/ssm-documents/ATL-RunAutomationPlaybook.json @@ -11,6 +11,12 @@ "description": "Variables to apply to the playbook. See playbook documentation for details on required parameters.", "displayType": "textarea", "allowedPattern": "^$|^\\w+\\=[^\\s|:();&]+(\\s\\w+\\=[^\\s|:();&]+)*$" + }, + "executionTimeout": { + "type": "String", + "default": "14400", + "description": "(Optional) The time in seconds for a command to complete before it is considered to have failed. Default is 3600 (1 hour). Maximum is 172800 (48 hours).", + "allowedPattern": "([1-9][0-9]{0,4})|(1[0-6][0-9]{4})|(17[0-1][0-9]{3})|(172[0-7][0-9]{2})|(172800)" } }, "mainSteps": [ @@ -23,7 +29,9 @@ "cd /opt/atlassian/dc-deployments-automation", "export PLAYBOOK_INVOCATION_EXTRA_PARAMS='{{InvocationVariables}}'", "./bin/ansible-with-atl-env ./inv/aws_node_local aws_slingshot_backup.yml /var/log/ansible-slingshot.log" - ] + ], + "workingDirectory": "{{ /opt/atlassian/dc-deployments-automation }}", + "timeoutSeconds": "{{ executionTimeout }}" } } ] From c65394b60c389f2f642fceaeba7c1dbd41f5ee7f Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Wed, 8 Jan 2020 15:39:37 +1100 Subject: [PATCH 15/16] DCD-880: Remove ssm playbook from automation repo - it will go into services quick start --- ssm-documents/ATL-RunAutomationPlaybook.json | 38 -------------------- 1 file changed, 38 deletions(-) delete mode 100644 ssm-documents/ATL-RunAutomationPlaybook.json diff --git a/ssm-documents/ATL-RunAutomationPlaybook.json b/ssm-documents/ATL-RunAutomationPlaybook.json deleted file mode 100644 index 6e4e2c5..0000000 --- a/ssm-documents/ATL-RunAutomationPlaybook.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "schemaVersion": "2.2", - "description": "Use this document to run Ansible Playbooks from the atlassian deployment automation repo", - "parameters": { - "Playbook": { - "type": "String", - "description": "The Playbook file to run, relative to the repository root." - }, - "InvocationVariables": { - "type": "String", - "description": "Variables to apply to the playbook. See playbook documentation for details on required parameters.", - "displayType": "textarea", - "allowedPattern": "^$|^\\w+\\=[^\\s|:();&]+(\\s\\w+\\=[^\\s|:();&]+)*$" - }, - "executionTimeout": { - "type": "String", - "default": "14400", - "description": "(Optional) The time in seconds for a command to complete before it is considered to have failed. Default is 3600 (1 hour). Maximum is 172800 (48 hours).", - "allowedPattern": "([1-9][0-9]{0,4})|(1[0-6][0-9]{4})|(17[0-1][0-9]{3})|(172[0-7][0-9]{2})|(172800)" - } - }, - "mainSteps": [ - { - "action": "aws:runShellScript", - "name": "runShellScript", - "inputs": { - "runCommand": [ - "#!/bin/bash", - "cd /opt/atlassian/dc-deployments-automation", - "export PLAYBOOK_INVOCATION_EXTRA_PARAMS='{{InvocationVariables}}'", - "./bin/ansible-with-atl-env ./inv/aws_node_local aws_slingshot_backup.yml /var/log/ansible-slingshot.log" - ], - "workingDirectory": "{{ /opt/atlassian/dc-deployments-automation }}", - "timeoutSeconds": "{{ executionTimeout }}" - } - } - ] -} \ No newline at end of file From 900f028a39d384c9e28a7528a549d73a63686927 Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Wed, 8 Jan 2020 15:42:40 +1100 Subject: [PATCH 16/16] DCD-880: Fix number of pipeline batches --- bitbucket-pipelines.yml | 76 +++++++++++++++++++++++------------------ 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 5e77c1b..10e3829 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -18,7 +18,7 @@ pipelines: - step: name: Pre Parallelization stage script: - - echo "Running tests in 33 batches" + - echo "Running tests in 34 batches" - step: name: Check if number of batches match actual number of scenarios script: @@ -38,7 +38,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 1 - + - step: name: Molecule Test Batch - 2 services: @@ -46,7 +46,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 2 - + - step: name: Molecule Test Batch - 3 services: @@ -54,7 +54,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 3 - + - step: name: Molecule Test Batch - 4 services: @@ -62,7 +62,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 4 - + - step: name: Molecule Test Batch - 5 services: @@ -70,7 +70,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 5 - + - step: name: Molecule Test Batch - 6 services: @@ -78,7 +78,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 6 - + - step: name: Molecule Test Batch - 7 services: @@ -86,7 +86,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 7 - + - step: name: Molecule Test Batch - 8 services: @@ -94,7 +94,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 8 - + - step: name: Molecule Test Batch - 9 services: @@ -102,7 +102,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 9 - + - step: name: Molecule Test Batch - 10 services: @@ -110,7 +110,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 10 - + - step: name: Molecule Test Batch - 11 services: @@ -118,7 +118,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 11 - + - step: name: Molecule Test Batch - 12 services: @@ -126,7 +126,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 12 - + - step: name: Molecule Test Batch - 13 services: @@ -134,7 +134,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 13 - + - step: name: Molecule Test Batch - 14 services: @@ -142,7 +142,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 14 - + - step: name: Molecule Test Batch - 15 services: @@ -150,7 +150,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 15 - + - step: name: Molecule Test Batch - 16 services: @@ -158,7 +158,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 16 - + - step: name: Molecule Test Batch - 17 services: @@ -166,7 +166,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 17 - + - step: name: Molecule Test Batch - 18 services: @@ -174,7 +174,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 18 - + - step: name: Molecule Test Batch - 19 services: @@ -182,7 +182,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 19 - + - step: name: Molecule Test Batch - 20 services: @@ -190,7 +190,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 20 - + - step: name: Molecule Test Batch - 21 services: @@ -198,7 +198,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 21 - + - step: name: Molecule Test Batch - 22 services: @@ -206,7 +206,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 22 - + - step: name: Molecule Test Batch - 23 services: @@ -214,7 +214,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 23 - + - step: name: Molecule Test Batch - 24 services: @@ -222,7 +222,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 24 - + - step: name: Molecule Test Batch - 25 services: @@ -230,7 +230,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 25 - + - step: name: Molecule Test Batch - 26 services: @@ -238,7 +238,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 26 - + - step: name: Molecule Test Batch - 27 services: @@ -246,7 +246,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 27 - + - step: name: Molecule Test Batch - 28 services: @@ -254,7 +254,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 28 - + - step: name: Molecule Test Batch - 29 services: @@ -262,7 +262,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 29 - + - step: name: Molecule Test Batch - 30 services: @@ -270,7 +270,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 30 - + - step: name: Molecule Test Batch - 31 services: @@ -278,7 +278,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 31 - + - step: name: Molecule Test Batch - 32 services: @@ -286,7 +286,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 32 - + - step: name: Molecule Test Batch - 33 services: @@ -294,3 +294,13 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 33 + + - step: + name: Molecule Test Batch - 34 + services: + - docker + script: + - apt-get update && ./bin/install-ansible --dev + - ./bin/run-tests-in-batches --batch 34 + +