diff --git a/README.md b/README.md index 864b060..72136e1 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,10 @@ them in the `Custom command-line parameters for Ansible` field: -e atl_product_download_url=http://s3.amazon.com/atlassian/jira-9.0.0-PRE-TEST.tar.gz -e atl_use_system_jdk=true -e atl_download_format=tarball +## Reporting issues + +If you find any bugs in this repository, or have feature requests or use cases +for us, please raise them in our [public Jira project](https://jira.atlassian.com/projects/SCALE/summary). ## Development diff --git a/aws_jira_dc_node.yml b/aws_jira_dc_node.yml index 41b7be6..be81d83 100644 --- a/aws_jira_dc_node.yml +++ b/aws_jira_dc_node.yml @@ -7,6 +7,7 @@ atl_product_family: "jira" atl_product_user: "jira" atl_product_edition: "jira-{{ lookup('env', 'ATL_PRODUCT_EDITION') | lower }}" + atl_install_jsd_as_obr: "{{ lookup('env', 'ATL_JSD_ASOBR') }}" atl_systemd_service_name: "jira.service" atl_startup_systemd_params: @@ -24,5 +25,6 @@ - role: product_common - role: product_install - role: database_init + - role: restore_backups - role: jira_config - role: product_startup diff --git a/bin/install-ansible b/bin/install-ansible index b515cdf..196562e 100755 --- a/bin/install-ansible +++ b/bin/install-ansible @@ -6,8 +6,12 @@ source /etc/os-release if [[ $ID = "amzn" ]]; then yum install -y \ python3-devel \ - python3-pip + python3-pip \ + python2-boto3 \ + python2-botocore + else + # FIXME: Currently assumes Debian-based apt-get update && \ apt-get install -y \ python3-dev \ diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index ec993c6..b51f0f9 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -14,7 +14,7 @@ pipelines: - step: name: Pre Parallelization stage script: - - echo "Running tests in 28 batches" + - echo "Running tests in 32 batches" - step: name: Check if number of batches match actual number of scenarios script: @@ -34,7 +34,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 1 - + - step: name: Molecule Test Batch - 2 services: @@ -42,7 +42,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 2 - + - step: name: Molecule Test Batch - 3 services: @@ -50,7 +50,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 3 - + - step: name: Molecule Test Batch - 4 services: @@ -58,7 +58,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 4 - + - step: name: Molecule Test Batch - 5 services: @@ -66,7 +66,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 5 - + - step: name: Molecule Test Batch - 6 services: @@ -74,7 +74,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 6 - + - step: name: Molecule Test Batch - 7 services: @@ -82,7 +82,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 7 - + - step: name: Molecule Test Batch - 8 services: @@ -90,7 +90,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 8 - + - step: name: Molecule Test Batch - 9 services: @@ -98,7 +98,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 9 - + - step: name: Molecule Test Batch - 10 services: @@ -106,7 +106,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 10 - + - step: name: Molecule Test Batch - 11 services: @@ -114,7 +114,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 11 - + - step: name: Molecule Test Batch - 12 services: @@ -122,7 +122,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 12 - + - step: name: Molecule Test Batch - 13 services: @@ -130,7 +130,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 13 - + - step: name: Molecule Test Batch - 14 services: @@ -138,7 +138,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 14 - + - step: name: Molecule Test Batch - 15 services: @@ -146,7 +146,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 15 - + - step: name: Molecule Test Batch - 16 services: @@ -154,7 +154,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 16 - + - step: name: Molecule Test Batch - 17 services: @@ -162,7 +162,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 17 - + - step: name: Molecule Test Batch - 18 services: @@ -170,7 +170,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 18 - + - step: name: Molecule Test Batch - 19 services: @@ -178,7 +178,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 19 - + - step: name: Molecule Test Batch - 20 services: @@ -186,7 +186,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 20 - + - step: name: Molecule Test Batch - 21 services: @@ -194,7 +194,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 21 - + - step: name: Molecule Test Batch - 22 services: @@ -202,7 +202,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 22 - + - step: name: Molecule Test Batch - 23 services: @@ -210,7 +210,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 23 - + - step: name: Molecule Test Batch - 24 services: @@ -218,7 +218,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 24 - + - step: name: Molecule Test Batch - 25 services: @@ -226,7 +226,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 25 - + - step: name: Molecule Test Batch - 26 services: @@ -234,7 +234,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 26 - + - step: name: Molecule Test Batch - 27 services: @@ -242,7 +242,7 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 27 - + - step: name: Molecule Test Batch - 28 services: @@ -250,5 +250,37 @@ pipelines: script: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 28 - - + + - step: + name: Molecule Test Batch - 29 + services: + - docker + script: + - apt-get update && ./bin/install-ansible --dev + - ./bin/run-tests-in-batches --batch 29 + + - step: + name: Molecule Test Batch - 30 + services: + - docker + script: + - apt-get update && ./bin/install-ansible --dev + - ./bin/run-tests-in-batches --batch 30 + + - step: + name: Molecule Test Batch - 31 + services: + - docker + script: + - apt-get update && ./bin/install-ansible --dev + - ./bin/run-tests-in-batches --batch 31 + + - step: + name: Molecule Test Batch - 32 + services: + - docker + script: + - apt-get update && ./bin/install-ansible --dev + - ./bin/run-tests-in-batches --batch 32 + + diff --git a/group_vars/aws_node_local.yml b/group_vars/aws_node_local.yml index aff781d..bb380b9 100644 --- a/group_vars/aws_node_local.yml +++ b/group_vars/aws_node_local.yml @@ -132,3 +132,4 @@ atl_rds_subnet_group_name: "{{ lookup('env', 'ATL_RDS_SUBNET_GROUP_NAME') }}" atl_rds_security_group: "{{ lookup('env', 'ATL_RDS_SECURITY_GROUP') }}" atl_backup_manifest_url: "{{ lookup('env', 'ATL_BACKUP_MANIFEST_URL') }}" +atl_restore_required: "{{ atl_backup_manifest_url is defined and atl_backup_manifest_url != '' }}" diff --git a/roles/crowd_config/tasks/main.yml b/roles/crowd_config/tasks/main.yml index ef7dcc2..b5b987c 100644 --- a/roles/crowd_config/tasks/main.yml +++ b/roles/crowd_config/tasks/main.yml @@ -5,16 +5,19 @@ src: server.xml.j2 dest: "{{ atl_product_installation_versioned }}/apache-tomcat/conf/server.xml" -- name: Override JVM memory settings. - # Ugly but necessary as the product installs this file so we need to make the change here. +- name: Set the minimum heap size (Xms) lineinfile: path: "{{ atl_product_installation_versioned }}/apache-tomcat/bin/setenv.sh" - backrefs: true - regexp: "^{{ item }}=" - line: "{{ item }}=\"{{ atl_jvm_heap }}\"" - with_items: - - 'JVM_MINIMUM_MEMORY' - - 'JVM_MAXIMUM_MEMORY' + regexp: '^(.*)Xms(\d+\w)(\s.*)$' + line: '\1Xms{{ atl_jvm_heap }}\3' + backrefs: yes + +- name: Set the maxmimum heap size (Xmx) + lineinfile: + path: "{{ atl_product_installation_versioned }}/apache-tomcat/bin/setenv.sh" + regexp: '^(.*)Xmx(\d+\w)(\s.*)$' + line: '\1Xmx{{ atl_jvm_heap }}\3' + backrefs: yes - name: Set Crowd home directory in crowd-init.properties file lineinfile: @@ -27,7 +30,13 @@ insertafter: "EOF" line: 'export CATALINA_OPTS="${CATALINA_OPTS} {{ atl_catalina_opts }} {{ atl_catalina_opts_extra }}"' -- name: Set JAVA_HOME #FIXME +- name: Set the Crowd node name via CATALINA_OPTS + lineinfile: + path: "{{ atl_product_installation_versioned }}/apache-tomcat/bin/setenv.sh" + insertafter: "EOF" + line: export CATALINA_OPTS="${CATALINA_OPTS} -Dcluster.node.name={{ ansible_ec2_instance_id }}-{{ ansible_ec2_local_ipv4 }}" + +- name: Set JAVA_HOME lineinfile: path: "{{ atl_product_installation_versioned }}/apache-tomcat/bin/setenv.sh" insertafter: "EOF" diff --git a/roles/database_init/tasks/main.yml b/roles/database_init/tasks/main.yml index 99638f7..8827f99 100644 --- a/roles/database_init/tasks/main.yml +++ b/roles/database_init/tasks/main.yml @@ -1,40 +1,40 @@ --- -- name: Create application DB user - postgresql_user: - login_host: "{{ atl_db_host }}" - login_user: "{{ atl_db_root_user }}" - login_password: "{{ atl_db_root_password }}" - port: "{{ atl_db_port }}" - name: "{{ atl_jdbc_user }}" - password: "{{ atl_jdbc_password }}" - expires: 'infinity' - tags: - - new_only +- block: -- name: Update root privs for new user - postgresql_privs: - login_host: "{{ atl_db_host }}" - login_user: "{{ atl_db_root_user }}" - login_password: "{{ atl_db_root_password }}" - database: postgres - roles: "{{ atl_db_root_user }}" - objs: "{{ atl_jdbc_user }}" - type: group - tags: - - new_only + - name: Create application DB user + postgresql_user: + login_host: "{{ atl_db_host }}" + login_user: "{{ atl_db_root_user }}" + login_password: "{{ atl_db_root_password }}" + port: "{{ atl_db_port }}" + name: "{{ atl_jdbc_user }}" + password: "{{ atl_jdbc_password }}" + expires: 'infinity' + + - name: Update root privs for new user + postgresql_privs: + login_host: "{{ atl_db_host }}" + login_user: "{{ atl_db_root_user }}" + login_password: "{{ atl_db_root_password }}" + database: postgres + roles: "{{ atl_db_root_user }}" + objs: "{{ atl_jdbc_user }}" + type: group + + - name: Create new application database + postgresql_db: + login_host: "{{ atl_db_host }}" + login_user: "{{ atl_db_root_user }}" + login_password: "{{ atl_db_root_password }}" + port: "{{ atl_db_port }}" + name: "{{ atl_jdbc_db_name }}" + owner: "{{ atl_jdbc_user }}" + encoding: "{{ atl_jdbc_encoding }}" + lc_collate: "{{ atl_jdbc_collation }}" + lc_ctype: "{{ atl_jdbc_ctype }}" + template: "{{ atl_jdbc_template }}" + register: db_created -- name: Create application database - postgresql_db: - login_host: "{{ atl_db_host }}" - login_user: "{{ atl_db_root_user }}" - login_password: "{{ atl_db_root_password }}" - port: "{{ atl_db_port }}" - name: "{{ atl_jdbc_db_name }}" - owner: "{{ atl_jdbc_user }}" - encoding: "{{ atl_jdbc_encoding }}" - lc_collate: "{{ atl_jdbc_collation }}" - lc_ctype: "{{ atl_jdbc_ctype }}" - template: "{{ atl_jdbc_template }}" tags: - new_only diff --git a/roles/fetch_backups/tasks/main.yml b/roles/fetch_backups/tasks/main.yml deleted file mode 100644 index 45d7e1f..0000000 --- a/roles/fetch_backups/tasks/main.yml +++ /dev/null @@ -1,69 +0,0 @@ ---- - -# This role will attempt to fetch and load the backup manifest from a -# remote HTTP or S3 URL. On successful completion the contents of JSON -# or YAML document will be in the var `atl_backup_manifest`. - -- block: - - - name: Ensure temp directory is present - file: - path: "{{ atl_installer_temp }}" - state: directory - mode: 0750 - owner: "{{ atl_product_user }}" - group: "{{ atl_product_user }}" - changed_when: false # For Molecule idempotence check - - - name: Parse the manifest URL - set_fact: - atl_backup_manifest_url: "{{ atl_backup_manifest_url | urlsplit }}" - - - name: Extract manifest file information - set_fact: - atl_backup_manifest_bucket: "{{ atl_backup_manifest_url.hostname }}" - atl_backup_manifest_path: "{{ atl_backup_manifest_url.path }}" - atl_backup_manifest_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest_url.path | basename }}" - - - name: Fetch the manifest from S3 - aws_s3: - mode: get - bucket: "{{ atl_backup_manifest_bucket }}" - object: "{{ atl_backup_manifest_path }}" - dest: "{{ atl_backup_manifest_dest }}" - when: atl_backup_manifest_url.scheme == 's3' - - - name: Fetch the manifest from remote host - get_url: - url: "{{ atl_backup_manifest_url }}" - dest: "{{ atl_backup_manifest_dest }}" - when: atl_backup_manifest_url.scheme != 's3' - - - name: Load parameters from manifest - include_vars: - file: "{{ atl_backup_manifest_dest }}" - name: atl_backup_manifest - - - name: Define the DB and home dump destinations - set_fact: - atl_backup_db_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.db_dump | basename }}" - atl_backup_home_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.shared_home_dump | basename }}" - - # FIXME: Here we fetch the backups. However we may wish to stream - # these directly from S3 to the target DB/FS to avoid requiring - # disk-space for the intermediate files. - - name: Fetch DB backup from S3 - aws_s3: - mode: get - bucket: "{{ atl_backup_manifest.db_dump | urlsplit('hostname') }}" - object: "{{ atl_backup_manifest.db_dump | urlsplit('path') }}" - dest: "{{ atl_backup_db_dest }}" - - - name: Fetch Home backup from S3 - aws_s3: - mode: get - bucket: "{{ atl_backup_manifest.shared_home_dump | urlsplit('hostname') }}" - object: "{{ atl_backup_manifest.shared_home_dump | urlsplit('path') }}" - dest: "{{ atl_backup_home_dest }}" - - when: atl_backup_manifest_url is defined and atl_backup_manifest_url != '' diff --git a/roles/product_install/defaults/main.yml b/roles/product_install/defaults/main.yml index 211e76e..6a9940e 100644 --- a/roles/product_install/defaults/main.yml +++ b/roles/product_install/defaults/main.yml @@ -19,10 +19,16 @@ atl_product_base_url: "{{ atl_release_base_url }}/{{ atl_product_family }}/downl atl_product_download_url: "{{ atl_product_base_url }}/atlassian-{{ atl_download_edition | default(atl_product_edition) }}-{{ atl_product_version }}{{ atl_download_suffix }}" atl_product_download_filename: "{{ atl_download_edition | default(atl_product_edition) }}.{{ atl_product_version }}{{ atl_download_suffix }}" -atl_product_download: "{{ atl_installer_temp }}/{{ atl_product_download_filename }}" +atl_product_temp_download: "{{ atl_installer_temp }}/{{ atl_product_download_filename }}" atl_product_varfile: "{{ atl_installer_temp }}/{{ atl_product_family }}.varfile" +atl_product_home_shared_download_dir: "{{ atl_shared_mountpoint }}/downloads" +atl_product_home_shared_download: "{{ atl_product_home_shared_download_dir }}/{{ atl_product_download_filename }}" +atl_product_home_shared_moving_lock: "{{ atl_product_home_shared_download }}_moving" +atl_product_home_shared_completed_lock: "{{ atl_product_home_shared_download }}_completed" + atl_marketplace_base: "https://marketplace.atlassian.com" +atl_mpac_products: "https://marketplace.atlassian.com/rest/2/products" atl_servicedesk_latest_url: "https://marketplace.atlassian.com/rest/2/products/key/jira-servicedesk/versions/latest" atl_servicedesk_versioned_url: "https://marketplace.atlassian.com/rest/2/products/key/jira-servicedesk/versions/name/{{ atl_product_version }}" atl_servicedesk_url_map: diff --git a/roles/product_install/molecule/bitbucket_latest/tests/test_default.py b/roles/product_install/molecule/bitbucket_latest/tests/test_default.py index 55c71c6..0592cbf 100644 --- a/roles/product_install/molecule/bitbucket_latest/tests/test_default.py +++ b/roles/product_install/molecule/bitbucket_latest/tests/test_default.py @@ -37,6 +37,16 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/opt/atlassian/tmp/bitbucket.' + upstream + '-x64.bin') + installer = host.file('/media/atl/downloads/bitbucket.' + upstream + '-x64.bin') assert installer.exists assert installer.user == 'root' + +def test_completed_lockfile(host): + upstream_fd = urllib.request.urlopen( + "https://marketplace.atlassian.com/rest/2/applications/bitbucket/versions/latest") + upstream_json = json.load(upstream_fd) + upstream = upstream_json['version'] + + lockfile = host.file('/media/atl/downloads/bitbucket.' + upstream + '-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' diff --git a/roles/product_install/molecule/confluence_latest/tests/test_default.py b/roles/product_install/molecule/confluence_latest/tests/test_default.py index 47245a4..01044a5 100644 --- a/roles/product_install/molecule/confluence_latest/tests/test_default.py +++ b/roles/product_install/molecule/confluence_latest/tests/test_default.py @@ -35,6 +35,15 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/opt/atlassian/tmp/confluence.'+upstream+'-x64.bin') + installer = host.file('/media/atl/downloads/confluence.'+upstream+'-x64.bin') assert installer.exists assert installer.user == 'root' + +def test_completed_lockfile(host): + upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/confluence/versions/latest") + upstream_json = json.load(upstream_fd) + upstream = upstream_json['version'] + + lockfile = host.file('/media/atl/downloads/confluence.'+upstream+'-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' diff --git a/roles/product_install/molecule/crowd_latest/Dockerfile.j2 b/roles/product_install/molecule/crowd_latest/Dockerfile.j2 new file mode 100644 index 0000000..e6aa95d --- /dev/null +++ b/roles/product_install/molecule/crowd_latest/Dockerfile.j2 @@ -0,0 +1,14 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi diff --git a/roles/product_install/molecule/crowd_latest/molecule.yml b/roles/product_install/molecule/crowd_latest/molecule.yml new file mode 100644 index 0000000..7fd3163 --- /dev/null +++ b/roles/product_install/molecule/crowd_latest/molecule.yml @@ -0,0 +1,30 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: amazon_linux2 + image: amazonlinux:2 + groups: + - aws_node_local + - name: ubuntu_lts + image: ubuntu:bionic + groups: + - aws_node_local +provisioner: + name: ansible + options: + skip-tags: runtime_pkg + lint: + name: ansible-lint + inventory: + links: + group_vars: ../../../../group_vars/ +verifier: + name: testinfra + lint: + name: flake8 + enabled: false diff --git a/roles/product_install/molecule/crowd_latest/playbook.yml b/roles/product_install/molecule/crowd_latest/playbook.yml new file mode 100644 index 0000000..490514e --- /dev/null +++ b/roles/product_install/molecule/crowd_latest/playbook.yml @@ -0,0 +1,12 @@ +--- +- name: Converge + hosts: all + vars: + atl_product_family: "crowd" + atl_product_edition: "crowd" + atl_product_user: "crowd" + atl_download_format: "tarball" + roles: + - role: linux_common + - role: product_common + - role: product_install diff --git a/roles/product_install/molecule/crowd_latest/tests/test_default.py b/roles/product_install/molecule/crowd_latest/tests/test_default.py new file mode 100644 index 0000000..6e5dc7c --- /dev/null +++ b/roles/product_install/molecule/crowd_latest/tests/test_default.py @@ -0,0 +1,52 @@ +import os +from six.moves import urllib +import json + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + +def test_version_downloaded(host): + verfile = host.file('/media/atl/crowd/shared/crowd.version') + assert verfile.exists + +def test_symlink_created(host): + target = host.file('/opt/atlassian/crowd/current') + assert target.exists + assert target.is_symlink + +def test_unpacked(host): + verfile = host.file('/opt/atlassian/crowd/current/start_crowd.sh') + assert verfile.exists + +def test_version_file_is_latest(host): + verfile = host.file('/media/atl/crowd/shared/crowd.version') + assert verfile.exists + + upstream_fd = urllib.request.urlopen( + "https://marketplace.atlassian.com/rest/2/applications/crowd/versions/latest") + upstream_json = json.load(upstream_fd) + upstream = upstream_json['version'] + + assert verfile.content.decode("UTF-8").strip() == upstream.strip() + +def test_latest_is_downloaded(host): + upstream_fd = urllib.request.urlopen( + "https://marketplace.atlassian.com/rest/2/applications/crowd/versions/latest") + upstream_json = json.load(upstream_fd) + upstream = upstream_json['version'] + + installer = host.file('/media/atl/downloads/crowd.' + upstream + '.tar.gz') + assert installer.exists + assert installer.user == 'root' + +def test_completed_lockfile(host): + upstream_fd = urllib.request.urlopen( + "https://marketplace.atlassian.com/rest/2/applications/crowd/versions/latest") + upstream_json = json.load(upstream_fd) + upstream = upstream_json['version'] + + lockfile = host.file('/media/atl/downloads/crowd.' + upstream + '.tar.gz_completed') + assert lockfile.exists + assert lockfile.user == 'root' diff --git a/roles/product_install/molecule/default/tests/test_default.py b/roles/product_install/molecule/default/tests/test_default.py index 70839b9..8e20d76 100644 --- a/roles/product_install/molecule/default/tests/test_default.py +++ b/roles/product_install/molecule/default/tests/test_default.py @@ -23,6 +23,15 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/opt/atlassian/tmp/jira-core.'+upstream+'-x64.bin') + installer = host.file('/media/atl/downloads/jira-core.'+upstream+'-x64.bin') assert installer.exists assert installer.user == 'root' + +def test_completed_lockfile(host): + upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/jira/versions/latest") + upstream_json = json.load(upstream_fd) + upstream = upstream_json['version'] + + lockfile = host.file('/media/atl/downloads/jira-core.'+upstream+'-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' \ No newline at end of file diff --git a/roles/product_install/molecule/jira_all/Dockerfile.j2 b/roles/product_install/molecule/jira_all/Dockerfile.j2 new file mode 100644 index 0000000..e6aa95d --- /dev/null +++ b/roles/product_install/molecule/jira_all/Dockerfile.j2 @@ -0,0 +1,14 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi diff --git a/roles/product_install/molecule/jira_all/molecule.yml b/roles/product_install/molecule/jira_all/molecule.yml new file mode 100644 index 0000000..7fd3163 --- /dev/null +++ b/roles/product_install/molecule/jira_all/molecule.yml @@ -0,0 +1,30 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: amazon_linux2 + image: amazonlinux:2 + groups: + - aws_node_local + - name: ubuntu_lts + image: ubuntu:bionic + groups: + - aws_node_local +provisioner: + name: ansible + options: + skip-tags: runtime_pkg + lint: + name: ansible-lint + inventory: + links: + group_vars: ../../../../group_vars/ +verifier: + name: testinfra + lint: + name: flake8 + enabled: false diff --git a/roles/product_install/molecule/jira_all/playbook.yml b/roles/product_install/molecule/jira_all/playbook.yml new file mode 100644 index 0000000..9bddc9a --- /dev/null +++ b/roles/product_install/molecule/jira_all/playbook.yml @@ -0,0 +1,28 @@ +--- +- name: Converge + hosts: all + vars: + atl_product_family: "jira" + atl_product_edition: "jira-software" + atl_product_user: "jira" + atl_product_version: "7.13.1" + atl_install_jsd_as_obr: true + atl_systemd_service_name: "jira.service" + atl_jdbc_encoding: 'UNICODE' + atl_jdbc_collation: 'C' + atl_jdbc_ctype: 'C' + atl_jdbc_template: 'template0' + pre_tasks: + - name: Create cache dir + file: + path: '/media/atl/jira/shared/' + state: directory + - name: Seed version + copy: + dest: '/media/atl/jira/shared/jira-core.version' + content: "7.13.1" + force: false # For idempotency check + roles: + - role: linux_common + - role: product_common + - role: product_install diff --git a/roles/product_install/molecule/jira_all/tests/test_default.py b/roles/product_install/molecule/jira_all/tests/test_default.py new file mode 100644 index 0000000..b9522ec --- /dev/null +++ b/roles/product_install/molecule/jira_all/tests/test_default.py @@ -0,0 +1,48 @@ +import os +from six.moves import urllib + +import testinfra.utils.ansible_runner +import json + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +def test_version_is_correct(host): + verfile = host.file('/media/atl/jira/shared/jira-software.version') + assert verfile.exists + + assert verfile.content.decode("UTF-8").strip() == "7.13.1" + +def test_is_downloaded(host): + installer = host.file('/media/atl/downloads/jira-software.7.13.1-x64.bin') + assert installer.exists + assert installer.user == 'root' + +def test_completed_lockfile(host): + lockfile = host.file('/media/atl/downloads/jira-software.7.13.1-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' + +def test_is_unpacked(host): + installer = host.file('/opt/atlassian/jira-software/7.13.1/atlassian-jira/') + assert installer.exists + assert installer.is_directory + assert installer.user == 'jira' + assert installer.mode == 0o0755 + +def test_obr_is_downloaded(host): + installer = host.file('/media/atl/downloads/jira-servicedesk-application-3.16.1.obr') + assert installer.exists + assert installer.user == 'root' + +def test_obr_completed_lockfile(host): + lockfile = host.file('/media/atl/downloads/jira-servicedesk-application-3.16.1.obr_completed') + assert lockfile.exists + assert lockfile.user == 'root' + +def test_obr_is_unpacked(host): + jsdjar = host.file('/media/atl/jira/shared/plugins/installed-plugins/jira-servicedesk-application-3.16.1.jar') + assert jsdjar.exists + assert jsdjar.user == 'jira' + assert jsdjar.mode == 0o0750 \ No newline at end of file diff --git a/roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py b/roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py index 05a6bb3..725ace4 100644 --- a/roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py +++ b/roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py @@ -14,10 +14,15 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "7.10.2" def test_is_downloaded(host): - installer = host.file('/opt/atlassian/tmp/jira-core.7.10.2-x64.bin') + installer = host.file('/media/atl/downloads/jira-core.7.10.2-x64.bin') assert installer.exists assert installer.user == 'root' +def test_completed_lockfile(host): + lockfile = host.file('/media/atl/downloads/jira-core.7.10.2-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' + def test_is_unpacked(host): installer = host.file('/opt/atlassian/jira-core/7.10.2/atlassian-jira/') assert installer.exists diff --git a/roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py b/roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py index ead6adf..6b215f2 100644 --- a/roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py +++ b/roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py @@ -14,10 +14,15 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "7.10.1" def test_is_downloaded(host): - installer = host.file('/opt/atlassian/tmp/jira-core.7.10.1-x64.bin') + installer = host.file('/media/atl/downloads/jira-core.7.10.1-x64.bin') assert installer.exists assert installer.user == 'root' +def test_completed_lockfile(host): + lockfile = host.file('/media/atl/downloads/jira-core.7.10.1-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' + def test_is_unpacked(host): installer = host.file('/opt/atlassian/jira-core/7.10.1/atlassian-jira/') assert installer.exists diff --git a/roles/product_install/molecule/jira_software_latest/tests/test_default.py b/roles/product_install/molecule/jira_software_latest/tests/test_default.py index 63451c8..80d53b8 100644 --- a/roles/product_install/molecule/jira_software_latest/tests/test_default.py +++ b/roles/product_install/molecule/jira_software_latest/tests/test_default.py @@ -35,6 +35,15 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/opt/atlassian/tmp/jira-software.'+upstream+'-x64.bin') + installer = host.file('/media/atl/downloads/jira-software.'+upstream+'-x64.bin') assert installer.exists assert installer.user == 'root' + +def test_completed_lockfile(host): + upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/jira/versions/latest") + upstream_json = json.load(upstream_fd) + upstream = upstream_json['version'] + + lockfile = host.file('/media/atl/downloads/jira-software.'+upstream+'-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' \ No newline at end of file diff --git a/roles/product_install/molecule/jira_tarball/tests/test_default.py b/roles/product_install/molecule/jira_tarball/tests/test_default.py index 2f5d09b..7155f41 100644 --- a/roles/product_install/molecule/jira_tarball/tests/test_default.py +++ b/roles/product_install/molecule/jira_tarball/tests/test_default.py @@ -23,6 +23,15 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/opt/atlassian/tmp/jira-core.'+upstream+'.tar.gz') + installer = host.file('/media/atl/downloads/jira-core.'+upstream+'.tar.gz') assert installer.exists assert installer.user == 'root' + +def test_completed_lockfile(host): + upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/jira/versions/latest") + upstream_json = json.load(upstream_fd) + upstream = upstream_json['version'] + + lockfile = host.file('/media/atl/downloads/jira-core.'+upstream+'.tar.gz_completed') + assert lockfile.exists + assert lockfile.user == 'root' \ No newline at end of file diff --git a/roles/product_install/molecule/jira_version_from_file/tests/test_default.py b/roles/product_install/molecule/jira_version_from_file/tests/test_default.py index 5f00577..589e7ce 100644 --- a/roles/product_install/molecule/jira_version_from_file/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_from_file/tests/test_default.py @@ -14,10 +14,15 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "7.9.0" def test_is_downloaded(host): - installer = host.file('/opt/atlassian/tmp/jira-core.7.9.0-x64.bin') + installer = host.file('/media/atl/downloads/jira-core.7.9.0-x64.bin') assert installer.exists assert installer.user == 'root' +def test_completed_lockfile(host): + lockfile = host.file('/media/atl/downloads/jira-core.7.9.0-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' + def test_is_unpacked(host): installer = host.file('/opt/atlassian/jira-core/7.9.0/atlassian-jira/') assert installer.exists diff --git a/roles/product_install/molecule/jira_version_latest/tests/test_default.py b/roles/product_install/molecule/jira_version_latest/tests/test_default.py index 70839b9..8e20d76 100644 --- a/roles/product_install/molecule/jira_version_latest/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_latest/tests/test_default.py @@ -23,6 +23,15 @@ def test_latest_is_downloaded(host): upstream_json = json.load(upstream_fd) upstream = upstream_json['version'] - installer = host.file('/opt/atlassian/tmp/jira-core.'+upstream+'-x64.bin') + installer = host.file('/media/atl/downloads/jira-core.'+upstream+'-x64.bin') assert installer.exists assert installer.user == 'root' + +def test_completed_lockfile(host): + upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/jira/versions/latest") + upstream_json = json.load(upstream_fd) + upstream = upstream_json['version'] + + lockfile = host.file('/media/atl/downloads/jira-core.'+upstream+'-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' \ No newline at end of file diff --git a/roles/product_install/molecule/jira_version_override/tests/test_default.py b/roles/product_install/molecule/jira_version_override/tests/test_default.py index 8b5c7a4..e4fa67d 100644 --- a/roles/product_install/molecule/jira_version_override/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_override/tests/test_default.py @@ -14,10 +14,15 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "7.13.2" def test_is_downloaded(host): - installer = host.file('/opt/atlassian/tmp/jira-core.7.13.2-x64.bin') + installer = host.file('/media/atl/downloads/jira-core.7.13.2-x64.bin') assert installer.exists assert installer.user == 'root' +def test_completed_lockfile(host): + lockfile = host.file('/media/atl/downloads/jira-core.7.13.2-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' + def test_is_unpacked(host): installer = host.file('/opt/atlassian/jira-core/7.13.2') assert installer.exists diff --git a/roles/product_install/molecule/servicedesk3/tests/test_default.py b/roles/product_install/molecule/servicedesk3/tests/test_default.py index 24afb62..ae78b3d 100644 --- a/roles/product_install/molecule/servicedesk3/tests/test_default.py +++ b/roles/product_install/molecule/servicedesk3/tests/test_default.py @@ -14,10 +14,15 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "3.9.0" def test_is_downloaded(host): - installer = host.file('/opt/atlassian/tmp/servicedesk.3.9.0-x64.bin') + installer = host.file('/media/atl/downloads/servicedesk.3.9.0-x64.bin') assert installer.exists assert installer.user == 'root' +def test_completed_lockfile(host): + lockfile = host.file('/media/atl/downloads/servicedesk.3.9.0-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' + def test_is_unpacked(host): installer = host.file('/opt/atlassian/jira-servicedesk/3.9.0') assert installer.exists diff --git a/roles/product_install/molecule/servicedesk4/tests/test_default.py b/roles/product_install/molecule/servicedesk4/tests/test_default.py index b660f23..bc4814e 100644 --- a/roles/product_install/molecule/servicedesk4/tests/test_default.py +++ b/roles/product_install/molecule/servicedesk4/tests/test_default.py @@ -14,10 +14,15 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == "4.1.0" def test_is_downloaded(host): - installer = host.file('/opt/atlassian/tmp/servicedesk.4.1.0-x64.bin') + installer = host.file('/media/atl/downloads/servicedesk.4.1.0-x64.bin') assert installer.exists assert installer.user == 'root' +def test_completed_lockfile(host): + lockfile = host.file('/media/atl/downloads/servicedesk.4.1.0-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' + def test_is_unpacked(host): installer = host.file('/opt/atlassian/jira-servicedesk/4.1.0') assert installer.exists diff --git a/roles/product_install/molecule/servicedesk_latest/tests/test_default.py b/roles/product_install/molecule/servicedesk_latest/tests/test_default.py index 2190295..bd4eea8 100644 --- a/roles/product_install/molecule/servicedesk_latest/tests/test_default.py +++ b/roles/product_install/molecule/servicedesk_latest/tests/test_default.py @@ -23,10 +23,15 @@ def test_version_is_correct(host): assert verfile.content.decode("UTF-8").strip() == sd def test_is_downloaded(host): - installer = host.file('/opt/atlassian/tmp/servicedesk.'+sd+'-x64.bin') + installer = host.file('/media/atl/downloads/servicedesk.'+sd+'-x64.bin') assert installer.exists assert installer.user == 'root' +def test_completed_lockfile(host): + lockfile = host.file('/media/atl/downloads/servicedesk.'+sd+'-x64.bin_completed') + assert lockfile.exists + assert lockfile.user == 'root' + def test_is_unpacked(host): installer = host.file('/opt/atlassian/jira-servicedesk/'+sd) assert installer.exists diff --git a/roles/product_install/tasks/jira-all_extra_tasks.yml b/roles/product_install/tasks/jira-all_extra_tasks.yml new file mode 120000 index 0000000..55832eb --- /dev/null +++ b/roles/product_install/tasks/jira-all_extra_tasks.yml @@ -0,0 +1 @@ +no_op.yml \ No newline at end of file diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml new file mode 100644 index 0000000..e602765 --- /dev/null +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -0,0 +1,174 @@ +--- + +- name: Get the installer product version info + uri: + url: "{{ atl_mpac_products }}/key/jira-software/versions/name/{{ atl_product_version }}" + return_content: yes + register: atl_product_version_info + +- name: Show the returned build number + debug: + msg="buildNumber={{ atl_product_version_info.json.buildNumber }}" + +- name: Get the JSD build version info + uri: + url: "{{ atl_mpac_products }}/key/jira-servicedesk/versions/latest?application=\ + jira&applicationBuild={{ atl_product_version_info.json.buildNumber }}" + return_content: yes + register: atl_jsd_build_info + +- name: Show the returned obr binary href + debug: + msg="obr_ref={{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}" + +- name: how about getting the obr filename + debug: + msg="obr_name=jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr" + +- name: is shared_home set ? + debug: + msg="atl_product_home_shared_download_dir={{ atl_product_home_shared_download_dir }}" +# For the first run a temp obr should be downloaded but moved to +# shared home to ensure all subsequent nodes have access +# to the same specific version binary. +# To prevent a race condition with multiple downloads at the same time +# a directory is used as a lockfile (atomic operation) when moving obr. + +- name: Set assumptions to avoid race condition + set_fact: + download_obr: true + move_obr: false + atl_obr_download_href: "{{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}" + atl_obr_filename: "jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr" + atl_obr_download: "{{ atl_installer_temp }}/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr" + atl_obr_shared_download: "{{ atl_product_home_shared_download_dir }}/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr" + atl_obr_moving_lock: "{{ atl_product_home_shared_download_dir }}/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr_moving" + atl_obr_completed_lock: "{{ atl_product_home_shared_download_dir }}/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr_completed" + +# Check for pre-downloaded obr on shared_home and completed lock dir. +- name: Check for completed lock directory + stat: + path: "{{ atl_obr_completed_lock }}" + register: completed_lock + +- name: Check for obr in home_shared + stat: + path: "{{ atl_obr_shared_download }}" + register: home_shared_download + +# If obr exists and lockdir exists use this obr instead +- name: Check lock directory and obr exists on shared_home + set_fact: + download_obr: false + atl_obr_download: "{{ atl_obr_shared_download }}" + when: + - home_shared_download.stat.exists + - completed_lock.stat.isdir is defined + - completed_lock.stat.isdir + +# Fetch obr if required +- name: download_obr is true so fetch and do all the things + block: + + # Fetch obr and copy to temp + - name: Fetch obr + get_url: + url: "{{ atl_obr_download_href }}" + dest: "{{ atl_obr_download }}" + mode: 0755 + force: false + register: atl_obr_completed + + # If obr was fetched make the lock directory + - name: Create moving_lock. + file: + path: "{{ atl_obr_moving_lock }}" + state: directory + when: + - atl_obr_completed is succeeded + register: moving_lock_created + + # Directory lock was created by this run? + # If so, then set a fact intending to move obr + - name: Move obr Scenario - lock created by this run + set_fact: + move_obr: true + when: + - moving_lock_created is succeeded + - moving_lock_created.changed + # Otherwise directory lock was either already created or + # could not be created. Fall back is to continue and install from temp + + when: download_obr + +# If the intention is to move obr to home_shared +- name: Move obr to home_shared + block: + + - name: Copy temp installer to home_shared + copy: + src: "{{ atl_obr_download }}" + dest: "{{ atl_obr_shared_download }}" + remote_src: true + register: copied + + - name: Create completed_lock once obr downloaded and copied + file: + path: "{{ atl_obr_completed_lock }}" + state: directory + when: copied is succeeded + register: completed_lock_created + + - name: Remove moving_lock to show that obr is completed + file: + path: "{{ atl_obr_moving_lock }}" + state: absent + when: + - completed_lock_created is succeeded + - copied is succeeded + register: moving_lock_removed + + - name: Delete old temp installer + file: + path: "{{ atl_obr_download }}" + state: absent + when: moving_lock_removed is succeeded + register: temp_deleted + + - name: Set install to home_shared location + set_fact: + atl_obr_download: "{{ atl_obr_shared_download }}" + when: temp_deleted is succeeded + + when: move_obr + +# At this point the binary is in {{ atl_obr_download }} +# (which is either on home_shared or temp) + +- name: Ensure instaled-plugins dir exists + file: + path: "{{ atl_product_home_shared }}/plugins/installed-plugins" + state: directory + mode: 0750 + owner: "{{ atl_product_user }}" + group: "{{ atl_product_user }}" + +# Note as ansible unarchive cant handle "-j junk paths" we need to ignore errors to bypass the path verify +- name: Unpack the obr into the installed-plugins dir + unarchive: + remote_src: true + src: "{{ atl_obr_download }}" + dest: "{{ atl_product_home_shared }}/plugins/installed-plugins" + creates: "{{ atl_product_home_shared }}/plugins/installed-plugins/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.jar" + list_files: no + exclude: + - M* + owner: "{{ atl_product_user }}" + group: "{{ atl_product_user }}" + mode: 0750 + ignore_errors: yes + +- name: Move JSD dependency jars into the installed-plugins dir + shell: + cmd: "mv {{ atl_product_home_shared }}/plugins/installed-plugins/dependencies/*.jar {{ atl_product_home_shared }}/plugins/installed-plugins" + creates: "{{ atl_product_home_shared }}/plugins/installed-plugins/servicedesk-core-ui-plugin-{{ atl_jsd_build_info.json.name }}-*.jar" diff --git a/roles/product_install/tasks/main.yml b/roles/product_install/tasks/main.yml index 77371fa..81cf0d0 100644 --- a/roles/product_install/tasks/main.yml +++ b/roles/product_install/tasks/main.yml @@ -116,6 +116,7 @@ - "{{ atl_product_home }}" - "{{ atl_product_installation_versioned }}" - "{{ atl_product_version_cache_dir }}" + - "{{ atl_product_home_shared_download_dir }}" changed_when: false # For Molecule idempotence check # At this point atl_product_version should be set, cache if necessary. @@ -125,18 +126,120 @@ dest: "{{ atl_product_version_cache }}" force: true +# For the first run a temp binary should be downloaded but moved to +# shared home to ensure all subsequent nodes have access +# to the same specific version binary. +# To prevent a race condition with multiple downloads at the same time +# a directory is used as a lockfile (atomic operation) when moving binary. -# Note: We don't the cache binary in the shared drive to the complexity -# around download race-conditions if multiple nodes are starting at -# the same time. When downloading from product-downloads.atlassian.com -# (which is a CDN) takes seconds anyway. -- name: Fetch product installer - get_url: - url: "{{ atl_product_download_url }}" - dest: "{{ atl_product_download }}" - mode: 0755 - force: false +- name: Set assumptions to avoid race condition + set_fact: + download_binary: true + move_binary: false + atl_product_download: "{{ atl_product_temp_download }}" +# Check for pre-downloaded binary on shared_home and completed lock dir. +- name: Check for completed lock directory + stat: + path: "{{ atl_product_home_shared_completed_lock }}" + register: completed_lock + +- name: Check for product installer in home_shared + stat: + path: "{{ atl_product_home_shared_download }}" + register: home_shared_download + +# If binary exists and lockdir exists use this binary instead +- name: Check lock directory and binary exists on shared_home + set_fact: + download_binary: false + atl_product_download: "{{ atl_product_home_shared_download }}" + when: + - home_shared_download.stat.exists + - completed_lock.stat.isdir is defined + - completed_lock.stat.isdir + +# Fetch binary if required +- name: download_binary is true so fetch and do all the things + block: + + # Fetch binary and copy to temp + - name: Fetch binary + get_url: + url: "{{ atl_product_download_url }}" + dest: "{{ atl_product_temp_download }}" + mode: 0755 + force: false + register: atl_product_completed + + # If product installer was fetched make the lock directory + - name: Create moving_lock. + file: + path: "{{ atl_product_home_shared_moving_lock }}" + state: directory + when: + - atl_product_completed is succeeded + register: moving_lock_created + + # Directory lock was created by this run? + # If so, then set a fact intending to move binary + - name: Move binary Scenario - lock created by this run + set_fact: + move_binary: true + when: + - moving_lock_created is succeeded + - moving_lock_created.changed + # Otherwise directory lock was either already created or + # could not be created. Fall back is to continue and install from temp + + when: download_binary + +# If the intention is to move binary to home_shared +- name: Move product installer to home_shared + block: + + - name: Copy temp installer to home_shared + copy: + src: "{{ atl_product_temp_download }}" + dest: "{{ atl_product_home_shared_download }}" + remote_src: true + when: + - moving_lock_created is succeeded + - moving_lock_created.changed + register: copied + + - name: Create completed_lock once product installer downloaded and copied + file: + path: "{{ atl_product_home_shared_completed_lock }}" + state: directory + when: copied is succeeded + register: completed_lock_created + + - name: Remove moving_lock to show that binary is completed + file: + path: "{{ atl_product_home_shared_moving_lock }}" + state: absent + when: + - completed_lock_created is succeeded + - copied is succeeded + register: moving_lock_removed + + - name: Delete old temp installer + file: + path: "{{ atl_product_temp_download }}" + state: absent + when: moving_lock_removed is succeeded + register: temp_deleted + + - name: Set install to home_shared location + set_fact: + atl_product_download: "{{ atl_product_home_shared_download }}" + when: temp_deleted is succeeded + + when: move_binary + +# At this point the binary is in {{ atl_product_download }} +# (which is either on home_shared or temp) - name: Unpack the downloaded application depending on format include_tasks: "unpack_{{ atl_download_format }}.yml" @@ -146,3 +249,9 @@ dest: "{{ atl_product_installation_current }}" state: link force: true + +- name: Include if jsd is requested to be installed from OBR + include_tasks: "jira-servicedesk_as_obr.yml" + when: + - atl_install_jsd_as_obr is defined + - atl_install_jsd_as_obr diff --git a/roles/product_install/tasks/unpack_installer.yml b/roles/product_install/tasks/unpack_installer.yml index 925dca0..f340463 100644 --- a/roles/product_install/tasks/unpack_installer.yml +++ b/roles/product_install/tasks/unpack_installer.yml @@ -10,6 +10,8 @@ # actions. For example, if root and the 'jira' user exists then it # will create 'jira1'; this potentially creates idempotency/upgrade # issues down the line. +# The variable {{ atl_product_download }} will be on temp for first nodes and shared_home for +# subsequent nodes. - name: Run the installer command: /bin/sh "{{ atl_product_download }}" -q -varfile "{{ atl_product_varfile }}" args: diff --git a/roles/product_startup/defaults/main.yml b/roles/product_startup/defaults/main.yml index ca1eda5..45d6f8a 100644 --- a/roles/product_startup/defaults/main.yml +++ b/roles/product_startup/defaults/main.yml @@ -14,5 +14,6 @@ atl_startup_exec_path: "{{ atl_product_installation_current }}/{{ atl_startup_sc atl_startup_exec_options: ["-fg"] atl_startup_systemd_params: [] - atl_systemd_service_name: "{{ atl_product_edition }}.service" + +atl_systemd_service_target: "multi-user.target" diff --git a/roles/product_startup/templates/product.service.j2 b/roles/product_startup/templates/product.service.j2 index 6b5077f..8310e88 100644 --- a/roles/product_startup/templates/product.service.j2 +++ b/roles/product_startup/templates/product.service.j2 @@ -15,4 +15,4 @@ ExecStart={{ atl_startup_exec_path }}{% for c in atl_startup_exec_options %} {{ Restart=on-failure [Install] -WantedBy=multi-target.target +WantedBy={{ atl_systemd_service_target }} diff --git a/roles/restore_backups/.yamllint b/roles/restore_backups/.yamllint new file mode 100644 index 0000000..a87f8ff --- /dev/null +++ b/roles/restore_backups/.yamllint @@ -0,0 +1,12 @@ +extends: default + +rules: + braces: + max-spaces-inside: 1 + level: error + brackets: + max-spaces-inside: 1 + level: error + line-length: disable + truthy: disable + trailing-spaces: false diff --git a/roles/restore_backups/defaults/main.yml b/roles/restore_backups/defaults/main.yml new file mode 100644 index 0000000..6561c7e --- /dev/null +++ b/roles/restore_backups/defaults/main.yml @@ -0,0 +1,4 @@ +--- + +atl_backup_home_restore_canary_filename: ".slingshot_home_restore" +atl_backup_home_restore_canary_path: "{{ atl_product_home_shared }}/{{ atl_backup_home_restore_canary_filename }}" diff --git a/roles/restore_backups/molecule/default/Dockerfile.j2 b/roles/restore_backups/molecule/default/Dockerfile.j2 new file mode 100644 index 0000000..e6aa95d --- /dev/null +++ b/roles/restore_backups/molecule/default/Dockerfile.j2 @@ -0,0 +1,14 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi diff --git a/roles/restore_backups/molecule/default/molecule.yml b/roles/restore_backups/molecule/default/molecule.yml new file mode 100644 index 0000000..7f082f6 --- /dev/null +++ b/roles/restore_backups/molecule/default/molecule.yml @@ -0,0 +1,36 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: amazon_linux2 + image: amazonlinux:2 + groups: + - aws_node_local + ulimits: + - nofile:262144:262144 + - name: ubuntu_lts + image: ubuntu:bionic + groups: + - aws_node_local + ulimits: + - nofile:262144:262144 +provisioner: + name: ansible + options: + skip-tags: runtime_pkg + lint: + name: ansible-lint + options: + x: ["701"] + inventory: + links: + group_vars: ../../../../group_vars/ +verifier: + name: testinfra + lint: + name: flake8 + enabled: false diff --git a/roles/restore_backups/molecule/default/playbook.yml b/roles/restore_backups/molecule/default/playbook.yml new file mode 100644 index 0000000..ffd0c12 --- /dev/null +++ b/roles/restore_backups/molecule/default/playbook.yml @@ -0,0 +1,10 @@ +--- +- name: Converge + hosts: all + vars: + atl_backup_manifest_url: '' + atl_backup_home_restore_canary_path: '/tmp/canary.tmp' + + roles: + # Should be no-op + - role: restore_backups diff --git a/roles/restore_backups/molecule/default/tests/test_default.py b/roles/restore_backups/molecule/default/tests/test_default.py new file mode 100644 index 0000000..0a7276f --- /dev/null +++ b/roles/restore_backups/molecule/default/tests/test_default.py @@ -0,0 +1,10 @@ +import os + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +def test_no_canary_file(host): + assert not host.file('atl_backup_home_restore_canary_path').exists diff --git a/roles/restore_backups/molecule/restore/Dockerfile.j2 b/roles/restore_backups/molecule/restore/Dockerfile.j2 new file mode 100644 index 0000000..e6aa95d --- /dev/null +++ b/roles/restore_backups/molecule/restore/Dockerfile.j2 @@ -0,0 +1,14 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi diff --git a/roles/restore_backups/molecule/restore/molecule.yml b/roles/restore_backups/molecule/restore/molecule.yml new file mode 100644 index 0000000..04c0973 --- /dev/null +++ b/roles/restore_backups/molecule/restore/molecule.yml @@ -0,0 +1,30 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: amazon_linux2 + image: amazonlinux:2 + groups: + - aws_node_local + ulimits: + - nofile:262144:262144 +provisioner: + name: ansible + options: + skip-tags: runtime_pkg + lint: + name: ansible-lint + options: + x: ["701"] + inventory: + links: + group_vars: ../../../../group_vars/ +verifier: + name: testinfra + lint: + name: flake8 + enabled: false diff --git a/roles/restore_backups/molecule/restore/playbook.yml b/roles/restore_backups/molecule/restore/playbook.yml new file mode 100644 index 0000000..073d24b --- /dev/null +++ b/roles/restore_backups/molecule/restore/playbook.yml @@ -0,0 +1,75 @@ +--- +- name: Converge + hosts: all + vars: + atl_backup_home_dest: "{{ test_archive }}" + atl_backup_home_restore_canary_path: '/tmp/canary.tmp' + atl_backup_id: 'test-backup' + atl_backup_manifest_url: 'fake_manifest' + + atl_product_edition: 'jira-software' + atl_product_home_shared: '/media/atl/jira/shared' + atl_product_user: 'jira' + atl_product_user_uid: '2001' + atl_product_version_cache: "{{ atl_product_home_shared }}/{{ atl_product_edition }}.version" + + test_archive: '/tmp/hello.tar.gz' + test_archive_file: 'hello.txt' + test_archive_source: '/tmp/hello' + test_pre_step_prefix: '[PRE-TEST]' + test_product_version_file: "/tmp/{{ atl_product_edition }}.version" + + pre_tasks: + - name: "{{ test_pre_step_prefix }} Install tar" + package: + state: present + name: tar + + - name: "{{ test_pre_step_prefix }} Install useradd and groupadd binaries" + package: + state: present + name: shadow-utils + + - name: "{{ test_pre_step_prefix }} Create application group" + group: + name: "{{ atl_product_user }}" + gid: "{{ atl_product_user_uid }}" + + - name: "{{ test_pre_step_prefix }} Create application user" + user: + name: "{{ atl_product_user }}" + uid: "{{ atl_product_user_uid }}" + group: "{{ atl_product_user }}" + + - block: + - name: "{{ test_pre_step_prefix }} Create a directory for the shared home archive" + file: + path: "{{ test_archive_source }}" + state: directory + mode: 0755 + - name: "{{ test_pre_step_prefix }} Create a file in the shared home" + lineinfile: + create: yes + line: 'Hello, world!' + path: "{{ test_archive_source }}/{{ test_archive_file }}" + mode: 0640 + - name: "{{ test_pre_step_prefix }} Create the version file in the shared home" + lineinfile: + create: yes + line: '8.5' + path: "{{ test_product_version_file }}" + mode: 0640 + - name: "{{ test_pre_step_prefix }} Archive the shared home" + archive: + path: + - "{{ test_archive_source }}" + - "{{ test_product_version_file }}" + dest: "{{ test_archive }}" + owner: "{{ atl_product_user }}" + + tasks: + - name: Install distro-specific restore support packages + include_tasks: "../../tasks/{{ ansible_distribution|lower }}.yml" + + - name: Restore shared home + include_tasks: "../../tasks/home_restore.yml" diff --git a/roles/restore_backups/molecule/restore/tests/test_default.py b/roles/restore_backups/molecule/restore/tests/test_default.py new file mode 100644 index 0000000..bcfa53d --- /dev/null +++ b/roles/restore_backups/molecule/restore/tests/test_default.py @@ -0,0 +1,39 @@ +import os +import pytest + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +@pytest.mark.parametrize('exe', [ + '/usr/bin/pg_dump', + '/usr/bin/pg_restore', + '/usr/bin/psql' +]) +def test_postgresql_amazon_linux_extras_exes(host, exe): + assert host.file(exe).exists + +def test_postgresql_version(host): + pg_dump_version_output = host.check_output('pg_dump --version') + assert '(PostgreSQL) 9.6' in pg_dump_version_output + +@pytest.mark.parametrize('file', [ + '/media/atl/jira/shared', + '/media/atl/jira/shared/hello', + '/media/atl/jira/shared/hello/hello.txt' +]) +def test_shared_home_owner(host, file): + assert host.file(file).exists + assert host.file(file).user == 'jira' + assert host.file(file).group == 'jira' + +def test_file_modes(host): + assert host.file('/media/atl/jira/shared/hello').mode == 0o755 + assert host.file('/media/atl/jira/shared/hello/hello.txt').mode == 0o640 + +def test_version_file_owned_by_root(host): + assert host.file('/media/atl/jira/shared/jira-software.version').exists + assert host.file('/media/atl/jira/shared/jira-software.version').user == 'root' + assert host.file('/media/atl/jira/shared/jira-software.version').group == 'root' \ No newline at end of file diff --git a/roles/restore_backups/tasks/amazon.yml b/roles/restore_backups/tasks/amazon.yml new file mode 100644 index 0000000..2444212 --- /dev/null +++ b/roles/restore_backups/tasks/amazon.yml @@ -0,0 +1,9 @@ +--- + +# Amazon Linux 2 supplies extra packages via a special command. +- name: Enable Postgresql from 'extras' + command: amazon-linux-extras install -y "postgresql{{ postgres_version }}" + args: + creates: /usr/bin/psql + environment: + PYTHON: /bin/python diff --git a/roles/restore_backups/tasks/home_restore.yml b/roles/restore_backups/tasks/home_restore.yml new file mode 100644 index 0000000..4c5f2ad --- /dev/null +++ b/roles/restore_backups/tasks/home_restore.yml @@ -0,0 +1,44 @@ +--- +- name: Check for the restore canary file + stat: + path: "{{ atl_backup_home_restore_canary_path }}" + register: restore_canary + +- block: + - name: Create shared home if necessary + file: + path: "{{ atl_product_home_shared }}" + state: directory + mode: 0750 + owner: "{{ atl_product_user }}" + group: "{{ atl_product_user }}" + + - name: Restore the shared-home backup + unarchive: + src: "{{ atl_backup_home_dest }}" + remote_src: yes + dest: "{{ atl_product_home_shared }}" + + - name: Set shared home owner and group to application user + file: + path: "{{ atl_product_home_shared }}" + recurse: yes + group: "{{ atl_product_user }}" + owner: "{{ atl_product_user }}" + state: directory + + - name: Set version file owner and group to root + file: + path: "{{ atl_product_version_cache }}" + group: root + owner: root + state: file + # Ignore the error in case there is no product version file in the backup + ignore_errors: yes + + - name: Create restore-canary if necessary + copy: + dest: "{{ atl_backup_home_restore_canary_path }}" + content: "{{ atl_backup_id }}" + + when: not restore_canary.stat.exists diff --git a/roles/restore_backups/tasks/main.yml b/roles/restore_backups/tasks/main.yml new file mode 100644 index 0000000..1c81430 --- /dev/null +++ b/roles/restore_backups/tasks/main.yml @@ -0,0 +1,115 @@ +--- + +# This role will attempt to fetch and load the backup manifest from a +# remote S3 URL. On successful completion the contents of JSON or YAML +# document will be in the var `atl_backup_manifest`. +# +# PREREQUISITES: +# * `atl_backup_manifest_url` points at the manifest. +# * The shared home filesystem is mounted if necessary (e.g. NFS/EFS). +# * The database has been created and the variable `db_created` is +# registered with the result (i.e: `register: db_created`). +# +# NOTE: The actual DB/FS restore operations could potentially be split +# out into discrete roles, but currently that is not required. +# +# TODO: Support HTTPS with authentication. Deferred until after the +# initial testing release. + +- block: + + - name: Ensure temp directory is present + file: + path: "{{ atl_installer_temp }}" + state: directory + mode: 0750 + owner: "{{ atl_product_user }}" + group: "{{ atl_product_user }}" + changed_when: false # For Molecule idempotence check + + - name: Parse the manifest URL + set_fact: + atl_backup_manifest_url: "{{ atl_backup_manifest_url | urlsplit }}" + + - name: Extract manifest file information + set_fact: + atl_backup_manifest_bucket: "{{ atl_backup_manifest_url.hostname }}" + atl_backup_manifest_path: "{{ atl_backup_manifest_url.path }}" + atl_backup_manifest_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest_url.path | basename }}" + + - name: Fetch the manifest from S3 + aws_s3: + mode: get + overwrite: different + bucket: "{{ atl_backup_manifest_bucket }}" + object: "{{ atl_backup_manifest_path }}" + dest: "{{ atl_backup_manifest_dest }}" + when: atl_backup_manifest_url.scheme == 's3' + + - name: Load parameters from manifest + include_vars: + file: "{{ atl_backup_manifest_dest }}" + name: atl_backup_manifest + + - name: Define the DB and home dump destinations + set_fact: + # FIXME: The manifest format is still undecided so the + # following usages will need to be updated once it settles.. + atl_backup_id: "{{ atl_backup_manifest.name }}" + atl_backup_db_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.artifacts.db.location.location | basename }}" + atl_backup_home_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.artifacts.sharedHome.location.location | basename }}" + + # FIXME: Here we fetch the backups. However we may wish to stream + # these directly from S3 to the target DB/FS to avoid requiring + # disk-space for the intermediate files. + - name: Fetch DB backup from S3 + aws_s3: + mode: get + overwrite: different + bucket: "{{ atl_backup_manifest.artifacts.db.location.location | urlsplit('hostname') }}" + object: "{{ atl_backup_manifest.artifacts.db.location.location | urlsplit('path') }}" + # We save the backup as a .tar file so that the postgresql_db module uses pg_restore instead of psql to do restore + # This can be removed when ansible 2.10 is released + dest: "{{ atl_backup_db_dest }}.tar" + + - name: Fetch Home backup from S3 + aws_s3: + mode: get + overwrite: different + bucket: "{{ atl_backup_manifest.artifacts.sharedHome.location.location | urlsplit('hostname') }}" + object: "{{ atl_backup_manifest.artifacts.sharedHome.location.location | urlsplit('path') }}" + dest: "{{ atl_backup_home_dest }}" + + - name: Install distro-specific restore support packages + include_tasks: "{{ ansible_distribution|lower }}.yml" + + + - name: Restore application database + postgresql_db: + login_host: "{{ atl_db_host }}" + login_user: "{{ atl_db_root_user }}" + login_password: "{{ atl_db_root_password }}" + port: "{{ atl_db_port }}" + name: "{{ atl_jdbc_db_name }}" + owner: "{{ atl_jdbc_user }}" + encoding: "{{ atl_jdbc_encoding }}" + lc_collate: "{{ atl_jdbc_collation }}" + lc_ctype: "{{ atl_jdbc_ctype }}" + template: "{{ atl_jdbc_template }}" + # Depends on fetch_backup roles + state: restore + target: "{{ atl_backup_db_dest }}.tar" + target_opts: "-Fc" + register: result + # managed DBs in cloud providers are not allowing full root access to the DB engine, we can safely ignore the COMMENT ON EXTENSION error + failed_when: + - result.rc != 0 + - '"COMMENT ON EXTENSION" not in result.msg' + when: db_created.changed and atl_backup_db_dest is defined + + + - name: Restore shared home + include_tasks: "home_restore.yml" + + + when: atl_restore_required