diff --git a/.gitignore b/.gitignore index 321ff61..453a538 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,6 @@ __pycache__ *.pyc /.venv \#*\# +.envrc .idea .vscode diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 5467a1a..a2618db 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -18,7 +18,7 @@ pipelines: - step: name: Pre Parallelization stage script: - - echo "Running tests in 36 batches" + - echo "Running tests in 37 batches" - step: name: Check if the template is up-to-date @@ -231,6 +231,15 @@ pipelines: - ./bin/install-ansible --dev - cd roles/product_install - pipenv run molecule test -s jira_all + - step: + name: product_install/jira_tarball_download_url + services: + - docker + script: + - export ANSIBLE_CONFIG=./ansible.cfg + - ./bin/install-ansible --dev + - cd roles/product_install + - pipenv run molecule test -s jira_tarball_download_url - step: name: product_install/jira_cached_with_downgrade services: @@ -357,6 +366,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/product_startup - pipenv run molecule test -s synchrony + - step: name: Run Snyk security scan services: diff --git a/roles/linux_common/tasks/amazon.yml b/roles/linux_common/tasks/amazon.yml index 9dc15ae..325397c 100644 --- a/roles/linux_common/tasks/amazon.yml +++ b/roles/linux_common/tasks/amazon.yml @@ -3,10 +3,11 @@ - name: Install Amazon-Linux-specific support packages yum: name: - - shadow-utils - - libxml2 - - git-{{ git_version }} - dejavu-sans-fonts + - file + - git-{{ git_version }} + - libxml2 + - shadow-utils - name: Limit the SSH ciphers lineinfile: diff --git a/roles/product_install/defaults/main.yml b/roles/product_install/defaults/main.yml index e3a929c..d768e62 100644 --- a/roles/product_install/defaults/main.yml +++ b/roles/product_install/defaults/main.yml @@ -37,6 +37,7 @@ atl_servicedesk_url_map: atl_servicedesk_metadata_url: "{{ atl_servicedesk_url_map[atl_product_version] | default(atl_servicedesk_url_map['default']) }}" atl_install_jsd_as_obr: false +atl_source_obr_from_marketplace: true atl_servicedesk_download_file: "{{ atl_installer_temp }}/jira-servicedesk.{{ atl_product_version }}.obr" force_version_update: false diff --git a/roles/product_install/molecule/jira_all/tests/test_default.py b/roles/product_install/molecule/jira_all/tests/test_default.py index cca4380..1d793bf 100644 --- a/roles/product_install/molecule/jira_all/tests/test_default.py +++ b/roles/product_install/molecule/jira_all/tests/test_default.py @@ -42,7 +42,7 @@ def test_obr_completed_lockfile(host): assert lockfile.user == 'root' def test_obr_is_unpacked(host): - jsdjar = host.file('/media/atl/jira/shared/plugins/installed-plugins/jira-servicedesk-application-4.14.0.jar') + jsdjar = host.file('/opt/atlassian/jira-software/8.14.0/atlassian-jira/WEB-INF/atlassian-bundled-plugins/jira-servicedesk-application-4.14.0.jar') assert jsdjar.exists assert jsdjar.user == 'jira' - assert jsdjar.mode == 0o0750 \ No newline at end of file + assert jsdjar.mode == 0o0644 \ No newline at end of file diff --git a/roles/product_install/molecule/jira_tarball_download_url/Dockerfile.j2 b/roles/product_install/molecule/jira_tarball_download_url/Dockerfile.j2 new file mode 100644 index 0000000..e6aa95d --- /dev/null +++ b/roles/product_install/molecule/jira_tarball_download_url/Dockerfile.j2 @@ -0,0 +1,14 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi diff --git a/roles/product_install/molecule/jira_tarball_download_url/converge.yml b/roles/product_install/molecule/jira_tarball_download_url/converge.yml new file mode 100644 index 0000000..6b455ae --- /dev/null +++ b/roles/product_install/molecule/jira_tarball_download_url/converge.yml @@ -0,0 +1,26 @@ +--- +- name: Converge + hosts: all + vars: + atl_product_family: "jira" + atl_product_edition: "jira-software" + atl_product_user: "jira" + # Version 8.14.0 is chosen deliberately as it is a version where jira and jira-software returns different buildNumber from MPAC + # See DCD-1216 for context + atl_product_version: "8.14.0" + atl_product_download_url: "https://www.atlassian.com/software/jira/downloads/binary/atlassian-jira-software-8.14.0.tar.gz" + + # This tests for an edge case where the product URL and OBR are source from a provided URL. + # Although marketplace (mpac) is used in this example, this could potentially be in a non mpac url. + # In this case it would not be possible to query atl_jsd_build_info to source the atl_jsd_build. One is therefore provided. + atl_install_jsd_as_obr: true + atl_jsd_build: "4.18.0" + atl_obr_download_url: "https://marketplace.atlassian.com/download/apps/1213632/version/1040180000" + # When using a tarball the following are also required: + atl_download_format: "tarball" + atl_use_system_jdk: true + + roles: + - role: linux_common + - role: product_common + - role: product_install diff --git a/roles/product_install/molecule/jira_tarball_download_url/molecule.yml b/roles/product_install/molecule/jira_tarball_download_url/molecule.yml new file mode 100644 index 0000000..3ea826a --- /dev/null +++ b/roles/product_install/molecule/jira_tarball_download_url/molecule.yml @@ -0,0 +1,23 @@ +--- +dependency: + name: galaxy +driver: + name: docker +platforms: + - name: amazon_linux2 + image: amazonlinux:2 + groups: + - aws_node_local + - name: ubuntu_lts + image: ubuntu:bionic + groups: + - aws_node_local +provisioner: + name: ansible + options: + skip-tags: runtime_pkg + inventory: + links: + group_vars: ../../../../group_vars/ +verifier: + name: testinfra diff --git a/roles/product_install/molecule/jira_tarball_download_url/tests/test_default.py b/roles/product_install/molecule/jira_tarball_download_url/tests/test_default.py new file mode 100644 index 0000000..b884603 --- /dev/null +++ b/roles/product_install/molecule/jira_tarball_download_url/tests/test_default.py @@ -0,0 +1,48 @@ +import os +from six.moves import urllib + +import testinfra.utils.ansible_runner +import json + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +def test_version_is_correct(host): + verfile = host.file('/media/atl/jira/shared/jira-software.version') + assert verfile.exists + + assert verfile.content.decode("UTF-8").strip() == "8.14.0" + +def test_is_downloaded(host): + installer = host.file('/media/atl/downloads/jira-software.8.14.0.tar.gz') + assert installer.exists + assert installer.user == 'root' + +def test_completed_lockfile(host): + lockfile = host.file('/media/atl/downloads/jira-software.8.14.0.tar.gz_completed') + assert lockfile.exists + assert lockfile.user == 'root' + +def test_is_unpacked(host): + installer = host.file('/opt/atlassian/jira-software/8.14.0/atlassian-jira/') + assert installer.exists + assert installer.is_directory + assert installer.user == 'jira' + assert installer.mode == 0o0755 + +def test_obr_is_downloaded(host): + installer = host.file('/media/atl/downloads/jira-servicedesk-application-4.18.0.obr') + assert installer.exists + assert installer.user == 'root' + +def test_obr_completed_lockfile(host): + lockfile = host.file('/media/atl/downloads/jira-servicedesk-application-4.18.0.obr_completed') + assert lockfile.exists + assert lockfile.user == 'root' + +def test_obr_is_unpacked(host): + jsdjar = host.file('/opt/atlassian/jira-software/8.14.0/atlassian-jira/WEB-INF/atlassian-bundled-plugins/jira-servicedesk-application-4.18.0.jar') + assert jsdjar.exists + assert jsdjar.user == 'jira' + assert jsdjar.mode == 0o0644 \ No newline at end of file diff --git a/roles/product_install/tasks/jira-servicedesk_as_obr.yml b/roles/product_install/tasks/jira-servicedesk_as_obr.yml index 3905a44..73e7c3b 100644 --- a/roles/product_install/tasks/jira-servicedesk_as_obr.yml +++ b/roles/product_install/tasks/jira-servicedesk_as_obr.yml @@ -1,33 +1,75 @@ --- -- name: Get the installer product version info - uri: - url: "{{ atl_mpac_products }}/key/jira/versions/name/{{ atl_product_version }}" - return_content: yes - register: atl_product_version_info +- name: Check for alternate obr download url + set_fact: + atl_source_obr_from_marketplace: false + when: + - atl_obr_download_url is defined -- name: Show the returned build number - debug: - msg="buildNumber={{ atl_product_version_info.json.buildNumber }}" +- name: Source jira-servicedesk obr from marketplace + block: -- name: Get the JSD build version info - uri: - url: "{{ atl_mpac_products }}/key/jira-servicedesk/versions/latest?application=\ - jira&applicationBuild={{ atl_product_version_info.json.buildNumber }}" - return_content: yes - register: atl_jsd_build_info + - name: Marketplace OBR - Get the installer product version info + uri: + url: "{{ atl_mpac_products }}/key/jira/versions/name/{{ atl_product_version }}" + return_content: yes + register: atl_product_version_info -- name: Show the returned obr binary href - debug: - msg="obr_ref={{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}" + - name: Marketplace OBR - Show the returned build number + debug: + msg="buildNumber={{ atl_product_version_info.json.buildNumber }}" -- name: how about getting the obr filename - debug: - msg="obr_name=jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr" + - name: Marketplace OBR - Get the JSD build version info + uri: + url: "{{ atl_mpac_products }}/key/jira-servicedesk/versions/latest?application=\ + jira&applicationBuild={{ atl_product_version_info.json.buildNumber }}" + return_content: yes + register: atl_jsd_build_info + + - name: Marketplace OBR - Show the returned obr binary href + debug: + msg="obr_ref={{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}" + + - name: Marketplace OBR - Set atl_obr_download_url + set_fact: + atl_obr_download_url: "{{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}" + + - name: Marketplace OBR - Set atl_jsd_build + set_fact: + atl_jsd_build: "{{ atl_jsd_build_info.json.name }}" + + - name: Marketplace OBR - Show the obr filename + debug: + msg="obr_name=jira-servicedesk-application-{{ atl_jsd_build }}.obr" + + - name: Marketplace OBR - Set the obr filename + set_fact: + atl_obr_filename: "jira-servicedesk-application-{{ atl_jsd_build }}.obr" + + when: + - atl_source_obr_from_marketplace | bool + +# When sourcing jira-servicedesk from marketplace the atl_jsd_build is source by querying api. +# In this case atl_jsd_build must be passed manually by flag + +- name: Source jira-servicedesk obr from alternate url + block: + + - name: Alternate URL OBR - Show the obr filename + debug: + msg="obr_name=jira-servicedesk-application-{{ atl_jsd_build }}.obr" + + - name: Alternate OBR - Set the obr filename + set_fact: + atl_obr_filename: "jira-servicedesk-application-{{ atl_jsd_build }}.obr" + + when: + - not atl_source_obr_from_marketplace | bool - name: is shared_home set ? debug: msg="atl_product_home_shared_download_dir={{ atl_product_home_shared_download_dir }}" + # For the first run a temp obr should be downloaded but moved to # shared home to ensure all subsequent nodes have access # to the same specific version binary. @@ -38,12 +80,10 @@ set_fact: download_obr: true move_obr: false - atl_obr_download_href: "{{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}" - atl_obr_filename: "jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr" - atl_obr_download: "{{ atl_installer_temp }}/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr" - atl_obr_shared_download: "{{ atl_product_home_shared_download_dir }}/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr" - atl_obr_moving_lock: "{{ atl_product_home_shared_download_dir }}/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr_moving" - atl_obr_completed_lock: "{{ atl_product_home_shared_download_dir }}/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr_completed" + atl_obr_download: "{{ atl_installer_temp }}/{{ atl_obr_filename }}" + atl_obr_shared_download: "{{ atl_product_home_shared_download_dir }}/{{ atl_obr_filename }}" + atl_obr_moving_lock: "{{ atl_product_home_shared_download_dir }}/{{ atl_obr_filename }}_moving" + atl_obr_completed_lock: "{{ atl_product_home_shared_download_dir }}/{{ atl_obr_filename }}_completed" # Check for pre-downloaded obr on shared_home and completed lock dir. - name: Check for completed lock directory @@ -56,6 +96,10 @@ path: "{{ atl_obr_shared_download }}" register: home_shared_download +- name: debug home_shared_download + debug: + var: home_shared_download + # If obr exists and lockdir exists use this obr instead - name: Check lock directory and obr exists on shared_home set_fact: @@ -63,22 +107,47 @@ atl_obr_download: "{{ atl_obr_shared_download }}" when: - home_shared_download.stat.exists + - home_shared_download.stat.mimetype is match("application/zip") - completed_lock.stat.isdir is defined - completed_lock.stat.isdir -# Fetch obr if required +# Fetch obr if required - note we validate it by mimetype rather than checksum due to https://ecosystem.atlassian.net/browse/AMKT-25526 - name: download_obr is true so fetch and do all the things block: + - debug: + var: atl_obr_download_url + - debug: + var: atl_obr_download # Fetch obr and copy to temp - name: Fetch obr get_url: - url: "{{ atl_obr_download_href }}" + url: "{{ atl_obr_download_url }}" dest: "{{ atl_obr_download }}" mode: 0755 - force: false + force: true + timeout: 600 register: atl_obr_completed + - name: Confirm the output from the download task + debug: + var: atl_obr_completed + + # get details about the obr + - name: Stat the new obr file + stat: + path: "{{ atl_obr_completed.dest }}" + get_mime: yes + register: atl_obr_stats + when: + - atl_obr_completed.dest is defined + + - name: fail if the downloaded OBR is not a zip file + fail: + msg: "The downloaded OBR was not detected as being a valid ZIP file: {{ atl_obr_stats }}" + when: + - (atl_obr_stats.stat.mimetype is not defined) or (atl_obr_stats.stat.mimetype is not match("application/zip")) + # If obr was fetched make the lock directory - name: Create moving_lock. file: @@ -154,23 +223,30 @@ group: "{{ atl_product_user }}" # Note as ansible unarchive cant handle "-j junk paths" we need to ignore errors to bypass the path verify -- name: Unpack the obr into the installed-plugins dir +- name: Unpack the obr into the atlassian-bundled-plugins dir unarchive: - remote_src: true + remote_src: yes src: "{{ atl_obr_download }}" - dest: "{{ atl_product_home_shared }}/plugins/installed-plugins" - creates: "{{ atl_product_home_shared }}/plugins/installed-plugins/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.jar" + dest: "{{ atl_product_installation_versioned }}/atlassian-jira/WEB-INF/atlassian-bundled-plugins" + creates: "{{ atl_product_installation_versioned }}/atlassian-jira/WEB-INF/atlassian-bundled-plugins/jira-servicedesk-application-{{ atl_jsd_build }}.jar" list_files: no exclude: - M* + - obr.xml owner: "{{ atl_product_user }}" group: "{{ atl_product_user }}" - mode: 0750 + mode: 0644 register: obr_unpack - ignore_errors: yes -- name: Move JSD dependency jars into the installed-plugins dir # noqa 503 - ignore lint info about when changed - shell: - cmd: "mv {{ atl_product_home_shared }}/plugins/installed-plugins/dependencies/*.jar {{ atl_product_home_shared }}/plugins/installed-plugins" - creates: "{{ atl_product_home_shared }}/plugins/installed-plugins/servicedesk-core-ui-plugin-{{ atl_jsd_build_info.json.name }}-*.jar" +- name: Move JSD dependency jars into the bundled-plugins dir # noqa 503 - ignore lint info about when changed + copy: + remote_src: yes + src: "{{ atl_product_installation_versioned }}/atlassian-jira/WEB-INF/atlassian-bundled-plugins/dependencies/" + dest: "{{ atl_product_installation_versioned }}/atlassian-jira/WEB-INF/atlassian-bundled-plugins/" when: obr_unpack.changed + +- name: Remove the empty dependencies folder # noqa 503 - ignore lint info about when changed + file: + path: "{{ atl_product_installation_versioned }}/atlassian-jira/WEB-INF/atlassian-bundled-plugins/dependencies" + state: absent + when: obr_unpack.changed \ No newline at end of file