Merged in ITSD-94667-alternative-servicedesk-obr (pull request #171)

ITSD-94667 alternative servicedesk obr

Approved-by: Brett Meehan
Approved-by: Vadym Kovalskiy
Approved-by: Adam Brokes
Approved-by: Dylan Rathbone

.gitignore

@@ -5,5 +5,6 @@ __pycache__
 *.pyc
 /.venv
 \#*\#
+.envrc
 .idea
 .vscode

bitbucket-pipelines.yml

@@ -18,7 +18,7 @@ pipelines:
     - step:
         name: Pre Parallelization stage
         script:
-          - echo "Running tests in 36 batches"
+          - echo "Running tests in 37 batches"
 
     - step:
         name: Check if the template is up-to-date
@@ -231,6 +231,15 @@ pipelines:
              - ./bin/install-ansible --dev
              - cd roles/product_install
              - pipenv run molecule test -s jira_all
+        - step:
+           name: product_install/jira_tarball_download_url
+           services:
+             - docker
+           script:
+             - export ANSIBLE_CONFIG=./ansible.cfg
+             - ./bin/install-ansible --dev
+             - cd roles/product_install
+             - pipenv run molecule test -s jira_tarball_download_url
         - step:
            name: product_install/jira_cached_with_downgrade
            services:
@@ -357,6 +366,7 @@ pipelines:
              - ./bin/install-ansible --dev
              - cd roles/product_startup
              - pipenv run molecule test -s synchrony
+
         - step:
            name: Run Snyk security scan
            services:

roles/linux_common/tasks/amazon.yml

@@ -3,10 +3,11 @@
 - name: Install Amazon-Linux-specific support packages
   yum:
     name:
-      - shadow-utils
-      - libxml2
-      - git-{{ git_version }}
       - dejavu-sans-fonts
+      - file
+      - git-{{ git_version }}
+      - libxml2
+      - shadow-utils
 
 - name: Limit the SSH ciphers
   lineinfile:

roles/product_install/defaults/main.yml

@@ -37,6 +37,7 @@ atl_servicedesk_url_map:
 atl_servicedesk_metadata_url: "{{ atl_servicedesk_url_map[atl_product_version] | default(atl_servicedesk_url_map['default']) }}"
 
 atl_install_jsd_as_obr: false
+atl_source_obr_from_marketplace: true
 atl_servicedesk_download_file: "{{ atl_installer_temp }}/jira-servicedesk.{{ atl_product_version }}.obr"
 
 force_version_update: false

roles/product_install/molecule/jira_all/tests/test_default.py

@@ -42,7 +42,7 @@ def test_obr_completed_lockfile(host):
     assert lockfile.user == 'root'
 
 def test_obr_is_unpacked(host):
-    jsdjar = host.file('/media/atl/jira/shared/plugins/installed-plugins/jira-servicedesk-application-4.14.0.jar')
+    jsdjar = host.file('/opt/atlassian/jira-software/8.14.0/atlassian-jira/WEB-INF/atlassian-bundled-plugins/jira-servicedesk-application-4.14.0.jar')
     assert jsdjar.exists
     assert jsdjar.user == 'jira'
-    assert jsdjar.mode == 0o0750
+    assert jsdjar.mode == 0o0644

roles/product_install/molecule/jira_tarball_download_url/Dockerfile.j2

@@ -0,0 +1,14 @@
+# Molecule managed
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi

roles/product_install/molecule/jira_tarball_download_url/converge.yml

@@ -0,0 +1,26 @@
+---
+- name: Converge
+  hosts: all
+  vars:
+    atl_product_family: "jira"
+    atl_product_edition: "jira-software"
+    atl_product_user: "jira"
+    # Version 8.14.0 is chosen deliberately as it is a version where jira and jira-software returns different buildNumber from MPAC
+    # See DCD-1216 for context
+    atl_product_version: "8.14.0"
+    atl_product_download_url: "https://www.atlassian.com/software/jira/downloads/binary/atlassian-jira-software-8.14.0.tar.gz"
+
+    # This tests for an edge case where the product URL and OBR are source from a provided URL.
+    # Although marketplace (mpac) is used in this example, this could potentially be in a non mpac url.
+    # In this case it would not be possible to query atl_jsd_build_info to source the atl_jsd_build. One is therefore provided.
+    atl_install_jsd_as_obr: true
+    atl_jsd_build: "4.18.0"
+    atl_obr_download_url: "https://marketplace.atlassian.com/download/apps/1213632/version/1040180000"
+    # When using a tarball the following are also required:
+    atl_download_format: "tarball"
+    atl_use_system_jdk: true
+
+  roles:
+    - role: linux_common
+    - role: product_common
+    - role: product_install

roles/product_install/molecule/jira_tarball_download_url/molecule.yml

@@ -0,0 +1,23 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+platforms:
+  - name: amazon_linux2
+    image: amazonlinux:2
+    groups:
+      - aws_node_local
+  - name: ubuntu_lts
+    image: ubuntu:bionic
+    groups:
+      - aws_node_local
+provisioner:
+  name: ansible
+  options:
+    skip-tags: runtime_pkg
+  inventory:
+    links:
+      group_vars: ../../../../group_vars/
+verifier:
+  name: testinfra

roles/product_install/molecule/jira_tarball_download_url/tests/test_default.py

@@ -0,0 +1,48 @@
+import os
+from six.moves import urllib
+
+import testinfra.utils.ansible_runner
+import json
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_version_is_correct(host):
+    verfile = host.file('/media/atl/jira/shared/jira-software.version')
+    assert verfile.exists
+
+    assert verfile.content.decode("UTF-8").strip() == "8.14.0"
+
+def test_is_downloaded(host):
+    installer = host.file('/media/atl/downloads/jira-software.8.14.0.tar.gz')
+    assert installer.exists
+    assert installer.user == 'root'
+
+def test_completed_lockfile(host):
+    lockfile = host.file('/media/atl/downloads/jira-software.8.14.0.tar.gz_completed')
+    assert lockfile.exists
+    assert lockfile.user == 'root'
+
+def test_is_unpacked(host):
+    installer = host.file('/opt/atlassian/jira-software/8.14.0/atlassian-jira/')
+    assert installer.exists
+    assert installer.is_directory
+    assert installer.user == 'jira'
+    assert installer.mode == 0o0755
+
+def test_obr_is_downloaded(host):
+    installer = host.file('/media/atl/downloads/jira-servicedesk-application-4.18.0.obr')
+    assert installer.exists
+    assert installer.user == 'root'
+
+def test_obr_completed_lockfile(host):
+    lockfile = host.file('/media/atl/downloads/jira-servicedesk-application-4.18.0.obr_completed')
+    assert lockfile.exists
+    assert lockfile.user == 'root'
+
+def test_obr_is_unpacked(host):
+    jsdjar = host.file('/opt/atlassian/jira-software/8.14.0/atlassian-jira/WEB-INF/atlassian-bundled-plugins/jira-servicedesk-application-4.18.0.jar')
+    assert jsdjar.exists
+    assert jsdjar.user == 'jira'
+    assert jsdjar.mode == 0o0644

roles/product_install/tasks/jira-servicedesk_as_obr.yml

@@ -1,33 +1,75 @@
 ---
 
-- name: Get the installer product version info
-  uri:
-    url: "{{ atl_mpac_products }}/key/jira/versions/name/{{ atl_product_version }}"
-    return_content: yes
-  register: atl_product_version_info
+- name: Check for alternate obr download url
+  set_fact:
+    atl_source_obr_from_marketplace: false
+  when:
+    - atl_obr_download_url is defined
 
-- name: Show the returned build number
-  debug:
-    msg="buildNumber={{ atl_product_version_info.json.buildNumber }}"
+- name: Source jira-servicedesk obr from marketplace
+  block:
 
-- name: Get the JSD build version info
-  uri:
-    url: "{{ atl_mpac_products }}/key/jira-servicedesk/versions/latest?application=\
-    jira&applicationBuild={{ atl_product_version_info.json.buildNumber }}"
-    return_content: yes
-  register: atl_jsd_build_info
+    - name: Marketplace OBR - Get the installer product version info
+      uri:
+        url: "{{ atl_mpac_products }}/key/jira/versions/name/{{ atl_product_version }}"
+        return_content: yes
+      register: atl_product_version_info
 
-- name: Show the returned obr binary href
-  debug:
-    msg="obr_ref={{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}"
+    - name: Marketplace OBR - Show the returned build number
+      debug:
+        msg="buildNumber={{ atl_product_version_info.json.buildNumber }}"
 
-- name: how about getting the obr filename
-  debug:
-    msg="obr_name=jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr"
+    - name: Marketplace OBR - Get the JSD build version info
+      uri:
+        url: "{{ atl_mpac_products }}/key/jira-servicedesk/versions/latest?application=\
+        jira&applicationBuild={{ atl_product_version_info.json.buildNumber }}"
+        return_content: yes
+      register: atl_jsd_build_info
+
+    - name: Marketplace OBR - Show the returned obr binary href
+      debug:
+        msg="obr_ref={{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}"
+
+    - name: Marketplace OBR - Set atl_obr_download_url
+      set_fact:
+        atl_obr_download_url: "{{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}"
+
+    - name: Marketplace OBR - Set atl_jsd_build
+      set_fact:
+        atl_jsd_build: "{{ atl_jsd_build_info.json.name }}"
+
+    - name: Marketplace OBR - Show the obr filename
+      debug:
+        msg="obr_name=jira-servicedesk-application-{{ atl_jsd_build }}.obr"
+
+    - name: Marketplace OBR - Set the obr filename
+      set_fact:
+        atl_obr_filename: "jira-servicedesk-application-{{ atl_jsd_build }}.obr"
+
+  when:
+    - atl_source_obr_from_marketplace | bool
+
+# When sourcing jira-servicedesk from marketplace the atl_jsd_build is source by querying api.
+# In this case atl_jsd_build must be passed manually by flag
+
+- name: Source jira-servicedesk obr from alternate url
+  block:
+
+    - name: Alternate URL OBR - Show the obr filename
+      debug:
+        msg="obr_name=jira-servicedesk-application-{{ atl_jsd_build }}.obr"
+
+    - name: Alternate OBR - Set the obr filename
+      set_fact:
+        atl_obr_filename: "jira-servicedesk-application-{{ atl_jsd_build }}.obr"
+
+  when:
+    - not atl_source_obr_from_marketplace | bool
 
 - name: is shared_home set ?
   debug:
     msg="atl_product_home_shared_download_dir={{ atl_product_home_shared_download_dir }}"
+
 # For the first run a temp obr should be downloaded but moved to
 # shared home to ensure all subsequent nodes have access
 # to the same specific version binary.
@@ -38,12 +80,10 @@
   set_fact:
     download_obr: true
     move_obr: false
-    atl_obr_download_href: "{{ atl_jsd_build_info.json._embedded.artifact._links.binary.href }}"
-    atl_obr_filename: "jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr"
-    atl_obr_download: "{{ atl_installer_temp }}/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr"
-    atl_obr_shared_download: "{{ atl_product_home_shared_download_dir }}/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr"
-    atl_obr_moving_lock: "{{ atl_product_home_shared_download_dir }}/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr_moving"
-    atl_obr_completed_lock: "{{ atl_product_home_shared_download_dir }}/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.obr_completed"
+    atl_obr_download: "{{ atl_installer_temp }}/{{ atl_obr_filename }}"
+    atl_obr_shared_download: "{{ atl_product_home_shared_download_dir }}/{{ atl_obr_filename }}"
+    atl_obr_moving_lock: "{{ atl_product_home_shared_download_dir }}/{{ atl_obr_filename }}_moving"
+    atl_obr_completed_lock: "{{ atl_product_home_shared_download_dir }}/{{ atl_obr_filename }}_completed"
 
 # Check for pre-downloaded obr on shared_home and completed lock dir.
 - name: Check for completed lock directory
@@ -56,6 +96,10 @@
     path: "{{ atl_obr_shared_download }}"
   register: home_shared_download
 
+- name: debug home_shared_download
+  debug:
+    var: home_shared_download
+
 # If obr exists and lockdir exists use this obr instead
 - name: Check lock directory and obr exists on shared_home
   set_fact:
@@ -63,22 +107,47 @@
     atl_obr_download: "{{ atl_obr_shared_download }}"
   when:
     - home_shared_download.stat.exists
+    - home_shared_download.stat.mimetype is match("application/zip")
     - completed_lock.stat.isdir is defined
     - completed_lock.stat.isdir
 
-# Fetch obr if required
+# Fetch obr if required - note we validate it by mimetype rather than checksum due to https://ecosystem.atlassian.net/browse/AMKT-25526
 - name: download_obr is true so fetch and do all the things
   block:
+    - debug:
+        var: atl_obr_download_url
+    - debug:
+        var: atl_obr_download
 
     # Fetch obr and copy to temp
     - name: Fetch obr
       get_url:
-        url: "{{ atl_obr_download_href }}"
+        url: "{{ atl_obr_download_url }}"
         dest: "{{ atl_obr_download }}"
         mode: 0755
-        force: false
+        force: true
+        timeout: 600
       register: atl_obr_completed
 
+    - name: Confirm the output from the download task
+      debug:
+        var: atl_obr_completed
+
+    # get details about the obr
+    - name: Stat the new obr file
+      stat:
+        path: "{{ atl_obr_completed.dest }}"
+        get_mime: yes
+      register: atl_obr_stats
+      when:
+        - atl_obr_completed.dest is defined
+
+    - name: fail if the downloaded OBR is not a zip file
+      fail:
+        msg: "The downloaded OBR was not detected as being a valid ZIP file: {{ atl_obr_stats }}"
+      when:
+        - (atl_obr_stats.stat.mimetype is not defined) or (atl_obr_stats.stat.mimetype is not match("application/zip"))
+
     # If obr was fetched make the lock directory
     - name: Create moving_lock.
       file:
@@ -154,23 +223,30 @@
     group: "{{ atl_product_user }}"
 
 # Note as ansible unarchive cant handle "-j junk paths" we need to ignore errors to bypass the path verify
-- name: Unpack the obr into the installed-plugins dir
+- name: Unpack the obr into the atlassian-bundled-plugins dir
   unarchive:
-    remote_src: true
+    remote_src: yes
     src: "{{ atl_obr_download }}"
-    dest: "{{ atl_product_home_shared }}/plugins/installed-plugins"
-    creates: "{{ atl_product_home_shared }}/plugins/installed-plugins/jira-servicedesk-application-{{ atl_jsd_build_info.json.name }}.jar"
+    dest: "{{ atl_product_installation_versioned }}/atlassian-jira/WEB-INF/atlassian-bundled-plugins"
+    creates: "{{ atl_product_installation_versioned }}/atlassian-jira/WEB-INF/atlassian-bundled-plugins/jira-servicedesk-application-{{ atl_jsd_build }}.jar"
     list_files: no
     exclude:
       - M*
+      - obr.xml
     owner: "{{ atl_product_user }}"
     group: "{{ atl_product_user }}"
-    mode: 0750
+    mode: 0644
   register: obr_unpack
-  ignore_errors: yes
 
-- name: Move JSD dependency jars into the installed-plugins dir      # noqa 503 - ignore lint info about when changed
-  shell:
-    cmd: "mv {{ atl_product_home_shared }}/plugins/installed-plugins/dependencies/*.jar {{ atl_product_home_shared }}/plugins/installed-plugins"
-    creates: "{{ atl_product_home_shared }}/plugins/installed-plugins/servicedesk-core-ui-plugin-{{ atl_jsd_build_info.json.name }}-*.jar"
+- name: Move JSD dependency jars into the bundled-plugins dir      # noqa 503 - ignore lint info about when changed
+  copy:
+    remote_src: yes
+    src: "{{ atl_product_installation_versioned }}/atlassian-jira/WEB-INF/atlassian-bundled-plugins/dependencies/"
+    dest: "{{ atl_product_installation_versioned }}/atlassian-jira/WEB-INF/atlassian-bundled-plugins/"
   when: obr_unpack.changed
+
+- name: Remove the empty dependencies folder      # noqa 503 - ignore lint info about when changed
+  file:
+    path: "{{ atl_product_installation_versioned }}/atlassian-jira/WEB-INF/atlassian-bundled-plugins/dependencies"
+    state: absent
+  when: obr_unpack.changed