DCD-224: Rework of ServiceDesk handling to deal with more corner-cases, and limit it to the product_download role.

roles/jira_config/defaults/main.yml

@@ -24,21 +24,3 @@ atl_tomcat_protocol: "HTTP/1.1"
 atl_tomcat_redirectport: ""
 atl_tomcat_scheme: "http"
 atl_tomcat_secure: "false"
-
-# As SD releases need to be mapped to a Jira release, and there's now
-# straightforward way of doing this via the Marketplace REST API we do
-# a course mapping here and rely on UPM to prompt to upgrade to the
-# latest. See:
-#
-#     https://marketplace.atlassian.com/apps/1213632/jira-service-desk/version-history
-#
-# for version details.
-atl_servicedesk_download_map:
-  '8':
-    version: "4.1.0"
-    url: "https://marketplace.atlassian.com/download/apps/1213632/version/1040010000"
-  '7':
-    version: "3.16.3"
-    url: "https://marketplace.atlassian.com/download/apps/1213632/version/1030160003"
-
-atl_servicedesk_download_file: "{{ atl_installer_temp }}/jira-servicedesk-{{ atl_servicedesk_version }}.obr"

roles/jira_config/molecule/default/playbook.yml

@@ -4,7 +4,6 @@
   vars:
     atl_product_family: "jira"
     atl_product_edition: "jira-software"
-    atl_product_bundle: "software"
     atl_product_user: "jira"
     atl_product_version: "7.13.2"
     atl_jdbc_user: 'atljira'

roles/jira_config/molecule/servicedesk_with_jira7/playbook.yml

@@ -1,26 +0,0 @@
----
-- name: Converge
-  hosts: all
-
-  vars:
-    atl_product_bundle: servicedesk
-    bundle_map:
-      all: software
-      servicedesk: software
-      core: core
-      software: software
-
-    atl_product_edition: "jira-{{ bundle_map[atl_product_bundle] }}"
-    atl_product_family: "jira"
-    atl_product_user: "jira"
-    atl_product_version: "7.13.3"
-
-    atl_jdbc_user: 'atljira'
-    atl_jvm_heap: 'PLACEHOLDER'
-    atl_cluster_node_id: 'FAKEID'
-
-  roles:
-    - role: linux_common
-    - role: product_common
-    - role: product_download
-    - role: jira_config

roles/jira_config/molecule/servicedesk_with_latest/playbook.yml

@@ -1,26 +0,0 @@
----
-- name: Converge
-  hosts: all
-
-  vars:
-    atl_product_bundle: servicedesk
-    bundle_map:
-      all: software
-      servicedesk: software
-      core: core
-      software: software
-
-    atl_product_edition: "jira-{{ bundle_map[atl_product_bundle] }}"
-    atl_product_family: "jira"
-    atl_product_user: "jira"
-    atl_product_version: "latest"
-
-    atl_jdbc_user: 'atljira'
-    atl_jvm_heap: 'PLACEHOLDER'
-    atl_cluster_node_id: 'FAKEID'
-
-  roles:
-    - role: linux_common
-    - role: product_common
-    - role: product_download
-    - role: jira_config

roles/jira_config/tasks/main.yml

@@ -27,44 +27,3 @@
   with_items:
     - 'JVM_MINIMUM_MEMORY'
     - 'JVM_MAXIMUM_MEMORY'
-
-
-# FIXME: The following is a bit of a special-case for now; ideally we
-# should convert this into a general plugin-installation task
-# (esp. when we get to supporting Portfolio), but there's no use for ATM.
-- name: Install ServiceDesk if required
-  block:
-
-    - name: Extract the Jira major version, default to 8 if not found or 'latest'
-      set_fact:
-        atl_jira_major_version: "{{ atl_product_version | regex_search('^\\d+') | default('8', true) }}"
-    - name: Extract the latest version number
-      set_fact:
-        atl_servicedesk_version: "{{ atl_servicedesk_download_map[atl_jira_major_version].version }}"
-    - name: Extract the download url
-      set_fact:
-        atl_servicedesk_download_url: "{{ atl_servicedesk_download_map[atl_jira_major_version].url }}"
-
-    - name: Download ServiceDesk
-      get_url:
-        url: "{{ atl_servicedesk_download_url }}"
-        dest: "{{ atl_servicedesk_download_file }}"
-        mode: 0640
-        force: false
-
-    - name: Extract the plugin to the shared plugin directory
-      command: /usr/bin/unzip -jn "{{ atl_servicedesk_download_file }}" "*.jar"
-      args:
-        chdir: "{{ atl_product_shared_plugins }}"
-        creates: "{{ atl_product_shared_plugins }}/*servicedesk*.jar"
-        warn: false
-
-    - name: Make plugins writable by the Jira user
-      file:
-        path: "{{ atl_product_shared_plugins }}"
-        owner: "{{ atl_product_user }}"
-        group: "{{ atl_product_user }}"
-        mode: 0750
-        recurse: true
-
-  when: atl_product_bundle == "servicedesk"

roles/product_download/defaults/main.yml

@@ -5,6 +5,17 @@ atl_product_version_cache: "{{ atl_product_home_shared }}/{{ atl_product_edition
 
 atl_release_base_url: "https://product-downloads.atlassian.com/software"
 atl_product_base_url: "{{ atl_release_base_url }}/{{ atl_product_family }}/downloads"
-atl_product_download_url: "{{ atl_product_base_url }}/atlassian-{{ atl_product_edition }}-{{ atl_product_version }}.tar.gz"
-atl_product_download_filename: "{{ atl_product_edition }}.{{ atl_product_version }}.tar.gz"
+atl_product_download_url: "{{ atl_product_base_url }}/atlassian-{{ atl_download_edition | default(atl_product_edition) }}-{{ atl_download_version | default(atl_product_version) }}.tar.gz"
+
+atl_product_download_filename: "{{ atl_download_edition | default(atl_product_edition) }}.{{ atl_download_version | default(atl_product_version) }}.tar.gz"
 atl_product_download: "{{ atl_installer_temp }}/{{ atl_product_download_filename }}"
+
+atl_marketplace_base: "https://marketplace.atlassian.com"
+atl_servicedesk_latest_url: "https://marketplace.atlassian.com/rest/2/products/key/jira-servicedesk/versions/latest"
+atl_servicedesk_versioned_url: "https://marketplace.atlassian.com/rest/2/products/key/jira-servicedesk/versions/name/{{ atl_product_version }}"
+atl_servicedesk_url_map:
+  latest: "{{ atl_servicedesk_latest_url }}"
+  default: "{{ atl_servicedesk_versioned_url }}"
+atl_servicedesk_metadata_url: "{{ atl_servicedesk_url_map[atl_product_version] | default(atl_servicedesk_url_map['default']) }}"
+
+atl_servicedesk_download_file: "{{ atl_installer_temp }}/jira-servicedesk.{{ atl_product_version }}.obr"

roles/product_download/molecule/servicedesk/playbook.yml

@@ -1,19 +0,0 @@
----
-- name: Converge
-  hosts: all
-  vars:
-    atl_product_bundle: servicedesk
-    bundle_map:
-      all: software
-      servicedesk: software
-      core: core
-      software: software
-
-    atl_product_edition: "jira-{{ bundle_map[atl_product_bundle] }}"
-    atl_product_family: "jira"
-    atl_product_user: "jira"
-
-  roles:
-    - role: linux_common
-    - role: product_common
-    - role: product_download

roles/product_download/molecule/servicedesk/tests/test_default.py

@@ -1,26 +0,0 @@
-import os
-import urllib.request
-
-import testinfra.utils.ansible_runner
-
-testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
-    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
-
-
-def test_version_is_correct(host):
-    verfile = host.file('/media/atl/jira/shared/jira-software.version')
-    assert verfile.exists
-
-    assert verfile.content.decode("UTF-8").strip() == "8.1.0"
-
-def test_is_downloaded(host):
-    installer = host.file('/opt/atlassian/tmp/jira-software.8.1.0.tar.gz')
-    assert installer.exists
-    assert installer.user == 'root'
-
-def test_is_unpacked(host):
-    installer = host.file('/opt/atlassian/jira-software/8.1.0')
-    assert installer.exists
-    assert installer.is_directory
-    assert installer.user == 'jira'
-    assert installer.mode == 0o0755

roles/product_download/molecule/servicedesk3/molecule.yml

@@ -17,6 +17,7 @@ platforms:
 provisioner:
   name: ansible
   options:
+    vv: true
     skip-tags: runtime_pkg
   lint:
     name: ansible-lint

roles/product_download/molecule/servicedesk3/playbook.yml

@@ -0,0 +1,13 @@
+---
+- name: Converge
+  hosts: all
+  vars:
+    atl_product_edition: "jira-servicedesk"
+    atl_product_family: "jira"
+    atl_product_user: "jira"
+    atl_product_version: "3.0.2"
+
+  roles:
+    - role: linux_common
+    - role: product_common
+    - role: product_download

roles/product_download/molecule/servicedesk3/tests/test_default.py

@@ -8,30 +8,30 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
 
 
 def test_version_is_correct(host):
-    verfile = host.file('/media/atl/jira/shared/jira-software.version')
+    verfile = host.file('/media/atl/jira/shared/jira-servicedesk.version')
     assert verfile.exists
 
-    assert verfile.content.decode("UTF-8").strip() == "7.13.3"
+    assert verfile.content.decode("UTF-8").strip() == "3.0.2"
 
 def test_is_downloaded(host):
-    installer = host.file('/opt/atlassian/tmp/jira-software.7.13.3.tar.gz')
+    installer = host.file('/opt/atlassian/tmp/jira-software.7.0.11.tar.gz')
     assert installer.exists
     assert installer.user == 'root'
 
 def test_is_unpacked(host):
-    installer = host.file('/opt/atlassian/jira-software/7.13.3')
+    installer = host.file('/opt/atlassian/jira-servicedesk/3.0.2')
     assert installer.exists
     assert installer.is_directory
     assert installer.user == 'jira'
     assert installer.mode == 0o0755
 
 def test_sd_is_downloaded(host):
-    installer = host.file('/opt/atlassian/tmp/jira-servicedesk-3.16.3.obr')
+    installer = host.file('/opt/atlassian/tmp/jira-servicedesk.3.0.2.obr')
     assert installer.exists
     assert installer.user == 'root'
 
 def test_is_unpacked(host):
-    installer = host.file('/media/atl/jira/shared/plugins/installed-plugins/jira-servicedesk-application-3.16.3.jar')
+    installer = host.file('/media/atl/jira/shared/plugins/installed-plugins/jira-servicedesk-application-3.0.2.jar')
     assert installer.exists
     assert installer.user == 'jira'
     assert installer.mode == 0o0750

roles/product_download/molecule/servicedesk4/molecule.yml

@@ -17,6 +17,7 @@ platforms:
 provisioner:
   name: ansible
   options:
+    vv: true
     skip-tags: runtime_pkg
   lint:
     name: ansible-lint

roles/product_download/molecule/servicedesk4/playbook.yml

@@ -0,0 +1,13 @@
+---
+- name: Converge
+  hosts: all
+  vars:
+    atl_product_edition: "jira-servicedesk"
+    atl_product_family: "jira"
+    atl_product_user: "jira"
+    atl_product_version: "4.1.0"
+
+  roles:
+    - role: linux_common
+    - role: product_common
+    - role: product_download

roles/product_download/molecule/servicedesk4/tests/test_default.py

@@ -8,10 +8,10 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
 
 
 def test_version_is_correct(host):
-    verfile = host.file('/media/atl/jira/shared/jira-software.version')
+    verfile = host.file('/media/atl/jira/shared/jira-servicedesk.version')
     assert verfile.exists
 
-    assert verfile.content.decode("UTF-8").strip() == "8.1.0"
+    assert verfile.content.decode("UTF-8").strip() == "4.1.0"
 
 def test_is_downloaded(host):
     installer = host.file('/opt/atlassian/tmp/jira-software.8.1.0.tar.gz')
@@ -19,14 +19,14 @@ def test_is_downloaded(host):
     assert installer.user == 'root'
 
 def test_is_unpacked(host):
-    installer = host.file('/opt/atlassian/jira-software/8.1.0')
+    installer = host.file('/opt/atlassian/jira-servicedesk/4.1.0')
     assert installer.exists
     assert installer.is_directory
     assert installer.user == 'jira'
     assert installer.mode == 0o0755
 
 def test_sd_is_downloaded(host):
-    installer = host.file('/opt/atlassian/tmp/jira-servicedesk-4.1.0.obr')
+    installer = host.file('/opt/atlassian/tmp/jira-servicedesk.4.1.0.obr')
     assert installer.exists
     assert installer.user == 'root'
 

roles/product_download/molecule/servicedesk_latest/molecule.yml

@@ -17,6 +17,7 @@ platforms:
 provisioner:
   name: ansible
   options:
+    vv: true
     skip-tags: runtime_pkg
   lint:
     name: ansible-lint

roles/product_download/molecule/servicedesk_latest/playbook.yml

@@ -0,0 +1,13 @@
+---
+- name: Converge
+  hosts: all
+  vars:
+    atl_product_edition: "jira-servicedesk"
+    atl_product_family: "jira"
+    atl_product_user: "jira"
+    atl_product_version: "latest"
+
+  roles:
+    - role: linux_common
+    - role: product_common
+    - role: product_download

roles/product_download/molecule/servicedesk_latest/tests/test_default.py

@@ -0,0 +1,46 @@
+import os
+import json
+import urllib.request
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+def get_latest():
+    data = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/products/key/jira-servicedesk/versions/latest")
+    meta = json.loads(data.read().decode("UTF-8"))
+    servicedesk = meta['name']
+    jira = meta['compatibilities'][0]['hosting']['server']['max']['version']
+    return (servicedesk, jira)
+
+(sd, jira) = get_latest()
+
+def test_version_is_correct(host):
+    verfile = host.file('/media/atl/jira/shared/jira-servicedesk.version')
+    assert verfile.exists
+
+    assert verfile.content.decode("UTF-8").strip() == sd
+
+def test_is_downloaded(host):
+    installer = host.file('/opt/atlassian/tmp/jira-software.'+jira+'.tar.gz')
+    assert installer.exists
+    assert installer.user == 'root'
+
+def test_is_unpacked(host):
+    installer = host.file('/opt/atlassian/jira-servicedesk/'+jira)
+    assert installer.exists
+    assert installer.is_directory
+    assert installer.user == 'jira'
+    assert installer.mode == 0o0755
+
+def test_sd_is_downloaded(host):
+    installer = host.file('/opt/atlassian/tmp/jira-servicedesk.'+sd+'.obr')
+    assert installer.exists
+    assert installer.user == 'root'
+
+def test_is_unpacked(host):
+    installer = host.file('/media/atl/jira/shared/plugins/installed-plugins/jira-servicedesk-application-'+sd+'.jar')
+    assert installer.exists
+    assert installer.user == 'jira'
+    assert installer.mode == 0o0750

roles/product_download/tasks/cached_version_fetch.yml

@@ -0,0 +1,21 @@
+---
+
+- name: Check for existing version cache file
+  stat:
+    path: "{{ atl_product_version_cache }}"
+  register: cached
+
+# Case: File exists, always use its value
+- name: Use version for product version
+  block:
+
+    - name: Read cached version from file
+      command: "cat {{ atl_product_version_cache }}"
+      register: atl_product_version_file
+      changed_when: false
+
+    - name: Set the local var to cached version
+      set_fact:
+        atl_product_version: "{{ atl_product_version_file.stdout }}"
+
+  when: cached.stat.exists

roles/product_download/tasks/cached_version_write.yml

@@ -0,0 +1,8 @@
+---
+
+# At this point atl_product_version should be set, cache if necessary.
+- name: Write override cached version when specified
+  template:
+    src: version.j2
+    dest: "{{ atl_product_version_cache }}"
+    force: false

roles/product_download/tasks/fetch_product.yml

@@ -0,0 +1,38 @@
+---
+
+# Note: We don't the cache binary in the shared drive to the complexity
+# around download race-conditions if multiple nodes are starting at
+# the same time. When downloading from product-downloads.atlassian.com
+# (which is a CDN) takes seconds anyway.
+- name: Fetch product installer
+  get_url:
+    url: "{{ atl_product_download_url }}"
+    dest: "{{ atl_product_download }}"
+    mode: 0644
+    force: false
+
+- name: Create target version directory
+  file:
+    path: "{{ atl_product_installation_base }}/{{ atl_product_version }}"
+    state: directory
+    owner: "{{ atl_product_user }}"
+
+- name: Unpack the product packages
+  unarchive:
+    remote_src: true
+    src: "{{ atl_product_download }}"
+    dest: "{{ atl_product_installation_base }}/{{ atl_product_version }}"
+    creates: "{{ atl_product_installation_base }}/{{ atl_product_version }}/bin/catalina.sh"
+    owner: "{{ atl_product_user }}"
+    group: "{{ atl_product_user }}"
+    mode: 0755
+    # Strip off the lead product/version specific directory to normaise naming
+    extra_opts:
+      - "--strip-components=1"
+
+- name: Symlink the installed version to current
+  file:
+    src: "{{ atl_product_installation_base }}/{{ atl_product_version }}"
+    dest: "{{ atl_product_installation_target }}"
+    state: link
+    force: true

roles/product_download/tasks/main.yml

@@ -1,86 +1,13 @@
 ---
 
-- name: Check for existing version cache file
-  stat:
-    path: "{{ atl_product_version_cache }}"
-  register: cached
+# Common case for most products and Jira editions.
+- name: Fetch ServiceDesk metadata if necessary
+  include_tasks: "servicedesk_download.yml"
+  when: atl_product_edition == "jira-servicedesk"
 
-# Case: File exists, always use its value
-- name: Use version for product version
-  block:
-
-    - name: Read cached version from file
-      command: "cat {{ atl_product_version_cache }}"
-      register: atl_product_version_file
-      changed_when: false
-
-    - name: Set the local var to cached version
-      set_fact:
-        atl_product_version: "{{ atl_product_version_file.stdout }}"
-
-  when: cached.stat.exists
-
-
-# Case: File doesn't exist and no version has been set; find latest.
-- name: Fetch and cache latest version when no override
-  block:
-
-    - name: Fetch the latest version from URL
-      set_fact:
-        atl_product_version_json: "{{ lookup('url', '{{ atl_product_latest_version_url }}') }}"
-
-    - name: Set the local var to retrieved version
-      set_fact:
-        atl_product_version: "{{ atl_product_version_json.version }}"
-
-  when:
-    not cached.stat.exists and
-    (atl_product_version is undefined or
-     not atl_product_version or
-     atl_product_version == "latest")
-
-
-# At this point atl_product_version should be set, cache if necessary.
-- name: Write override cached version when specified
-  template:
-    src: version.j2
-    dest: "{{ atl_product_version_cache }}"
-    force: false
-
-
-# Note: We don't cache this in the shared drive to the complexity
-# around download race-conditions if multiple nodes are starting at
-# the same time. When downloading from product-downloads.atlassian.com
-# (which is a CDN) takes seconds anyway.
-- name: Fetch product installer
-  get_url:
-    url: "{{ atl_product_download_url }}"
-    dest: "{{ atl_product_download }}"
-    mode: 0644
-    force: false
-
-- name: Create target version directory
-  file:
-    path: "{{ atl_product_installation_base }}/{{ atl_product_version }}"
-    state: directory
-    owner: "{{ atl_product_user }}"
-
-- name: Unpack the product packages
-  unarchive:
-    remote_src: true
-    src: "{{ atl_product_download }}"
-    dest: "{{ atl_product_installation_base }}/{{ atl_product_version }}"
-    creates: "{{ atl_product_installation_base }}/{{ atl_product_version }}/bin/catalina.sh"
-    owner: "{{ atl_product_user }}"
-    group: "{{ atl_product_user }}"
-    mode: 0755
-    # Strip off the lead product/version specific directory to normaise naming
-    extra_opts:
-      - "--strip-components=1"
-
-- name: Symlink the installed version to current
-  file:
-    src: "{{ atl_product_installation_base }}/{{ atl_product_version }}"
-    dest: "{{ atl_product_installation_target }}"
-    state: link
-    force: true
+# Differences for ServiceDesk become fiddly enough that it's better to
+# just split them out and move common functionality into individual
+# task files.
+- name: Fetch product metadata if necessary
+  include_tasks: "product_download.yml"
+  when: atl_product_edition != "jira-servicedesk"

roles/product_download/tasks/product_download.yml

@@ -0,0 +1,31 @@
+---
+
+- name: Check for and load cached version
+  include_tasks: "cached_version_fetch.yml"
+
+
+# Case: File doesn't exist and no version has been set; find latest.
+- name: Fetch and cache latest version when no override
+  block:
+
+    - name: Fetch the latest version from URL
+      set_fact:
+        atl_product_version_json: "{{ lookup('url', '{{ atl_product_latest_version_url }}') }}"
+
+    - name: Set the local var to retrieved version
+      set_fact:
+        atl_product_version: "{{ atl_product_version_json.version }}"
+
+  when:
+    not cached.stat.exists and
+    (atl_product_version is undefined or
+     not atl_product_version or
+     atl_product_version == "latest")
+
+
+- name: Cache download version if necessary
+  include_tasks: "cached_version_write.yml"
+
+
+- name: Fetch and unpack the product distribution
+  include_tasks: "fetch_product.yml"

roles/product_download/tasks/servicedesk_download.yml

@@ -0,0 +1,71 @@
+---
+
+- name: Check for and load cached version
+  include_tasks: "cached_version_fetch.yml"
+
+
+# Case: File doesn't exist and no version has been set; find latest.
+- name: Fetch and cache latest version when no override
+  block:
+
+    - name: Fetch the ServiceDesk metadata from marketplace
+      set_fact:
+        atl_servicedesk_version_json: "{{ lookup('url', '{{ atl_servicedesk_latest_url }}') }}"
+
+    - name: Set the Jira version for ServiceDesk
+      set_fact:
+        atl_product_version: "{{ atl_servicedesk_version_json.name }}"
+        atl_download_version: "{{ atl_servicedesk_version_json.name }}"
+
+  when:
+    not cached.stat.exists and
+    (atl_product_version is undefined or
+     not atl_product_version or
+     atl_product_version == "latest")
+
+
+- name: Cache download version if necessary
+  include_tasks: "cached_version_write.yml"
+
+
+# We now have a ServiceDesk version, fetch the metadata to give us the
+# download URL and Jira versioning information.
+- name: Fetch the ServiceDesk metadata from marketplace
+  set_fact:
+    atl_servicedesk_version_json: "{{ lookup('url', '{{ atl_servicedesk_versioned_url }}') }}"
+
+- name: Extract the version and download information from the ServiceDesk metadata
+  set_fact:
+    atl_download_edition: "jira-software"
+    atl_download_version: "{{ atl_servicedesk_version_json.compatibilities[0].hosting.server.max.version }}"
+    atl_servicedesk_download_url: "{{ atl_servicedesk_version_json._embedded.artifact._links.binary.href }}"
+
+
+- name: Fetch and unpack the Jira edition
+  include_tasks: "fetch_product.yml"
+
+
+# FIXME: The following is a bit of a special-case for now; ideally we
+# should convert this into a general plugin-installation role
+# (esp. when we get to supporting Portfolio), but there's no use for ATM.
+- name: Download ServiceDesk
+  get_url:
+    url: "{{ atl_servicedesk_download_url }}"
+    dest: "{{ atl_servicedesk_download_file }}"
+    mode: 0640
+    force: false
+
+- name: Extract the plugin to the shared plugin directory
+  command: /usr/bin/unzip -jn "{{ atl_servicedesk_download_file }}" "*.jar"
+  args:
+    chdir: "{{ atl_product_shared_plugins }}"
+    creates: "{{ atl_product_shared_plugins }}/*servicedesk*.jar"
+    warn: false
+
+- name: Make plugins writable by the Jira user
+  file:
+    path: "{{ atl_product_shared_plugins }}"
+    owner: "{{ atl_product_user }}"
+    group: "{{ atl_product_user }}"
+    mode: 0750
+    recurse: true