Merge branch 'master' into feature/ITOPSENG-101-support-JiraProduct-All

roles/product_install/defaults/main.yml

@@ -19,9 +19,14 @@ atl_product_base_url: "{{ atl_release_base_url }}/{{ atl_product_family }}/downl
 atl_product_download_url: "{{ atl_product_base_url }}/atlassian-{{ atl_download_edition | default(atl_product_edition) }}-{{ atl_product_version }}{{ atl_download_suffix }}"
 
 atl_product_download_filename: "{{ atl_download_edition | default(atl_product_edition) }}.{{ atl_product_version }}{{ atl_download_suffix }}"
-atl_product_download: "{{ atl_installer_temp }}/{{ atl_product_download_filename }}"
+atl_product_temp_download: "{{ atl_installer_temp }}/{{ atl_product_download_filename }}"
 atl_product_varfile: "{{ atl_installer_temp }}/{{ atl_product_family }}.varfile"
 
+atl_product_home_shared_download_dir: "{{ atl_product_home_shared }}/downloads"
+atl_product_home_shared_download: "{{ atl_product_home_shared_download_dir }}/{{ atl_product_download_filename }}"
+atl_product_home_shared_moving_lock: "{{ atl_product_home_shared_download }}_moving"
+atl_product_home_shared_completed_lock: "{{ atl_product_home_shared_download }}_completed"
+
 atl_marketplace_base: "https://marketplace.atlassian.com"
 atl_mpac_products: "https://marketplace.atlassian.com/rest/2/products"
 atl_servicedesk_latest_url: "https://marketplace.atlassian.com/rest/2/products/key/jira-servicedesk/versions/latest"

roles/product_install/molecule/bitbucket_latest/tests/test_default.py

@@ -37,6 +37,16 @@ def test_latest_is_downloaded(host):
     upstream_json = json.load(upstream_fd)
     upstream = upstream_json['version']
 
-    installer = host.file('/opt/atlassian/tmp/bitbucket.' + upstream + '-x64.bin')
+    installer = host.file('/media/atl/bitbucket/shared/downloads/bitbucket.' + upstream + '-x64.bin')
     assert installer.exists
     assert installer.user == 'root'
+
+def test_completed_lockfile(host):
+    upstream_fd = urllib.request.urlopen(
+        "https://marketplace.atlassian.com/rest/2/applications/bitbucket/versions/latest")
+    upstream_json = json.load(upstream_fd)
+    upstream = upstream_json['version']
+
+    lockfile = host.file('/media/atl/bitbucket/shared/downloads/bitbucket.' + upstream + '-x64.bin_completed')
+    assert lockfile.exists
+    assert lockfile.user == 'root'

roles/product_install/molecule/confluence_latest/tests/test_default.py

@@ -35,6 +35,15 @@ def test_latest_is_downloaded(host):
     upstream_json = json.load(upstream_fd)
     upstream = upstream_json['version']
 
-    installer = host.file('/opt/atlassian/tmp/confluence.'+upstream+'-x64.bin')
+    installer = host.file('/media/atl/confluence/shared-home/downloads/confluence.'+upstream+'-x64.bin')
     assert installer.exists
     assert installer.user == 'root'
+
+def test_completed_lockfile(host):
+    upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/confluence/versions/latest")
+    upstream_json = json.load(upstream_fd)
+    upstream = upstream_json['version']
+
+    lockfile = host.file('/media/atl/confluence/shared-home/downloads/confluence.'+upstream+'-x64.bin_completed')
+    assert lockfile.exists
+    assert lockfile.user == 'root'

roles/product_install/molecule/crowd_latest/Dockerfile.j2

@@ -0,0 +1,14 @@
+# Molecule managed
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi

roles/product_install/molecule/crowd_latest/molecule.yml

@@ -0,0 +1,30 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+lint:
+  name: yamllint
+platforms:
+  - name: amazon_linux2
+    image: amazonlinux:2
+    groups:
+      - aws_node_local
+  - name: ubuntu_lts
+    image: ubuntu:bionic
+    groups:
+      - aws_node_local
+provisioner:
+  name: ansible
+  options:
+    skip-tags: runtime_pkg
+  lint:
+    name: ansible-lint
+  inventory:
+    links:
+      group_vars: ../../../../group_vars/
+verifier:
+  name: testinfra
+  lint:
+    name: flake8
+    enabled: false

roles/product_install/molecule/crowd_latest/playbook.yml

@@ -0,0 +1,12 @@
+---
+- name: Converge
+  hosts: all
+  vars:
+    atl_product_family: "crowd"
+    atl_product_edition: "crowd"
+    atl_product_user: "crowd"
+    atl_download_format: "tarball"
+  roles:
+    - role: linux_common
+    - role: product_common
+    - role: product_install

roles/product_install/molecule/crowd_latest/tests/test_default.py

@@ -0,0 +1,52 @@
+import os
+from six.moves import urllib
+import json
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+def test_version_downloaded(host):
+    verfile = host.file('/media/atl/crowd/shared/crowd.version')
+    assert verfile.exists
+
+def test_symlink_created(host):
+    target = host.file('/opt/atlassian/crowd/current')
+    assert target.exists
+    assert target.is_symlink
+
+def test_unpacked(host):
+    verfile = host.file('/opt/atlassian/crowd/current/start_crowd.sh')
+    assert verfile.exists
+
+def test_version_file_is_latest(host):
+    verfile = host.file('/media/atl/crowd/shared/crowd.version')
+    assert verfile.exists
+
+    upstream_fd = urllib.request.urlopen(
+        "https://marketplace.atlassian.com/rest/2/applications/crowd/versions/latest")
+    upstream_json = json.load(upstream_fd)
+    upstream = upstream_json['version']
+
+    assert verfile.content.decode("UTF-8").strip() == upstream.strip()
+
+def test_latest_is_downloaded(host):
+    upstream_fd = urllib.request.urlopen(
+        "https://marketplace.atlassian.com/rest/2/applications/crowd/versions/latest")
+    upstream_json = json.load(upstream_fd)
+    upstream = upstream_json['version']
+
+    installer = host.file('/media/atl/crowd/shared/downloads/crowd.' + upstream + '.tar.gz')
+    assert installer.exists
+    assert installer.user == 'root'
+
+def test_completed_lockfile(host):
+    upstream_fd = urllib.request.urlopen(
+        "https://marketplace.atlassian.com/rest/2/applications/crowd/versions/latest")
+    upstream_json = json.load(upstream_fd)
+    upstream = upstream_json['version']
+
+    lockfile = host.file('/media/atl/crowd/shared/downloads/crowd.' + upstream + '.tar.gz_completed')
+    assert lockfile.exists
+    assert lockfile.user == 'root'

roles/product_install/molecule/default/tests/test_default.py

@@ -23,6 +23,15 @@ def test_latest_is_downloaded(host):
     upstream_json = json.load(upstream_fd)
     upstream = upstream_json['version']
 
-    installer = host.file('/opt/atlassian/tmp/jira-core.'+upstream+'-x64.bin')
+    installer = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'-x64.bin')
     assert installer.exists
     assert installer.user == 'root'
+
+def test_completed_lockfile(host):
+    upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/jira/versions/latest")
+    upstream_json = json.load(upstream_fd)
+    upstream = upstream_json['version']
+
+    lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'-x64.bin_completed')
+    assert lockfile.exists
+    assert lockfile.user == 'root'

roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py

@@ -14,10 +14,15 @@ def test_version_is_correct(host):
     assert verfile.content.decode("UTF-8").strip() == "7.10.2"
 
 def test_is_downloaded(host):
-    installer = host.file('/opt/atlassian/tmp/jira-core.7.10.2-x64.bin')
+    installer = host.file('/media/atl/jira/shared/downloads/jira-core.7.10.2-x64.bin')
     assert installer.exists
     assert installer.user == 'root'
 
+def test_completed_lockfile(host):
+    lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.7.10.2-x64.bin_completed')
+    assert lockfile.exists
+    assert lockfile.user == 'root'
+
 def test_is_unpacked(host):
     installer = host.file('/opt/atlassian/jira-core/7.10.2/atlassian-jira/')
     assert installer.exists

roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py

@@ -14,10 +14,15 @@ def test_version_is_correct(host):
     assert verfile.content.decode("UTF-8").strip() == "7.10.1"
 
 def test_is_downloaded(host):
-    installer = host.file('/opt/atlassian/tmp/jira-core.7.10.1-x64.bin')
+    installer = host.file('/media/atl/jira/shared/downloads/jira-core.7.10.1-x64.bin')
     assert installer.exists
     assert installer.user == 'root'
 
+def test_completed_lockfile(host):
+    lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.7.10.1-x64.bin_completed')
+    assert lockfile.exists
+    assert lockfile.user == 'root'
+
 def test_is_unpacked(host):
     installer = host.file('/opt/atlassian/jira-core/7.10.1/atlassian-jira/')
     assert installer.exists

roles/product_install/molecule/jira_software_latest/tests/test_default.py

@@ -35,6 +35,15 @@ def test_latest_is_downloaded(host):
     upstream_json = json.load(upstream_fd)
     upstream = upstream_json['version']
 
-    installer = host.file('/opt/atlassian/tmp/jira-software.'+upstream+'-x64.bin')
+    installer = host.file('/media/atl/jira/shared/downloads/jira-software.'+upstream+'-x64.bin')
     assert installer.exists
     assert installer.user == 'root'
+
+def test_completed_lockfile(host):
+    upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/jira/versions/latest")
+    upstream_json = json.load(upstream_fd)
+    upstream = upstream_json['version']
+
+    lockfile = host.file('/media/atl/jira/shared/downloads/jira-software.'+upstream+'-x64.bin_completed')
+    assert lockfile.exists
+    assert lockfile.user == 'root'

roles/product_install/molecule/jira_tarball/tests/test_default.py

@@ -23,6 +23,15 @@ def test_latest_is_downloaded(host):
     upstream_json = json.load(upstream_fd)
     upstream = upstream_json['version']
 
-    installer = host.file('/opt/atlassian/tmp/jira-core.'+upstream+'.tar.gz')
+    installer = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'.tar.gz')
     assert installer.exists
     assert installer.user == 'root'
+
+def test_completed_lockfile(host):
+    upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/jira/versions/latest")
+    upstream_json = json.load(upstream_fd)
+    upstream = upstream_json['version']
+
+    lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'.tar.gz_completed')
+    assert lockfile.exists
+    assert lockfile.user == 'root'

roles/product_install/molecule/jira_version_from_file/tests/test_default.py

@@ -14,10 +14,15 @@ def test_version_is_correct(host):
     assert verfile.content.decode("UTF-8").strip() == "7.9.0"
 
 def test_is_downloaded(host):
-    installer = host.file('/opt/atlassian/tmp/jira-core.7.9.0-x64.bin')
+    installer = host.file('/media/atl/jira/shared/downloads/jira-core.7.9.0-x64.bin')
     assert installer.exists
     assert installer.user == 'root'
 
+def test_completed_lockfile(host):
+    lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.7.9.0-x64.bin_completed')
+    assert lockfile.exists
+    assert lockfile.user == 'root'
+
 def test_is_unpacked(host):
     installer = host.file('/opt/atlassian/jira-core/7.9.0/atlassian-jira/')
     assert installer.exists

roles/product_install/molecule/jira_version_latest/tests/test_default.py

@@ -23,6 +23,15 @@ def test_latest_is_downloaded(host):
     upstream_json = json.load(upstream_fd)
     upstream = upstream_json['version']
 
-    installer = host.file('/opt/atlassian/tmp/jira-core.'+upstream+'-x64.bin')
+    installer = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'-x64.bin')
     assert installer.exists
     assert installer.user == 'root'
+
+def test_completed_lockfile(host):
+    upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/jira/versions/latest")
+    upstream_json = json.load(upstream_fd)
+    upstream = upstream_json['version']
+
+    lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.'+upstream+'-x64.bin_completed')
+    assert lockfile.exists
+    assert lockfile.user == 'root'

roles/product_install/molecule/jira_version_override/tests/test_default.py

@@ -14,10 +14,15 @@ def test_version_is_correct(host):
     assert verfile.content.decode("UTF-8").strip() == "7.13.2"
 
 def test_is_downloaded(host):
-    installer = host.file('/opt/atlassian/tmp/jira-core.7.13.2-x64.bin')
+    installer = host.file('/media/atl/jira/shared/downloads/jira-core.7.13.2-x64.bin')
     assert installer.exists
     assert installer.user == 'root'
 
+def test_completed_lockfile(host):
+    lockfile = host.file('/media/atl/jira/shared/downloads/jira-core.7.13.2-x64.bin_completed')
+    assert lockfile.exists
+    assert lockfile.user == 'root'
+
 def test_is_unpacked(host):
     installer = host.file('/opt/atlassian/jira-core/7.13.2')
     assert installer.exists

roles/product_install/molecule/servicedesk3/tests/test_default.py

@@ -14,10 +14,15 @@ def test_version_is_correct(host):
     assert verfile.content.decode("UTF-8").strip() == "3.9.0"
 
 def test_is_downloaded(host):
-    installer = host.file('/opt/atlassian/tmp/servicedesk.3.9.0-x64.bin')
+    installer = host.file('/media/atl/jira/shared/downloads/servicedesk.3.9.0-x64.bin')
     assert installer.exists
     assert installer.user == 'root'
 
+def test_completed_lockfile(host):
+    lockfile = host.file('/media/atl/jira/shared/downloads/servicedesk.3.9.0-x64.bin_completed')
+    assert lockfile.exists
+    assert lockfile.user == 'root'
+
 def test_is_unpacked(host):
     installer = host.file('/opt/atlassian/jira-servicedesk/3.9.0')
     assert installer.exists

roles/product_install/molecule/servicedesk4/tests/test_default.py

@@ -14,10 +14,15 @@ def test_version_is_correct(host):
     assert verfile.content.decode("UTF-8").strip() == "4.1.0"
 
 def test_is_downloaded(host):
-    installer = host.file('/opt/atlassian/tmp/servicedesk.4.1.0-x64.bin')
+    installer = host.file('/media/atl/jira/shared/downloads/servicedesk.4.1.0-x64.bin')
     assert installer.exists
     assert installer.user == 'root'
 
+def test_completed_lockfile(host):
+    lockfile = host.file('/media/atl/jira/shared/downloads/servicedesk.4.1.0-x64.bin_completed')
+    assert lockfile.exists
+    assert lockfile.user == 'root'
+
 def test_is_unpacked(host):
     installer = host.file('/opt/atlassian/jira-servicedesk/4.1.0')
     assert installer.exists

roles/product_install/molecule/servicedesk_latest/tests/test_default.py

@@ -23,10 +23,15 @@ def test_version_is_correct(host):
     assert verfile.content.decode("UTF-8").strip() == sd
 
 def test_is_downloaded(host):
-    installer = host.file('/opt/atlassian/tmp/servicedesk.'+sd+'-x64.bin')
+    installer = host.file('/media/atl/jira/shared/downloads/servicedesk.'+sd+'-x64.bin')
     assert installer.exists
     assert installer.user == 'root'
 
+def test_completed_lockfile(host):
+    lockfile = host.file('/media/atl/jira/shared/downloads/servicedesk.'+sd+'-x64.bin_completed')
+    assert lockfile.exists
+    assert lockfile.user == 'root'
+
 def test_is_unpacked(host):
     installer = host.file('/opt/atlassian/jira-servicedesk/'+sd)
     assert installer.exists

roles/product_install/tasks/main.yml

@@ -116,6 +116,7 @@
     - "{{ atl_product_home }}"
     - "{{ atl_product_installation_versioned }}"
     - "{{ atl_product_version_cache_dir }}"
+    - "{{ atl_product_home_shared_download_dir }}"
   changed_when: false  # For Molecule idempotence check
 
 # At this point atl_product_version should be set, cache if necessary.
@@ -125,18 +126,120 @@
     dest: "{{ atl_product_version_cache }}"
     force: true
 
+# For the first run a temp binary should be downloaded but moved to
+# shared home to ensure all subsequent nodes have access
+# to the same specific version binary.
+# To prevent a race condition with multiple downloads at the same time
+# a directory is used as a lockfile (atomic operation) when moving binary.
 
-# Note: We don't the cache binary in the shared drive to the complexity
-# around download race-conditions if multiple nodes are starting at
-# the same time. When downloading from product-downloads.atlassian.com
-# (which is a CDN) takes seconds anyway.
-- name: Fetch product installer
-  get_url:
-    url: "{{ atl_product_download_url }}"
-    dest: "{{ atl_product_download }}"
-    mode: 0755
-    force: false
+- name: Set assumptions to avoid race condition
+  set_fact:
+    download_binary: true
+    move_binary: false
+    atl_product_download: "{{ atl_product_temp_download }}"
 
+# Check for pre-downloaded binary on shared_home and completed lock dir.
+- name: Check for completed lock directory
+  stat:
+    path: "{{ atl_product_home_shared_completed_lock }}"
+  register: completed_lock
+
+- name: Check for product installer in home_shared
+  stat:
+    path: "{{ atl_product_home_shared_download }}"
+  register: home_shared_download
+
+# If binary exists and lockdir exists use this binary instead
+- name: Check lock directory and binary exists on shared_home
+  set_fact:
+    download_binary: false
+    atl_product_download: "{{ atl_product_home_shared_download }}"
+  when:
+    - home_shared_download.stat.exists
+    - completed_lock.stat.isdir is defined
+    - completed_lock.stat.isdir
+
+# Fetch binary if required
+- name: download_binary is true so fetch and do all the things
+  block:
+
+    # Fetch binary and copy to temp
+    - name: Fetch binary
+      get_url:
+        url: "{{ atl_product_download_url }}"
+        dest: "{{ atl_product_temp_download }}"
+        mode: 0755
+        force: false
+      register: atl_product_completed
+
+    # If product installer was fetched make the lock directory
+    - name: Create moving_lock.
+      file:
+        path: "{{ atl_product_home_shared_moving_lock }}"
+        state: directory
+      when:
+        - atl_product_completed is succeeded
+      register: moving_lock_created
+
+    # Directory lock was created by this run?
+    # If so, then set a fact intending to move binary
+    - name: Move binary Scenario - lock created by this run
+      set_fact:
+        move_binary: true
+      when:
+        - moving_lock_created is succeeded
+        - moving_lock_created.changed
+        # Otherwise directory lock was either already created or
+        # could not be created. Fall back is to continue and install from temp
+
+  when: download_binary
+
+# If the intention is to move binary to home_shared
+- name: Move product installer to home_shared
+  block:
+
+    - name: Copy temp installer to home_shared
+      copy:
+        src: "{{ atl_product_temp_download }}"
+        dest: "{{ atl_product_home_shared_download }}"
+        remote_src: true
+      when:
+        - moving_lock_created is succeeded
+        - moving_lock_created.changed
+      register: copied
+
+    - name: Create completed_lock once product installer downloaded and copied
+      file:
+        path: "{{ atl_product_home_shared_completed_lock }}"
+        state: directory
+      when: copied is succeeded
+      register: completed_lock_created
+
+    - name: Remove moving_lock to show that binary is completed
+      file:
+        path: "{{ atl_product_home_shared_moving_lock }}"
+        state: absent
+      when:
+        - completed_lock_created is succeeded
+        - copied is succeeded
+      register: moving_lock_removed
+
+    - name: Delete old temp installer
+      file:
+        path: "{{ atl_product_temp_download }}"
+        state: absent
+      when: moving_lock_removed is succeeded
+      register: temp_deleted
+
+    - name: Set install to home_shared location
+      set_fact:
+        atl_product_download: "{{ atl_product_home_shared_download }}"
+      when: temp_deleted is succeeded
+
+  when: move_binary
+
+# At this point the binary is in {{ atl_product_download }}
+# (which is either on home_shared or temp)
 - name: Unpack the downloaded application depending on format
   include_tasks: "unpack_{{ atl_download_format }}.yml"
 

roles/product_install/tasks/unpack_installer.yml

@@ -10,6 +10,8 @@
 # actions. For example, if root and the 'jira' user exists then it
 # will create 'jira1'; this potentially creates idempotency/upgrade
 # issues down the line.
+# The variable {{ atl_product_download }} will be on temp for first nodes and shared_home for
+# subsequent nodes.
 - name: Run the installer
   command: /bin/sh "{{ atl_product_download }}" -q -varfile "{{ atl_product_varfile }}"
   args: