Merged in DCD-621-jira-dbconfig-with-aurora-db-type (pull request #18)

DCD-621: externalizes db type based in dbconfig.xml for Jira

Approved-by: Adam Brokes <abrokes@atlassian.com>
Approved-by: Ben Partridge <bpartridge@atlassian.com>
Approved-by: Steve Smith <ssmith@atlassian.com>

bin/run-tests-in-batches

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -e
+#set -x
+
+# Install dev packages from Pipfile.lock if necessary.
+pipenv sync --dev
+
+BATCH_SIZE=3
+BATCH_NUMBER=""
+batch_args="$1"
+
+case $batch_args in
+    -b|--batch)
+    BATCH_NUMBER="$2"
+    ;;
+    --default)
+    BATCH_NUMBER=""
+    ;;
+esac
+
+if [[ -z "$BATCH_NUMBER" ]]; then
+    echo "Batch number not passed in. Please pass a batch number to run tests."
+    exit -1
+fi
+
+case ${BATCH_NUMBER} in
+    ''|*[1-9]*) ;;
+    *) echo "Bad Input for Batch number. ${BATCH_NUMBER} is not a valid batch number (Should be a number >= 1)" && exit -1 ;;
+esac
+
+scenarios=( $(ls -d1 roles/**/molecule/* | sort) )
+
+offset=$(( ${BATCH_NUMBER} - 1))
+test_start_index=$(( ${offset} * $BATCH_SIZE ))
+scenario_batch="${scenarios[@]:$test_start_index:$BATCH_SIZE}"
+
+echo "Scenarios that will be executed as part of this batch: ${scenario_batch}"
+
+for scenario in ${scenario_batch}; do
+    pushd $(dirname $(dirname $scenario))
+    pipenv run \
+       molecule test --all
+    popd
+done;

bitbucket-pipelines.yml

@@ -1,18 +1,91 @@
 ---
 
 image: atlassian/default-image:2
+options:
+  size: 2x
 
 definitions:
   services:
     docker:
-      memory: 3072
+      memory: 4096
 
 pipelines:
   default:
     - step:
+        name: Pre Parallelization stage
+        script:
+          - echo "Running tests in 9 batches of 3 scenarios (max) each"
+    - parallel:
+        - step:
+            name: Molecule test batch 1
             services:
               - docker
             script:
               - apt-get update && apt-get install -y virtualenv python-dev
               - ./bin/install-ansible
-          - ./bin/run-all-tests
+              - ./bin/run-all-tests --batch 1
+        - step:
+            name: Molecule test batch 2
+            services:
+              - docker
+            script:
+              - apt-get update && apt-get install -y virtualenv python-dev
+              - ./bin/install-ansible
+              - ./bin/run-all-tests --batch 2
+        - step:
+            name: Molecule test batch 3
+            services:
+              - docker
+            script:
+              - apt-get update && apt-get install -y virtualenv python-dev
+              - ./bin/install-ansible
+              - ./bin/run-all-tests --batch 3
+        - step:
+            name: Molecule test batch 4
+            services:
+              - docker
+            script:
+              - apt-get update && apt-get install -y virtualenv python-dev
+              - ./bin/install-ansible
+              - ./bin/run-all-tests --batch 4
+        - step:
+            name: Molecule test batch 5
+            services:
+              - docker
+            script:
+              - apt-get update && apt-get install -y virtualenv python-dev
+              - ./bin/install-ansible
+              - ./bin/run-all-tests --batch 5
+        - step:
+            name: Molecule test batch 6
+            services:
+              - docker
+            script:
+              - apt-get update && apt-get install -y virtualenv python-dev
+              - ./bin/install-ansible
+              - ./bin/run-all-tests --batch 6
+        - step:
+            name: Molecule test batch 7
+            services:
+              - docker
+            script:
+              - apt-get update && apt-get install -y virtualenv python-dev
+              - ./bin/install-ansible
+              - ./bin/run-all-tests --batch 7
+        - step:
+            name: Molecule test batch 8
+            services:
+              - docker
+            script:
+              - apt-get update && apt-get install -y virtualenv python-dev
+              - ./bin/install-ansible
+              - ./bin/run-all-tests --batch 8
+        - step:
+            name: Molecule test batch 9
+            services:
+              - docker
+            script:
+              - apt-get update && apt-get install -y virtualenv python-dev
+              - ./bin/install-ansible
+              - ./bin/run-all-tests --batch 9
+

group_vars/aws_node_local.yml

@@ -65,6 +65,10 @@ atl_db_removeabandoned: "{{ lookup('env', 'ATL_DB_REMOVEABANDONED') or 'true' }}
 atl_db_removeabandonedtimeout: "{{ lookup('env', 'ATL_DB_REMOVEABANDONEDTIMEOUT') or '300' }}"
 atl_db_testwhileidle: "{{ lookup('env', 'ATL_DB_TESTWHILEIDLE') or 'true'}}"
 atl_db_testonborrow: "{{ lookup('env', 'ATL_DB_TESTONBORROW') or 'false' }}"
+atl_db_engine_to_db_type_map:
+  aurora_postgres: "postgresaurora96"
+  rds_postgres: "postgres72"
+atl_db_type: "{{ atl_db_engine_to_db_type_map[atl_db_engine] | default('postgres72') }}"
 
 atl_jdbc_db_name: "{{ lookup('env', 'ATL_JDBC_DB_NAME') }}"
 atl_jdbc_user: "{{ lookup('env', 'ATL_JDBC_USER') }}"

roles/jira_config/molecule/aurora/Dockerfile.j2

@@ -0,0 +1,14 @@
+# Molecule managed
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi

roles/jira_config/molecule/aurora/molecule.yml

@@ -0,0 +1,36 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+lint:
+  name: yamllint
+platforms:
+  - name: amazon_linux2
+    image: amazonlinux:2
+    groups:
+      - aws_node_local
+    ulimits:
+      - nofile:262144:262144
+  - name: ubuntu_lts
+    image: ubuntu:bionic
+    groups:
+      - aws_node_local
+    ulimits:
+      - nofile:262144:262144
+provisioner:
+  name: ansible
+  options:
+    skip-tags: runtime_pkg
+  lint:
+    name: ansible-lint
+    options:
+      x: ["701"]
+  inventory:
+    links:
+      group_vars: ../../../../group_vars/
+verifier:
+  name: testinfra
+  lint:
+    name: flake8
+    enabled: false

roles/jira_config/molecule/aurora/playbook.yml

@@ -0,0 +1,32 @@
+---
+- name: Converge
+  hosts: all
+  vars:
+    atl_product_family: "jira"
+    atl_product_edition: "jira-software"
+    atl_product_user: "jira"
+    atl_product_version: "7.13.2"
+
+    # dbconfig.xml variables
+    atl_jdbc_user: 'atljira'
+    atl_jdbc_password: 'molecule_password'
+    atl_jvm_heap: 'PLACEHOLDER'
+    atl_cluster_node_id: 'FAKEID'
+    atl_db_poolminsize: 1111
+    atl_db_poolmaxsize: 1111
+    atl_db_minidle: 1111
+    atl_db_maxidle: 1111
+    atl_db_maxwaitmillis: 1111
+    atl_db_minevictableidletimemillis: 1111
+    atl_db_timebetweenevictionrunsmillis: 1111
+    atl_db_removeabandoned: 'false'
+    atl_db_removeabandonedtimeout: 1111
+    atl_db_testwhileidle: 'false'
+    atl_db_testonborrow: 'true'
+    atl_db_engine: 'aurora_postgres'
+
+  roles:
+    - role: linux_common
+    - role: product_common
+    - role: product_install
+    - role: jira_config

roles/jira_config/molecule/aurora/tests/test_default.py

@@ -0,0 +1,75 @@
+import os
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_dbconfig_file(host):
+    f = host.file('/var/atlassian/application-data/jira/dbconfig.xml')
+    assert f.exists
+    assert f.user == 'jira'
+    assert f.contains("<database-type>postgresaurora96</database-type>")
+    assert f.contains("<driver-class>org.postgresql.Driver</driver-class>")
+    assert f.contains("<username>atljira</username>")
+    assert f.contains("<password>molecule_password</password>")
+
+    assert f.contains("<pool-min-size>1111</pool-min-size>")
+    assert f.contains("<pool-max-size>1111</pool-max-size>")
+    assert f.contains("<pool-min-idle>1111</pool-min-idle>")
+    assert f.contains("<pool-max-idle>1111</pool-max-idle>")
+
+    assert f.contains("<pool-max-wait>1111</pool-max-wait>")
+    assert f.contains("<min-evictable-idle-time-millis>1111</min-evictable-idle-time-millis>")
+    assert f.contains("<pool-remove-abandoned>false</pool-remove-abandoned>")
+    assert f.contains("<time-between-eviction-runs-millis>1111</time-between-eviction-runs-millis>")
+    assert f.contains("<min-evictable-idle-time-millis>1111</min-evictable-idle-time-millis>")
+    assert f.contains("<pool-remove-abandoned>false</pool-remove-abandoned>")
+    assert f.contains("<pool-remove-abandoned-timeout>1111</pool-remove-abandoned-timeout>")
+    assert f.contains("<pool-test-while-idle>false</pool-test-while-idle>")
+    assert f.contains("<pool-test-on-borrow>true</pool-test-on-borrow>")
+
+
+def test_setenv_file(host):
+    f = host.file('/opt/atlassian/jira-software/current/bin/setenv.sh')
+    assert f.exists
+    assert f.contains('^JVM_MINIMUM_MEMORY="PLACEHOLDER"')
+    assert f.contains('^JVM_MAXIMUM_MEMORY="PLACEHOLDER"')
+    assert f.contains('^JIRA_HOME="/var/atlassian/application-data/jira"')
+    assert f.contains('^export CATALINA_OPTS="')
+
+
+def test_cluster_file(host):
+    f = host.file('/var/atlassian/application-data/jira/cluster.properties')
+    assert f.exists
+    assert f.contains('jira.node.id = FAKEID')
+    assert f.contains('jira.shared.home = /media/atl/jira/shared')
+
+
+def test_server_file(host):
+    f = host.file('/opt/atlassian/jira-software/current/conf/server.xml')
+    assert f.exists
+    assert f.contains('Connector port="8080"')
+    assert f.contains('Server port="8005"')
+    assert f.contains('<Context path=""')
+    assert f.contains('maxThreads="200"')
+    assert f.contains('minSpareThreads="10"')
+    assert f.contains('connectionTimeout="20000"')
+    assert f.contains('enableLookups="false"')
+    assert f.contains('protocol="HTTP/1.1"')
+    assert f.contains('redirectPort=""')
+    assert f.contains('acceptCount="10"')
+    assert f.contains('secure="false"')
+    assert f.contains('scheme="http"')
+    assert not f.contains('proxyName=')
+    assert not f.contains('proxyPort=')
+
+
+def test_install_permissions(host):
+    assert host.file('/opt/atlassian/jira-software/current/conf/server.xml').user == 'root'
+    assert host.file('/opt/atlassian/jira-software/current/atlassian-jira/WEB-INF/web.xml').user == 'root'
+
+    assert host.file('/opt/atlassian/jira-software/current/logs/').user == 'jira'
+    assert host.file('/opt/atlassian/jira-software/current/work/').user == 'jira'
+    assert host.file('/opt/atlassian/jira-software/current/temp/').user == 'jira'

roles/jira_config/molecule/default/playbook.yml

@@ -23,6 +23,7 @@
     atl_db_removeabandonedtimeout: 1111
     atl_db_testwhileidle: 'false'
     atl_db_testonborrow: 'true'
+    atl_db_engine: 'rds_postgres'
 
   roles:
     - role: linux_common

roles/jira_config/molecule/default/tests/test_default.py

@@ -10,6 +10,7 @@ def test_dbconfig_file(host):
     f = host.file('/var/atlassian/application-data/jira/dbconfig.xml')
     assert f.exists
     assert f.user == 'jira'
+    assert f.contains("<database-type>postgres72</database-type>")
     assert f.contains("<driver-class>org.postgresql.Driver</driver-class>")
     assert f.contains("<username>atljira</username>")
     assert f.contains("<password>molecule_password</password>")

roles/jira_config/templates/dbconfig.xml.j2

@@ -3,7 +3,7 @@
 <jira-database-config>
   <name>defaultDS</name>
   <delegator-name>default</delegator-name>
-  <database-type>postgres72</database-type>
+  <database-type>{{ atl_db_type }}</database-type>
   <schema-name>public</schema-name>
   <jdbc-datasource>
     <url>{{ atl_jdbc_url }}</url>

roles/product_install/molecule/bitbucket_latest/playbook.yml

@@ -7,7 +7,6 @@
     atl_product_user: "bitbucket"
 
     atl_product_home: "{{ atl_shared_mountpoint }}/{{ atl_product_edition }}"
-
   roles:
     - role: linux_common
     - role: product_common

roles/product_install/molecule/bitbucket_latest/tests/test_default.py

@@ -24,17 +24,19 @@ def test_version_file_is_latest(host):
     verfile = host.file('/media/atl/bitbucket/shared/bitbucket.version')
     assert verfile.exists
 
-    upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/bitbucket/versions/latest")
+    upstream_fd = urllib.request.urlopen(
+        "https://marketplace.atlassian.com/rest/2/applications/bitbucket/versions/latest")
     upstream_json = json.load(upstream_fd)
     upstream = upstream_json['version']
 
     assert verfile.content.decode("UTF-8").strip() == upstream.strip()
 
 def test_latest_is_downloaded(host):
-    upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/bitbucket/versions/latest")
+    upstream_fd = urllib.request.urlopen(
+        "https://marketplace.atlassian.com/rest/2/applications/bitbucket/versions/latest")
     upstream_json = json.load(upstream_fd)
     upstream = upstream_json['version']
 
-    installer = host.file('/opt/atlassian/tmp/bitbucket.'+upstream+'.bin')
+    installer = host.file('/opt/atlassian/tmp/bitbucket.' + upstream + '-x64.bin')
     assert installer.exists
     assert installer.user == 'root'

roles/product_install/molecule/confluence_latest/tests/test_default.py

@@ -35,6 +35,6 @@ def test_latest_is_downloaded(host):
     upstream_json = json.load(upstream_fd)
     upstream = upstream_json['version']
 
-    installer = host.file('/opt/atlassian/tmp/confluence.'+upstream+'.bin')
+    installer = host.file('/opt/atlassian/tmp/confluence.'+upstream+'-x64.bin')
     assert installer.exists
     assert installer.user == 'root'

roles/product_install/molecule/default/tests/test_default.py

@@ -23,6 +23,6 @@ def test_latest_is_downloaded(host):
     upstream_json = json.load(upstream_fd)
     upstream = upstream_json['version']
 
-    installer = host.file('/opt/atlassian/tmp/jira-core.'+upstream+'.bin')
+    installer = host.file('/opt/atlassian/tmp/jira-core.'+upstream+'-x64.bin')
     assert installer.exists
     assert installer.user == 'root'

roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py

@@ -14,7 +14,7 @@ def test_version_is_correct(host):
     assert verfile.content.decode("UTF-8").strip() == "7.10.2"
 
 def test_is_downloaded(host):
-    installer = host.file('/opt/atlassian/tmp/jira-core.7.10.2.bin')
+    installer = host.file('/opt/atlassian/tmp/jira-core.7.10.2-x64.bin')
     assert installer.exists
     assert installer.user == 'root'
 

roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py

@@ -14,7 +14,7 @@ def test_version_is_correct(host):
     assert verfile.content.decode("UTF-8").strip() == "7.10.1"
 
 def test_is_downloaded(host):
-    installer = host.file('/opt/atlassian/tmp/jira-core.7.10.1.bin')
+    installer = host.file('/opt/atlassian/tmp/jira-core.7.10.1-x64.bin')
     assert installer.exists
     assert installer.user == 'root'
 

roles/product_install/molecule/jira_software_latest/tests/test_default.py

@@ -35,6 +35,6 @@ def test_latest_is_downloaded(host):
     upstream_json = json.load(upstream_fd)
     upstream = upstream_json['version']
 
-    installer = host.file('/opt/atlassian/tmp/jira-software.'+upstream+'.bin')
+    installer = host.file('/opt/atlassian/tmp/jira-software.'+upstream+'-x64.bin')
     assert installer.exists
     assert installer.user == 'root'

roles/product_install/molecule/jira_version_from_file/tests/test_default.py

@@ -14,7 +14,7 @@ def test_version_is_correct(host):
     assert verfile.content.decode("UTF-8").strip() == "7.9.0"
 
 def test_is_downloaded(host):
-    installer = host.file('/opt/atlassian/tmp/jira-core.7.9.0.bin')
+    installer = host.file('/opt/atlassian/tmp/jira-core.7.9.0-x64.bin')
     assert installer.exists
     assert installer.user == 'root'
 

roles/product_install/molecule/jira_version_latest/tests/test_default.py

@@ -23,6 +23,6 @@ def test_latest_is_downloaded(host):
     upstream_json = json.load(upstream_fd)
     upstream = upstream_json['version']
 
-    installer = host.file('/opt/atlassian/tmp/jira-core.'+upstream+'.bin')
+    installer = host.file('/opt/atlassian/tmp/jira-core.'+upstream+'-x64.bin')
     assert installer.exists
     assert installer.user == 'root'

roles/product_install/molecule/jira_version_override/tests/test_default.py

@@ -14,7 +14,7 @@ def test_version_is_correct(host):
     assert verfile.content.decode("UTF-8").strip() == "7.13.2"
 
 def test_is_downloaded(host):
-    installer = host.file('/opt/atlassian/tmp/jira-core.7.13.2.bin')
+    installer = host.file('/opt/atlassian/tmp/jira-core.7.13.2-x64.bin')
     assert installer.exists
     assert installer.user == 'root'
 

roles/product_install/molecule/servicedesk3/tests/test_default.py

@@ -14,7 +14,7 @@ def test_version_is_correct(host):
     assert verfile.content.decode("UTF-8").strip() == "3.9.0"
 
 def test_is_downloaded(host):
-    installer = host.file('/opt/atlassian/tmp/servicedesk.3.9.0.bin')
+    installer = host.file('/opt/atlassian/tmp/servicedesk.3.9.0-x64.bin')
     assert installer.exists
     assert installer.user == 'root'
 

roles/product_install/molecule/servicedesk4/tests/test_default.py

@@ -14,7 +14,7 @@ def test_version_is_correct(host):
     assert verfile.content.decode("UTF-8").strip() == "4.1.0"
 
 def test_is_downloaded(host):
-    installer = host.file('/opt/atlassian/tmp/servicedesk.4.1.0.bin')
+    installer = host.file('/opt/atlassian/tmp/servicedesk.4.1.0-x64.bin')
     assert installer.exists
     assert installer.user == 'root'
 

roles/product_install/molecule/servicedesk_latest/tests/test_default.py

@@ -23,7 +23,7 @@ def test_version_is_correct(host):
     assert verfile.content.decode("UTF-8").strip() == sd
 
 def test_is_downloaded(host):
-    installer = host.file('/opt/atlassian/tmp/servicedesk.'+sd+'.bin')
+    installer = host.file('/opt/atlassian/tmp/servicedesk.'+sd+'-x64.bin')
     assert installer.exists
     assert installer.user == 'root'