Merge remote-tracking branch 'origin/master' into ITOPS-1677-add-crowd-support-in-dc-deployment

aws_bitbucket_dc_node.yml

@@ -17,6 +17,9 @@
       - "UMask=0027"
       - "LimitNOFILE=4096"
       - "Environment=BITBUCKET_HOME={{ atl_product_home }}"
+      - "Environment=JVM_MAXIMUM_MEMORY={{ atl_jvm_heap }}"
+      - "Environment=JVM_MINIMUM_MEMORY={{ atl_jvm_heap }}"
+      - "Environment=JVM_SUPPORT_RECOMMENDED_ARGS={{ atl_jvm_opts }}"
     atl_startup_exec_options:
       - "-fg"
       - "--no-search"

group_vars/aws_node_local.yml

@@ -78,6 +78,7 @@ atl_jdbc_query_params_for_engine:
 atl_jdbc_url: "jdbc:postgresql://{{ atl_db_host }}:{{ atl_db_port }}/{{ atl_jdbc_db_name }}{{ atl_jdbc_query_params_for_engine[atl_db_engine]| default('') }}"
 
 atl_jvm_heap: "{{ lookup('env', 'ATL_JVM_HEAP') or '2048m' }}"
+atl_jvm_opts: "{{ lookup('env', 'ATL_JVM_OPTS') or '' }}"
 atl_catalina_opts: "{{ lookup('env', 'ATL_CATALINA_OPTS') or '' }}"
 atl_proxy_name:  "{{ lookup('env', 'ATL_PROXY_NAME') | lower }}"
 atl_proxy_port:  "{{ lookup('env', 'ATL_TOMCAT_PROXYPORT') }}"

roles/bitbucket_config/molecule/default/Dockerfile.j2

@@ -0,0 +1,14 @@
+# Molecule managed
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi

roles/bitbucket_config/molecule/default/molecule.yml

@@ -0,0 +1,36 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+lint:
+  name: yamllint
+platforms:
+  - name: amazon_linux2
+    image: amazonlinux:2
+    groups:
+      - aws_node_local
+    ulimits:
+      - nofile:262144:262144
+  - name: ubuntu_lts
+    image: ubuntu:bionic
+    groups:
+      - aws_node_local
+    ulimits:
+      - nofile:262144:262144
+provisioner:
+  name: ansible
+  options:
+    skip-tags: runtime_pkg
+  lint:
+    name: ansible-lint
+    options:
+      x: ["701"]
+  inventory:
+    links:
+      group_vars: ../../../../group_vars/
+verifier:
+  name: testinfra
+  lint:
+    name: flake8
+    enabled: false

roles/bitbucket_config/molecule/default/playbook.yml

@@ -0,0 +1,19 @@
+---
+- name: Converge
+  hosts: all
+  vars:
+    atl_product_family: "stash"
+    atl_product_edition: "bitbucket"
+    atl_product_user: "bitbucket"
+    atl_product_version: "6.3.1"
+
+    atl_product_home: "{{ atl_shared_mountpoint }}/{{ atl_product_edition }}"
+
+    atl_jdbc_user: 'bb_db_user'
+    atl_jdbc_password: 'molecule_password'
+
+  roles:
+    - role: linux_common
+    - role: product_common
+    - role: product_install
+    - role: bitbucket_config

roles/bitbucket_config/molecule/default/tests/test_default.py

@@ -0,0 +1,16 @@
+import os
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_config_file(host):
+    f = host.file('/media/atl/bitbucket/shared/bitbucket.properties')
+    assert f.exists
+    assert f.user == 'bitbucket'
+
+    assert f.contains("jdbc.driver=org.postgresql.Driver")
+    assert f.contains("jdbc.user=bb_db_user")
+    assert f.contains("jdbc.password=molecule_password")

roles/bitbucket_config/tasks/main.yml

@@ -22,3 +22,4 @@
     group: "root"
     mode: "u=rwX,g=rX,o=rX"
     recurse: true
+  changed_when: false  # For Molecule idempotence check

roles/product_common/molecule/default/tests/test_default.py

@@ -13,8 +13,7 @@ def test_user_created(host):
 
 
 @pytest.mark.parametrize('exe', [
-    '/usr/bin/git',
+    '/usr/bin/git'
-    '/usr/bin/psql'
 ])
 def test_package_exes(host, exe):
     assert host.file(exe).exists

roles/product_install/molecule/default/tests/test_default.py

@@ -2,6 +2,7 @@ import os
 from six.moves import urllib
 
 import testinfra.utils.ansible_runner
+import json
 
 testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
     os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
@@ -11,14 +12,16 @@ def test_version_file_is_latest(host):
     verfile = host.file('/media/atl/jira/shared/jira-core.version')
     assert verfile.exists
 
-    upstream_fd = urllib.request.urlopen("https://s3.amazonaws.com/atlassian-software/releases/jira-core/latest")
+    upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/jira/versions/latest")
-    upstream = upstream_fd.read()
+    upstream_json = json.load(upstream_fd)
+    upstream = upstream_json['version']
 
-    assert verfile.content.decode("UTF-8").strip() == upstream.decode("UTF-8").strip()
+    assert verfile.content.decode("UTF-8").strip() == upstream.strip()
 
 def test_latest_is_downloaded(host):
-    upstream_fd = urllib.request.urlopen("https://s3.amazonaws.com/atlassian-software/releases/jira-core/latest")
+    upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/jira/versions/latest")
-    upstream = upstream_fd.read().decode("UTF-8").strip()
+    upstream_json = json.load(upstream_fd)
+    upstream = upstream_json['version']
 
     installer = host.file('/opt/atlassian/tmp/jira-core.'+upstream+'.bin')
     assert installer.exists

roles/product_install/molecule/jira_software_latest/tests/test_default.py

@@ -2,6 +2,7 @@ import os
 from six.moves import urllib
 
 import testinfra.utils.ansible_runner
+import json
 
 testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
     os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
@@ -23,14 +24,16 @@ def test_version_file_is_latest(host):
     verfile = host.file('/media/atl/jira/shared/jira-software.version')
     assert verfile.exists
 
-    upstream_fd = urllib.request.urlopen("https://s3.amazonaws.com/atlassian-software/releases/jira-software/latest")
+    upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/jira/versions/latest")
-    upstream = upstream_fd.read()
+    upstream_json = json.load(upstream_fd)
+    upstream = upstream_json['version']
 
-    assert verfile.content.decode("UTF-8").strip() == upstream.decode("UTF-8").strip()
+    assert verfile.content.decode("UTF-8").strip() == upstream.strip()
 
 def test_latest_is_downloaded(host):
-    upstream_fd = urllib.request.urlopen("https://s3.amazonaws.com/atlassian-software/releases/jira-software/latest")
+    upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/jira/versions/latest")
-    upstream = upstream_fd.read().decode("UTF-8").strip()
+    upstream_json = json.load(upstream_fd)
+    upstream = upstream_json['version']
 
     installer = host.file('/opt/atlassian/tmp/jira-software.'+upstream+'.bin')
     assert installer.exists

roles/product_install/molecule/jira_version_latest/tests/test_default.py

@@ -2,6 +2,7 @@ import os
 from six.moves import urllib
 
 import testinfra.utils.ansible_runner
+import json
 
 testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
     os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
@@ -11,14 +12,16 @@ def test_version_file_is_latest(host):
     verfile = host.file('/media/atl/jira/shared/jira-core.version')
     assert verfile.exists
 
-    upstream_fd = urllib.request.urlopen("https://s3.amazonaws.com/atlassian-software/releases/jira-core/latest")
+    upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/jira/versions/latest")
-    upstream = upstream_fd.read()
+    upstream_json = json.load(upstream_fd)
+    upstream = upstream_json['version']
 
-    assert verfile.content.decode("UTF-8").strip() == upstream.decode("UTF-8").strip()
+    assert verfile.content.decode("UTF-8").strip() == upstream.strip()
 
 def test_latest_is_downloaded(host):
-    upstream_fd = urllib.request.urlopen("https://s3.amazonaws.com/atlassian-software/releases/jira-core/latest")
+    upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/jira/versions/latest")
-    upstream = upstream_fd.read().decode("UTF-8").strip()
+    upstream_json = json.load(upstream_fd)
+    upstream = upstream_json['version']
 
     installer = host.file('/opt/atlassian/tmp/jira-core.'+upstream+'.bin')
     assert installer.exists

roles/product_startup/molecule/bitbucket/playbook.yml

@@ -8,11 +8,16 @@
 
     atl_product_home: "{{ atl_shared_mountpoint }}/{{ atl_product_edition }}"
     atl_product_home_shared: "{{ atl_product_home }}/shared"
+    atl_jvm_heap: "dummy_heap"
+    atl_jvm_opts: "dummy_opts"
 
     atl_startup_systemd_params:
       - "UMask=0027"
       - "LimitNOFILE=4096"
       - "Environment=BITBUCKET_HOME={{ atl_product_home }}"
+      - "Environment=JVM_MAXIMUM_MEMORY={{ atl_jvm_heap }}"
+      - "Environment=JVM_MINIMUM_MEMORY={{ atl_jvm_heap }}"
+      - "Environment=JVM_SUPPORT_RECOMMENDED_ARGS={{ atl_jvm_opts }}"
 
     atl_startup_exec_options:
       - "-fg"

roles/product_startup/molecule/bitbucket/tests/test_default.py

@@ -12,3 +12,6 @@ def test_service_file(host):
     assert f.contains("^UMask=0027$")
     assert f.contains("^LimitNOFILE=4096$")
     assert f.contains("^Environment=BITBUCKET_HOME=/media/atl/bitbucket$")
+    assert f.contains("^Environment=JVM_MINIMUM_MEMORY=dummy_heap$")
+    assert f.contains("^Environment=JVM_MAXIMUM_MEMORY=dummy_heap$")
+    assert f.contains("^Environment=JVM_SUPPORT_RECOMMENDED_ARGS=dummy_opts$")

roles/product_startup/molecule/default/tests/test_default.py

@@ -9,3 +9,6 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
 def test_service_file(host):
     f = host.file('/etc/systemd/system/jira-software.service')
     assert f.contains("^ExecStart=/opt/atlassian/jira-software/current/bin/start-jira.sh -fg$")
+    assert f.user == 'root'
+    assert f.group == 'root'
+    assert f.mode == 0o0640

roles/product_startup/tasks/main.yml

@@ -4,6 +4,7 @@
   template:
     src: "product.service.j2"
     dest: "/etc/systemd/system/{{ atl_systemd_service_name }}"
+<<<<<<< HEAD
   when: atl_product_edition != "crowd"
   notify:
     - Enable Product
@@ -14,6 +15,11 @@
     src: "crowd.service.j2"
     dest: "/etc/systemd/system/{{ atl_systemd_service_name }}"
   when: atl_product_edition == "crowd"
+=======
+    owner: root
+    group: root
+    mode: 0640
+>>>>>>> origin/master
   notify:
     - Enable Product
     - Restart Product

roles/synchrony_config/files/start-synchrony

@@ -55,9 +55,11 @@ function waitForConfluenceConfigInSharedHome() {
 
 waitForConfluenceConfigInSharedHome
 
+# Additional settings are in /etc/atl.synchrony
+export SYNCHRONY_JWT_PRIVATE_KEY
+export SYNCHRONY_JWT_PUBLIC_KEY
+
 exec ${_RUNJAVA} \
      -classpath ${SYNCHRONY_CLASSPATH} \
      ${ATL_SYNCHRONY_JVM_PROPERTIES} \
-     -Djwt.private.key=${SYNCHRONY_JWT_PRIVATE_KEY} \
-     -Djwt.public.key=${SYNCHRONY_JWT_PUBLIC_KEY} \
      synchrony.core sql

roles/synchrony_config/tasks/main.yml

@@ -12,4 +12,4 @@
     src: "atl.synchrony.j2"
     dest: "/etc/atl.synchrony"
     group: "{{ atl_product_user }}"
-    mode: "0640"    
+    mode: "0640"

roles/synchrony_config/templates/atl.synchrony.j2

@@ -9,11 +9,12 @@ ATL_SYNCHRONY_JAR_PATH="{{ atl_product_installation_current }}/confluence/WEB-IN
 AWS_EC2_PRIVATE_IP="{{ atl_local_ipv4 }}"
 _RUNJAVA="{{ atl_product_installation_current }}/jre/bin/java"
 
+SYNCHRONY_DATABASE_USERNAME="{{ atl_jdbc_user }}"
+SYNCHRONY_DATABASE_PASSWORD="{{ atl_jdbc_password }}"
+
 ATL_SYNCHRONY_JVM_PROPERTIES="{{ atl_synchrony_stack_space }} {{ atl_synchrony_memory }} \
  -Dsynchrony.cluster.impl=hazelcast-btf \
  -Dsynchrony.database.url={{ atl_jdbc_url }} \
- -Dsynchrony.database.username={{ atl_jdbc_user }} \
- -Dsynchrony.database.password={{ atl_jdbc_password }} \
  -Dsynchrony.bind={{ atl_local_ipv4 }} \
  -Dsynchrony.cluster.bind={{ atl_local_ipv4 }} \
  -Dcluster.interfaces={{ atl_local_ipv4 }} \