From a912ac504481fd3af5c90cbd4d2287631f8c8897 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Fri, 21 Aug 2020 11:05:42 +1000 Subject: [PATCH 1/7] DCD-1020: Move JRE home & binary location to group-vars. --- group_vars/aws_node_local.yml | 3 +++ roles/synchrony_config/defaults/main.yml | 1 - roles/synchrony_config/templates/atl.synchrony.j2 | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/group_vars/aws_node_local.yml b/group_vars/aws_node_local.yml index 56fb19d..692268d 100644 --- a/group_vars/aws_node_local.yml +++ b/group_vars/aws_node_local.yml @@ -34,6 +34,9 @@ atl_product_installation_versioned: "{{ atl_product_installation_base }}/{{ atl_ atl_product_installation_current: "{{ atl_product_installation_base }}/current" atl_installer_temp: "{{ atl_installation_base }}/tmp" +atl_jre_home: "{{ 'java' if atl_use_system_jdk else (atl_product_installation_current + '/jre') }}" +atl_java_binary: "{{ atl_jre_home }}/bin/java" + atl_product_logs_default: &logs_default - path: "{{ atl_product_installation_current }}/logs/*" type: product diff --git a/roles/synchrony_config/defaults/main.yml b/roles/synchrony_config/defaults/main.yml index 383d67c..e6607e3 100644 --- a/roles/synchrony_config/defaults/main.yml +++ b/roles/synchrony_config/defaults/main.yml @@ -3,6 +3,5 @@ # FIXME: Add others as map? atl_synchrony_cluster_type: "aws" -atl_synchrony_java_binary: "{{ 'java' if atl_use_system_jdk else (atl_product_installation_current + '/jre/bin/java') }}" atl_synchrony_memory: "{{ lookup('env', 'ATL_SYNCHRONY_MEMORY') or '-Xmx2g' }}" atl_synchrony_stack_space: "{{ lookup('env', 'ATL_SYNCHRONY_STACK_SPACE') or '-Xss2048k' }}" diff --git a/roles/synchrony_config/templates/atl.synchrony.j2 b/roles/synchrony_config/templates/atl.synchrony.j2 index bf36ba1..809bc91 100644 --- a/roles/synchrony_config/templates/atl.synchrony.j2 +++ b/roles/synchrony_config/templates/atl.synchrony.j2 @@ -7,7 +7,7 @@ ATL_CONFLUENCE_SHARED_CONFIG_FILE="{{ atl_product_home_shared }}/confluence.cfg. ATL_SYNCHRONY_JAR_PATH="{{ atl_product_installation_current }}/confluence/WEB-INF/packages/synchrony-standalone.jar" AWS_EC2_PRIVATE_IP="{{ atl_local_ipv4 }}" -_RUNJAVA="{{ atl_synchrony_java_binary }}" +_RUNJAVA="{{ atl_java_binary }}" SYNCHRONY_DATABASE_USERNAME="{{ atl_jdbc_user }}" SYNCHRONY_DATABASE_PASSWORD="{{ atl_jdbc_password }}" From b66d08b4433603c05eed01d325d09dd266c24856 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Fri, 21 Aug 2020 13:02:21 +1000 Subject: [PATCH 2/7] DCD-1020: Use the major java version and let per-OS roles apply the approriate naming. --- group_vars/aws_node_local.yml | 2 +- roles/product_common/defaults/main.yml | 1 - .../product_common/molecule/system_jdk/tests/test_default.py | 4 +++- roles/product_common/tasks/amazon.yml | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/group_vars/aws_node_local.yml b/group_vars/aws_node_local.yml index 692268d..9ce6b3f 100644 --- a/group_vars/aws_node_local.yml +++ b/group_vars/aws_node_local.yml @@ -9,7 +9,7 @@ # # https://docs.ansible.com/ansible/latest/user_guide/playbooks_variables.html -java_version: "1.8.0" +java_major_version: "8" postgres_version: "9.6" git_version: "2.14.4" atl_shared_mountpoint: "/media/atl" diff --git a/roles/product_common/defaults/main.yml b/roles/product_common/defaults/main.yml index ce62ba4..4e9f728 100644 --- a/roles/product_common/defaults/main.yml +++ b/roles/product_common/defaults/main.yml @@ -1,5 +1,4 @@ --- -java_version: "1.8.0" java_major_version: "8" postgres_version: "9.6" diff --git a/roles/product_common/molecule/system_jdk/tests/test_default.py b/roles/product_common/molecule/system_jdk/tests/test_default.py index 77d3c52..e615c9f 100644 --- a/roles/product_common/molecule/system_jdk/tests/test_default.py +++ b/roles/product_common/molecule/system_jdk/tests/test_default.py @@ -14,7 +14,9 @@ def test_user_created(host): @pytest.mark.parametrize('exe', [ '/usr/bin/git', - '/usr/bin/javac' + '/usr/bin/javac', + '/usr/lib/jvm/java/bin/java', + '/usr/lib/jvm/java/jre/bin/java' ]) def test_package_exes(host, exe): assert host.file(exe).exists diff --git a/roles/product_common/tasks/amazon.yml b/roles/product_common/tasks/amazon.yml index c02f864..4d33f79 100644 --- a/roles/product_common/tasks/amazon.yml +++ b/roles/product_common/tasks/amazon.yml @@ -3,7 +3,7 @@ - name: Install JDK if necessary on Amazon Linux yum: name: - - java-{{ java_version }}-openjdk-devel + - java-1.{{ java_major_version }}.0-openjdk-devel when: atl_use_system_jdk - name: Install other base packages on Amazon Linux From 43996fffcdd83115e70cbd09d9bf9360dd47a2b4 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Fri, 21 Aug 2020 13:03:35 +1000 Subject: [PATCH 3/7] DCD-1020: Make/enforce assumption that JVM packages setup a common symlink. --- group_vars/aws_node_local.yml | 6 ++++-- roles/crowd_config/tasks/main.yml | 4 ++-- roles/product_common/tasks/ubuntu.yml | 17 ++++++++++++++--- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/group_vars/aws_node_local.yml b/group_vars/aws_node_local.yml index 9ce6b3f..f951617 100644 --- a/group_vars/aws_node_local.yml +++ b/group_vars/aws_node_local.yml @@ -34,8 +34,10 @@ atl_product_installation_versioned: "{{ atl_product_installation_base }}/{{ atl_ atl_product_installation_current: "{{ atl_product_installation_base }}/current" atl_installer_temp: "{{ atl_installation_base }}/tmp" -atl_jre_home: "{{ 'java' if atl_use_system_jdk else (atl_product_installation_current + '/jre') }}" -atl_java_binary: "{{ atl_jre_home }}/bin/java" +# NOTE: If using the non-installer java we expect to have the full JDK +# installed and linked to `/usr/lib/jvm/java`, usually via `alternatives`. +atl_java_home: "{{ '/usr/lib/jvm/java' if atl_use_system_jdk else (atl_product_installation_current + '/jre') }}" +atl_java_binary: "{{ atl_java_home }}/bin/java" atl_product_logs_default: &logs_default - path: "{{ atl_product_installation_current }}/logs/*" diff --git a/roles/crowd_config/tasks/main.yml b/roles/crowd_config/tasks/main.yml index 726211a..e3b7c72 100644 --- a/roles/crowd_config/tasks/main.yml +++ b/roles/crowd_config/tasks/main.yml @@ -51,7 +51,7 @@ lineinfile: path: "{{ atl_product_installation_versioned }}/apache-tomcat/bin/setenv.sh" insertafter: "EOF" - line: 'export JAVA_HOME=/usr/lib/jvm/jre-{{ java_version }}-openjdk' + line: "export JAVA_HOME={{ atl_java_home }}" - name: Create application directories file: @@ -152,4 +152,4 @@ - name: Remove crowd.xml to prevent duplicates from appearing in cluster reporting file: path: "{{ atl_product_installation_versioned }}/apache-tomcat/conf/Catalina/localhost/crowd.xml" - state: absent \ No newline at end of file + state: absent diff --git a/roles/product_common/tasks/ubuntu.yml b/roles/product_common/tasks/ubuntu.yml index 55c6311..2b0eef8 100644 --- a/roles/product_common/tasks/ubuntu.yml +++ b/roles/product_common/tasks/ubuntu.yml @@ -1,9 +1,20 @@ --- - name: Install JDK if necessary on Ubuntu - package: - name: - - openjdk-{{ java_major_version }}-jdk-headless + block: + + - name: Install JDK package + package: + name: + - openjdk-{{ java_major_version }}-jdk-headless + + - name: Ensure common symlink exists + alternatives: + link: "/usr/lib/jvm/java" + name: "java_sdk" + path: "/usr/lib/jvm/java-8-openjdk-amd64" + priority: 99 + when: atl_use_system_jdk tags: - runtime_pkg From d22c0222edfc4a7bf27f5f7bf04e15b0e5a7a561 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Fri, 21 Aug 2020 13:56:13 +1000 Subject: [PATCH 4/7] DCD-1020: We need a JRE symlink too. --- group_vars/aws_node_local.yml | 7 +++++-- roles/product_common/tasks/ubuntu.yml | 9 ++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/group_vars/aws_node_local.yml b/group_vars/aws_node_local.yml index f951617..4f8cb77 100644 --- a/group_vars/aws_node_local.yml +++ b/group_vars/aws_node_local.yml @@ -35,9 +35,12 @@ atl_product_installation_current: "{{ atl_product_installation_base }}/current" atl_installer_temp: "{{ atl_installation_base }}/tmp" # NOTE: If using the non-installer java we expect to have the full JDK -# installed and linked to `/usr/lib/jvm/java`, usually via `alternatives`. +# installed and linked to `/usr/lib/jvm/java` and # `/usr/lib/jvm/jre`. +# See product_common/task/ubuntu.yml for an example of apropriate +# linking using `alternatives`. atl_java_home: "{{ '/usr/lib/jvm/java' if atl_use_system_jdk else (atl_product_installation_current + '/jre') }}" -atl_java_binary: "{{ atl_java_home }}/bin/java" +atl_jre_home: "{{ '/usr/lib/jvm/jre' if atl_use_system_jdk else atl_java_home }}" +atl_java_binary: "{{ atl_jre_home }}/bin/java" atl_product_logs_default: &logs_default - path: "{{ atl_product_installation_current }}/logs/*" diff --git a/roles/product_common/tasks/ubuntu.yml b/roles/product_common/tasks/ubuntu.yml index 2b0eef8..0b033c5 100644 --- a/roles/product_common/tasks/ubuntu.yml +++ b/roles/product_common/tasks/ubuntu.yml @@ -8,13 +8,20 @@ name: - openjdk-{{ java_major_version }}-jdk-headless - - name: Ensure common symlink exists + - name: Ensure common JDK symlink exists alternatives: link: "/usr/lib/jvm/java" name: "java_sdk" path: "/usr/lib/jvm/java-8-openjdk-amd64" priority: 99 + - name: Ensure common JRE symlink exists + alternatives: + link: "/usr/lib/jvm/jre" + name: "jre" + path: "/usr/lib/jvm/java-8-openjdk-amd64/jre" + priority: 99 + when: atl_use_system_jdk tags: - runtime_pkg From 25c2ebb563960d28a5b1d9533d92058cbdc3507c Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Mon, 24 Aug 2020 13:05:42 +1000 Subject: [PATCH 5/7] DCD-1020: Add installation and linking of Google Noto fonts as JVM fallbacks. --- roles/confluence_config/defaults/main.yml | 4 ++++ .../molecule/default/tests/test_default.py | 11 ++++++++++- roles/confluence_config/tasks/amazon_fonts.yml | 15 +++++++++++++++ roles/confluence_config/tasks/main.yml | 10 ++++++++++ roles/confluence_config/tasks/ubuntu_fonts.yml | 15 +++++++++++++++ 5 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 roles/confluence_config/tasks/amazon_fonts.yml create mode 100644 roles/confluence_config/tasks/ubuntu_fonts.yml diff --git a/roles/confluence_config/defaults/main.yml b/roles/confluence_config/defaults/main.yml index 86502fc..4bd4d10 100644 --- a/roles/confluence_config/defaults/main.yml +++ b/roles/confluence_config/defaults/main.yml @@ -1,6 +1,10 @@ --- atl_jvm_heap: "2048m" +atl_fonts_fallback_dirs: + - "{{ atl_java_home }}/lib/fonts/fallback/" + - "{{ atl_jre_home }}/lib/fonts/fallback/" + atl_autologin_cookie_age: "{{ lookup('env', 'ATL_AUTOLOGIN_COOKIE_AGE') }}" diff --git a/roles/confluence_config/molecule/default/tests/test_default.py b/roles/confluence_config/molecule/default/tests/test_default.py index 39e69c5..d3ff80e 100644 --- a/roles/confluence_config/molecule/default/tests/test_default.py +++ b/roles/confluence_config/molecule/default/tests/test_default.py @@ -78,4 +78,13 @@ def test_confluence_config_file(host): assert f.contains('ec2.amazonaws.com') assert f.contains('jdbc:postgresql://postgres-db.ap-southeast-2.rds.amazonaws.com:5432/confluence') assert f.contains('molecule_password') - assert f.contains('select 1;') \ No newline at end of file + assert f.contains('select 1;') + +@pytest.mark.parametrize('font', [ +# '/usr/lib/jvm/java/lib/fonts/fallback/NotoSansJavanese-Regular.ttf', +# '/usr/lib/jvm/jre/lib/fonts/fallback/NotoSansJavanese-Regular.ttf' + '/opt/atlassian/confluence/current/jre/lib/fonts/fallback/NotoSansJavanese-Regular.ttf' +]) +def test_fonts_installed_and_linked(host, font): + f = host.file(font) + assert f.exists diff --git a/roles/confluence_config/tasks/amazon_fonts.yml b/roles/confluence_config/tasks/amazon_fonts.yml new file mode 100644 index 0000000..4fc5302 --- /dev/null +++ b/roles/confluence_config/tasks/amazon_fonts.yml @@ -0,0 +1,15 @@ +--- + +- name: Install Google Noto fonts for language coverage + yum: + name: + - "google-noto-*" + +- name: Link the language fonts into the JDK/JRE + # Not idiomatic, but cleaner that messing with nested lookups... + shell: + cmd: "ln -sf /usr/share/fonts/google-noto*/* {{ item }}/" + creates: "{{ item }}/NotoSansJavanese-Regular.ttf" + warn: false + with_items: "{{ atl_fonts_fallback_dirs }}" + changed_when: false # For Molecule idempotence check diff --git a/roles/confluence_config/tasks/main.yml b/roles/confluence_config/tasks/main.yml index d9c4af2..07b202c 100644 --- a/roles/confluence_config/tasks/main.yml +++ b/roles/confluence_config/tasks/main.yml @@ -125,3 +125,13 @@ tags: - skip_on_stack_update ignore_errors: yes # For Molecule as it has no db test framework included + +- name: Create JVM font fallback directories + file: + path: "{{ item }}" + state: directory + mode: 0755 + with_items: "{{ atl_fonts_fallback_dirs }}" + +- name: Install & configure distro language fonts + include_tasks: "{{ ansible_distribution|lower }}_fonts.yml" diff --git a/roles/confluence_config/tasks/ubuntu_fonts.yml b/roles/confluence_config/tasks/ubuntu_fonts.yml new file mode 100644 index 0000000..084aa3f --- /dev/null +++ b/roles/confluence_config/tasks/ubuntu_fonts.yml @@ -0,0 +1,15 @@ +--- + +- name: Install Google Noto fonts for language coverage + package: + name: + - "fonts-noto" + +- name: Link the language fonts into the JDK/JRE + # Not idiomatic, but cleaner that messing with nested lookups... + shell: + cmd: "ln -sf /usr/share/fonts/truetype/noto/* {{ item }}/" + creates: "{{ item }}/NotoSansJavanese-Regular.ttf" + warn: false + with_items: "{{ atl_fonts_fallback_dirs }}" + changed_when: false # For Molecule idempotence check From db84840a96016b5cf3e10edf294da21bcb01d555 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Mon, 24 Aug 2020 13:23:17 +1000 Subject: [PATCH 6/7] DCD-1020: Add test for font installation with system JDK. --- .../molecule/system_jdk/Dockerfile.j2 | 14 ++++++++ .../molecule/system_jdk/molecule.yml | 32 +++++++++++++++++++ .../molecule/system_jdk/playbook.yml | 30 +++++++++++++++++ .../molecule/system_jdk/tests/test_default.py | 20 ++++++++++++ 4 files changed, 96 insertions(+) create mode 100644 roles/confluence_config/molecule/system_jdk/Dockerfile.j2 create mode 100644 roles/confluence_config/molecule/system_jdk/molecule.yml create mode 100644 roles/confluence_config/molecule/system_jdk/playbook.yml create mode 100644 roles/confluence_config/molecule/system_jdk/tests/test_default.py diff --git a/roles/confluence_config/molecule/system_jdk/Dockerfile.j2 b/roles/confluence_config/molecule/system_jdk/Dockerfile.j2 new file mode 100644 index 0000000..e6aa95d --- /dev/null +++ b/roles/confluence_config/molecule/system_jdk/Dockerfile.j2 @@ -0,0 +1,14 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi diff --git a/roles/confluence_config/molecule/system_jdk/molecule.yml b/roles/confluence_config/molecule/system_jdk/molecule.yml new file mode 100644 index 0000000..33c377c --- /dev/null +++ b/roles/confluence_config/molecule/system_jdk/molecule.yml @@ -0,0 +1,32 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: amazon_linux2 + image: amazonlinux:2 + groups: + - aws_node_local + - name: ubuntu_lts + image: ubuntu:bionic + groups: + - aws_node_local +provisioner: + name: ansible + options: + skip-tags: runtime_pkg + lint: + name: ansible-lint + options: + x: ["701"] + inventory: + links: + group_vars: ../../../../group_vars/ +verifier: + name: testinfra + lint: + name: flake8 + enabled: false diff --git a/roles/confluence_config/molecule/system_jdk/playbook.yml b/roles/confluence_config/molecule/system_jdk/playbook.yml new file mode 100644 index 0000000..e518a8e --- /dev/null +++ b/roles/confluence_config/molecule/system_jdk/playbook.yml @@ -0,0 +1,30 @@ +--- +- name: Converge + hosts: all + vars: + atl_product_family: "confluence" + atl_product_edition: "confluence" + atl_product_user: "confluence" + atl_product_version: "latest" + atl_db_engine: "postgres" + atl_db_host: "postgres-db.ap-southeast-2.rds.amazonaws.com" + atl_jdbc_db_name: "confluence" + atl_jdbc_user: 'confluence' + atl_jdbc_password: 'molecule_password' + atl_jvm_heap: 'PLACEHOLDER' + atl_cluster_node_id: 'FAKEID' + atl_autologin_cookie_age: "COOKIEAGE" + atl_local_ipv4: "1.1.1.1" + atl_tomcat_scheme: "http" + atl_proxy_name: "localhost" + atl_proxy_port: "80" + atl_db_preferredtestquery: "select 1;" + atl_use_system_jdk: true + atl_download_format: "tarball" + + roles: + - role: linux_common + - role: product_common + - role: product_install + - role: confluence_common + - role: confluence_config diff --git a/roles/confluence_config/molecule/system_jdk/tests/test_default.py b/roles/confluence_config/molecule/system_jdk/tests/test_default.py new file mode 100644 index 0000000..7cb395e --- /dev/null +++ b/roles/confluence_config/molecule/system_jdk/tests/test_default.py @@ -0,0 +1,20 @@ +import os +import pytest + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + +def test_seraph_file(host): + f = host.file('/opt/atlassian/confluence/current/confluence/WEB-INF/classes/seraph-config.xml') + assert f.exists + assert f.contains('COOKIEAGE') + +@pytest.mark.parametrize('font', [ + '/usr/lib/jvm/java/lib/fonts/fallback/NotoSansJavanese-Regular.ttf', + '/usr/lib/jvm/jre/lib/fonts/fallback/NotoSansJavanese-Regular.ttf' +]) +def test_fonts_installed_and_linked(host, font): + f = host.file(font) + assert f.exists From ff6889e539f1e65327a3975955675ae2e5db16ab Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Mon, 24 Aug 2020 13:34:44 +1000 Subject: [PATCH 7/7] DCD-1020: Update pipelines. --- bitbucket-pipelines.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index ca1af07..c9cd2fb 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -18,7 +18,7 @@ pipelines: - step: name: Pre Parallelization stage script: - - echo "Running tests in 37 batches" + - echo "Running tests in 38 batches" - step: name: Check if number of batches match actual number of scenarios script: @@ -327,4 +327,12 @@ pipelines: - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 37 + - step: + name: Molecule Test Batch - 38 + services: + - docker + script: + - apt-get update && ./bin/install-ansible --dev + - ./bin/run-tests-in-batches --batch 38 +