Merged in fix-bb-home (pull request #192)

Multiple fixes for Bitbucket DataCenter

Approved-by: Vadym Kovalskiy
Approved-by: Lee Goolsbee

aws_bitbucket_dc_node.yml

@@ -8,16 +8,16 @@
     atl_product_edition: "bitbucket"
     atl_product_user: "bitbucket"
 
-    atl_product_home: "{{ atl_shared_mountpoint }}/{{ atl_product_edition }}"
-
     atl_use_system_jdk: true
-    java_major_version: "11" # BB 8 will drop JDK 8 support
+    java_major_version: "17"
     atl_download_format: "tarball"
 
+    atl_product_home: "/var/atlassian/application-data/bitbucket"
     atl_nfs_mountpoint: "{{ atl_shared_mountpoint }}/bitbucket/shared"
     atl_nfs_target: "{{ atl_shared_mountpoint }}/bitbucket/shared"
     atl_nfs_version: "3"
 
+
     atl_startup_systemd_params:
       - "UMask=0027"
       - "LimitNOFILE=4096"
@@ -38,7 +38,7 @@
     - { role: nfs_mount, when : (atl_fileserver_host is defined) and (atl_fileserver_host|length > 0) }
     - role: product_common
     - role: product_install
-    - role: database_init
+    - { role: database_init, tags: [database] }
     - role: bitbucket_config
     - role: product_startup
     - role: bitbucket_dataset_restore

aws_bitbucket_mesh_node.yml

@@ -0,0 +1,34 @@
+---
+- hosts: aws_node_local
+  become: true
+
+  vars:
+    # See group_vars/aws_node_local.yml, which pull vars from the environment.
+    atl_product_family: "stash"
+    atl_product_edition: "mesh"
+    atl_product_user: "bitbucket"
+    atl_product_home: "{{ atl_home_base }}/{{ atl_product_edition }}"
+    atl_systemd_service_name: "mesh.service"
+    atl_startup_systemd_params:
+      - 'UMask=0027'
+      - 'Environment=MESH_HOME={{ atl_home_base }}/{{ atl_product_edition }}'
+      - 'Environment=JAVA_HOME=/usr/lib/jvm/java'
+      - 'Environment=JRE_HOME=/usr/lib/jvm/java'
+      - 'Environment=JMX_REMOTE_AUTH=password'
+      - 'Environment=JMX_PASSWORD_FILE={{ atl_home_base }}/{{ atl_product_edition }}/jmx.access'
+      - 'Environment="JVM_SUPPORT_RECOMMENDED_ARGS=-Dmesh.enabled=true -Dplugin.bitbucket-git.mesh.sidecar.child-process=false -Dcom.sun.management.jmxremote.port=4444 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath={{ atl_home_base }}/{{ atl_product_edition }}/log"'
+      - 'Environment=JVM_MAXIMUM_MEMORY={{ atl_jvm_heap }}'
+      - 'PassEnvironment=JMX_REMOTE_AUTH JMX_PASSWORD_FILE JAVA_HOME'
+
+    atl_startup_exec_path: "{{ mesh_install_dir }}/current/bin/start-mesh.sh"
+    atl_stop_exec_path: "{{ mesh_install_dir }}/current/bin/stop-mesh.sh"
+    atl_systemd_service_target: "multi-user.target"
+    atl_startup_exec_options: []
+
+  roles:
+    - role: linux_common
+    - role: aws_common
+    - role: aws_shared_fs_config
+    - role: product_common
+    - role: bitbucket_mesh
+    - role: product_startup

group_vars/aws_node_local.yml

@@ -107,6 +107,8 @@ atl_db_engine_to_db_type_map:
   rds_postgres: "postgres72"
 atl_db_type: "{{ atl_db_engine_to_db_type_map[atl_db_engine] | default('postgres72') }}"
 
+atl_download_secret_name: "{{ lookup('env', 'ATL_DOWNLOAD_SECRET_NAME') or '' }}"
+
 atl_jdbc_db_name: "{{ lookup('env', 'ATL_JDBC_DB_NAME') }}"
 atl_jdbc_user: "{{ lookup('env', 'ATL_JDBC_USER') }}"
 atl_jdbc_password: "{{ lookup('env', 'ATL_JDBC_PASSWORD') }}"

roles/bitbucket_config/molecule/default/converge.yml

@@ -16,6 +16,7 @@
     atl_elasticsearch_password: password
 
     atl_bitbucket_properties_raw: "key1=val1 key2=val2    key3=val3"
+    skip_shared_home_symlink: true
 
   roles:
     - role: linux_common

roles/bitbucket_config/molecule/default/tests/test_default.py

@@ -15,9 +15,9 @@ def test_config_file(host):
     assert f.contains("jdbc.user=bb_db_user")
     assert f.contains("jdbc.password=molecule_password")
 
-    assert f.contains("plugin.search.elasticsearch.username=bitbucket")
+    assert f.contains("plugin.search.config.username=bitbucket")
-    assert f.contains("plugin.search.elasticsearch.password=password")
+    assert f.contains("plugin.search.config.password=password")
-    assert not f.contains("plugin.search.elasticsearch.aws.region")
+    assert not f.contains("plugin.search.config.aws.region")
 
     assert f.contains("^key1=val1$")
     assert f.contains("^key2=val2$")

roles/bitbucket_config/molecule/iam_elasticsearch/converge.yml

@@ -13,7 +13,7 @@
     atl_jdbc_password: 'molecule_password'
 
     atl_aws_region: us-east-2
-
+    skip_shared_home_symlink: true
   roles:
     - role: linux_common
     - role: product_common

roles/bitbucket_config/molecule/iam_elasticsearch/tests/test_default.py

@@ -10,6 +10,6 @@ def test_config_file(host):
     f = host.file('/media/atl/bitbucket/shared/bitbucket.properties')
     assert f.exists
 
-    assert not f.contains("plugin.search.elasticsearch.username")
+    assert not f.contains("plugin.search.config.username")
-    assert not f.contains("plugin.search.elasticsearch.password")
+    assert not f.contains("plugin.search.config.password")
-    assert f.contains("plugin.search.elasticsearch.aws.region=us-east-2")
+    assert f.contains("plugin.search.config.aws.region=us-east-2")

roles/bitbucket_config/templates/bitbucket.properties.j2

@@ -12,15 +12,15 @@ hazelcast.network.aws.region={{ atl_aws_region }}
 hazelcast.network.aws.tag.value={{ atl_aws_stack_name }}
 hazelcast.group.name={{ atl_aws_stack_name }}
 hazelcast.group.password={{ atl_aws_stack_name }}
-plugin.search.elasticsearch.baseurl={{ atl_elasticsearch_endpoint }}
+plugin.search.config.baseurl={{ atl_elasticsearch_endpoint }}
 {% if elasticsearch_should_auth_with_iam | bool %}
-plugin.search.elasticsearch.aws.region={{ atl_aws_region }}
+plugin.search.config.aws.region={{ atl_aws_region }}
 {% else %}
-plugin.search.elasticsearch.username={{ atl_elasticsearch_username }}
+plugin.search.config.username={{ atl_elasticsearch_username }}
-plugin.search.elasticsearch.password={{ atl_elasticsearch_password }}
+plugin.search.config.password={{ atl_elasticsearch_password }}
 {% endif %}
 setup.displayName=Bitbucket
-setup.baseUrl = {{ atl_bitbucket_baseurl }}
+setup.baseUrl={{ atl_bitbucket_baseurl }}
 setup.license={{ atl_bitbucket_license_key }}
 setup.sysadmin.username=admin
 setup.sysadmin.password={{ atl_bitbucket_admin_password }}

roles/bitbucket_mesh/defaults/main.yml

@@ -0,0 +1,7 @@
+mesh_install_dir: /opt/atlassian/mesh
+bitbucket_mesh_maven_repo: https://packages.atlassian.com/maven-external
+bitbucket_mesh_version: "1.3.1"
+
+# if basic_auth is required for download of atlassian installable artifact, provide the name of an AWS Secrets Manager secret
+#  with values for both password and username
+atl_download_secret_name: ''

roles/bitbucket_mesh/handlers/main.yml

@@ -0,0 +1,19 @@
+---
+
+- name: Restart Product
+  ansible.builtin.service:
+    name: "{{ atl_systemd_service_name }}"
+    state: restarted
+  when:
+    - atl_startup_restart
+    - molecule_yml is not defined
+  no_log: true
+
+- name: Enable Product
+  ansible.builtin.service:
+    name: "{{ atl_systemd_service_name }}"
+    enabled: true
+  when:
+    - atl_startup_enable
+    - molecule_yml is not defined
+  no_log: true

roles/bitbucket_mesh/tasks/main.yml

@@ -0,0 +1,88 @@
+---
+
+- name: Create Bitbucket dirs if necessary
+  ansible.builtin.file:
+    path: "{{ item }}"
+    owner: "{{ atl_product_user_uid }}"
+    group: "{{ atl_product_user_uid }}"
+    mode: 0750
+    state: directory
+    recurse: no
+  with_items:
+    - "{{ atl_home_base }}/{{ atl_product_edition }}"
+    - "{{ atl_home_base }}/{{ atl_product_user }}"
+    - "{{ mesh_install_dir }}"
+
+
+# optionally grab basic_auth creds from secrets_manager secret called 'download_atlassian'
+- name: set basic_auth facts if the secret exists
+  ansible.builtin.set_fact:
+    download_atlassian_password: "{{ lookup('amazon.aws.aws_secret', atl_download_secret_name + '.password', region=ansible_ec2_placement_region, bypath=false, nested=true, on_denied='skip', on_missing='skip') }}"
+    download_atlassian_username: "{{ lookup('amazon.aws.aws_secret', atl_download_secret_name + '.username', region=ansible_ec2_placement_region, bypath=false, nested=true, on_denied='skip', on_missing='skip') }}"
+  failed_when: false
+  ignore_errors: yes
+  no_log: true
+  when:
+    - ansible_ec2_placement_region is defined
+    - atl_download_secret_name is defined
+  tags:
+    - runtime_pkg
+
+# Fetch binary and copy to temp
+# optionally use basic_auth creds from secrets_manager
+- name: Fetch binary
+  ansible.builtin.get_url:
+    url: "{{ atl_product_download_url }}"
+    dest: "{{ mesh_install_dir }}"
+    url_password: "{{ download_atlassian_password | default(omit) }}"
+    url_username: "{{ download_atlassian_username | default(omit) }}"
+    owner: "{{ atl_product_user }}"
+    group: "{{ atl_product_user }}"
+    mode: 0644
+    force: false
+  register: maven_download
+
+- name: extract the downloaded artifact
+  ansible.builtin.unarchive:
+    src: "{{ maven_download.dest }}"
+    dest: "/opt/atlassian/mesh/"
+    creates: "/opt/atlassian/mesh/atlassian-bitbucket-mesh-{{ atl_product_version }}"
+    mode: 0755
+    owner: "{{ atl_product_user }}"
+    group: "{{ atl_product_user }}"
+  register: mesh_extract
+  when:
+    - maven_download.changed | bool
+
+# the owner/group on the unarchive above isn't thorough
+- name: adjust permissions on the extracted directory
+  ansible.builtin.file:
+    state: directory
+    path: "/opt/atlassian/mesh/atlassian-bitbucket-mesh-{{ atl_product_version }}"
+    owner: "{{ atl_product_user }}"
+    group: "{{ atl_product_user }}"
+    recurse: yes
+
+- name: symlink to the current version
+  ansible.builtin.file:
+    src: "/opt/atlassian/mesh/atlassian-bitbucket-mesh-{{ atl_product_version }}"
+    dest: "/opt/atlassian/mesh/current"
+    state: link
+  when:
+    - mesh_extract.changed | bool
+
+- name: touch the jmx password file
+  ansible.builtin.file:
+    path: "{{ atl_home_base }}/{{ atl_product_edition }}/jmx.access"
+    state: touch
+    owner: "{{ atl_product_user_uid }}"
+    group: "{{ atl_product_user_uid }}"
+    mode: 0600
+
+# - name: template out mesh.properties
+#   ansible.builtin.template:
+#     src: mesh.properties.j2
+#     dest: "{{ atl_home_base }}/{{ atl_product_edition }}/mesh.properties"
+#     owner: "{{ atl_product_user }}"
+#     group: "{{ atl_product_user }}"
+#     mode: 0600

roles/bitbucket_mesh/templates/mesh.properties.j2

@@ -0,0 +1,10 @@
+# Listen for gRPC requests on all interfaces by default. This allows connecting to the node remotely
+grpc.server.address=0.0.0.0
+
+authentication.token={{ ansible_hostname | hash('md5') }}
+node.name={{ ansible_hostname }}
+node.id={{ ansible_hostname }}
+
+jmx.enabled={{ mesh_jmx_enabled | default(false) }}
+management.metrics.export.jmx.domain={{ mesh_jmx_export_domain | default("") }}
+metrics.tags.host={{ ansible_hostname }}

roles/product_install/defaults/main.yml

@@ -19,7 +19,8 @@ atl_download_suffix: "{{ atl_download_format_suffix_map[atl_download_format] }}"
 
 atl_release_base_url: "https://product-downloads.atlassian.com/software"
 atl_product_base_url: "{{ atl_release_base_url }}/{{ atl_product_family }}/downloads"
-atl_product_download_url: "{{ atl_product_base_url }}/atlassian-{{ atl_download_edition | default(atl_product_edition) }}-{{ atl_product_version }}{{ atl_download_suffix }}"
+atl_product_download_default_url: "{{ atl_product_base_url }}/atlassian-{{ atl_download_edition | default(atl_product_edition) }}-{{ atl_product_version }}{{ atl_download_suffix }}"
+atl_product_download_url: "{{ lookup('env', 'ATL_PRODUCT_DOWNLOAD_URL') or atl_product_download_default_url }}"
 
 atl_product_download_filename: "{{ atl_download_edition | default(atl_product_edition) }}.{{ atl_product_version }}{{ atl_download_suffix }}"
 atl_product_temp_download: "{{ atl_installer_temp }}/{{ atl_product_download_filename }}"
@@ -44,3 +45,4 @@ atl_source_obr_from_marketplace: true
 atl_servicedesk_download_file: "{{ atl_installer_temp }}/jira-servicedesk.{{ atl_product_version }}.obr"
 
 force_version_update: false
+skip_shared_home_symlink: false

roles/product_install/molecule/bitbucket_latest/converge.yml

@@ -7,6 +7,7 @@
     atl_product_user: "bitbucket"
 
     atl_product_home: "{{ atl_shared_mountpoint }}/{{ atl_product_edition }}"
+    skip_shared_home_symlink: true
   roles:
     - role: linux_common
     - role: product_common

roles/product_install/tasks/bitbucket_extra_tasks.yml

@@ -1 +0,0 @@
-no_op.yml

roles/product_install/tasks/bitbucket_extra_tasks.yml

@@ -0,0 +1,17 @@
+---
+- name: Create the product local home directory
+  ansible.builtin.file:
+    path: "{{ atl_product_home }}"
+    state: directory
+    owner: "{{ atl_product_user }}"
+    group: "{{ atl_product_user }}"
+    mode: "0754"
+  changed_when: false  # For Molecule idempotence check
+
+- name: Symlink the the shared home
+  ansible.builtin.file:
+    src: "{{ atl_shared_mountpoint }}/{{ atl_product_edition }}/shared"
+    dest: "{{ atl_product_home }}/shared"
+    state: link
+    force: true
+  when: not skip_shared_home_symlink | bool

roles/product_install/tasks/main.yml

@@ -1,6 +1,4 @@
 ---
-
-
 - name: Check for existing version cache file
   ansible.builtin.stat:
     path: "{{ atl_product_version_cache }}"

roles/product_startup/templates/product.service.j2

@@ -17,7 +17,7 @@ StandardError=journal+console
 
 {# Bitbucket DC sets up its service start/stop logging in a different way to all our other DC offerings and so we exclude it from having the catalina.out redirect and foreground startup #}
 {% if (atl_product_family == "stash") %}
-PIDFile={{ atl_product_home }}/log/bitbucket.pid
+PIDFile={{ atl_product_home }}/log/{{ atl_product_edition }}.pid
 ExecStart={{ atl_startup_exec_path }}{% for c in atl_startup_exec_options %} {{ c }}{% endfor %}
 
 ExecStop={{ atl_stop_exec_path }}