first pass at incorporating mesh setup/config

Pipfile

@@ -10,6 +10,7 @@ boto3 = "==1.26.158"
 botocore = "==1.29.158"
 lxml = "==4.9.2"
 psycopg2-binary = "==2.9.6"
+petname = "==2.6"
 
 [dev-packages]
 molecule = "==4.0.4"

aws_bitbucket_mesh_node.yml

@@ -20,8 +20,8 @@
       - 'Environment=JVM_MAXIMUM_MEMORY={{ atl_jvm_heap }}'
       - 'PassEnvironment=JMX_REMOTE_AUTH JMX_PASSWORD_FILE JAVA_HOME'
 
-    atl_startup_exec_path: "{{ mesh_install_dir }}/current/bin/start-mesh.sh"
-    atl_stop_exec_path: "{{ mesh_install_dir }}/current/bin/stop-mesh.sh"
+    atl_startup_exec_path: "{{ atl_product_installation_base }}/current/bin/start-mesh.sh"
+    atl_stop_exec_path: "{{ atl_product_installation_base }}/current/bin/stop-mesh.sh"
     atl_systemd_service_target: "multi-user.target"
     atl_startup_exec_options: []
 
@@ -30,5 +30,7 @@
     - role: aws_common
     - role: aws_shared_fs_config
     - role: product_common
-    - role: bitbucket_mesh
+    - role: product_install
+      tags: [skip_on_stack_update]
+    - role: bitbucket_mesh_config
     - role: product_startup

roles/bitbucket_mesh/defaults/main.yml

@@ -1,7 +0,0 @@
-mesh_install_dir: /opt/atlassian/mesh
-bitbucket_mesh_maven_repo: https://packages.atlassian.com/maven-external
-bitbucket_mesh_version: "1.3.1"
-
-# if basic_auth is required for download of atlassian installable artifact, provide the name of an AWS Secrets Manager secret
-#  with values for both password and username
-atl_download_secret_name: ''

roles/bitbucket_mesh/handlers/main.yml

@@ -1,19 +0,0 @@
----
-
-- name: Restart Product
-  ansible.builtin.service:
-    name: "{{ atl_systemd_service_name }}"
-    state: restarted
-  when:
-    - atl_startup_restart
-    - molecule_yml is not defined
-  no_log: true
-
-- name: Enable Product
-  ansible.builtin.service:
-    name: "{{ atl_systemd_service_name }}"
-    enabled: true
-  when:
-    - atl_startup_enable
-    - molecule_yml is not defined
-  no_log: true

roles/bitbucket_mesh/tasks/main.yml

@@ -1,88 +0,0 @@
----
-
-- name: Create Bitbucket dirs if necessary
-  ansible.builtin.file:
-    path: "{{ item }}"
-    owner: "{{ atl_product_user_uid }}"
-    group: "{{ atl_product_user_uid }}"
-    mode: 0750
-    state: directory
-    recurse: no
-  with_items:
-    - "{{ atl_home_base }}/{{ atl_product_edition }}"
-    - "{{ atl_home_base }}/{{ atl_product_user }}"
-    - "{{ mesh_install_dir }}"
-
-
-# optionally grab basic_auth creds from secrets_manager secret called 'download_atlassian'
-- name: set basic_auth facts if the secret exists
-  ansible.builtin.set_fact:
-    download_atlassian_password: "{{ lookup('amazon.aws.aws_secret', atl_download_secret_name + '.password', region=ansible_ec2_placement_region, bypath=false, nested=true, on_denied='skip', on_missing='skip') }}"
-    download_atlassian_username: "{{ lookup('amazon.aws.aws_secret', atl_download_secret_name + '.username', region=ansible_ec2_placement_region, bypath=false, nested=true, on_denied='skip', on_missing='skip') }}"
-  failed_when: false
-  ignore_errors: yes
-  no_log: true
-  when:
-    - ansible_ec2_placement_region is defined
-    - atl_download_secret_name is defined
-  tags:
-    - runtime_pkg
-
-# Fetch binary and copy to temp
-# optionally use basic_auth creds from secrets_manager
-- name: Fetch binary
-  ansible.builtin.get_url:
-    url: "{{ atl_product_download_url }}"
-    dest: "{{ mesh_install_dir }}"
-    url_password: "{{ download_atlassian_password | default(omit) }}"
-    url_username: "{{ download_atlassian_username | default(omit) }}"
-    owner: "{{ atl_product_user }}"
-    group: "{{ atl_product_user }}"
-    mode: 0644
-    force: false
-  register: maven_download
-
-- name: extract the downloaded artifact
-  ansible.builtin.unarchive:
-    src: "{{ maven_download.dest }}"
-    dest: "/opt/atlassian/mesh/"
-    creates: "/opt/atlassian/mesh/atlassian-bitbucket-mesh-{{ atl_product_version }}"
-    mode: 0755
-    owner: "{{ atl_product_user }}"
-    group: "{{ atl_product_user }}"
-  register: mesh_extract
-  when:
-    - maven_download.changed | bool
-
-# the owner/group on the unarchive above isn't thorough
-- name: adjust permissions on the extracted directory
-  ansible.builtin.file:
-    state: directory
-    path: "/opt/atlassian/mesh/atlassian-bitbucket-mesh-{{ atl_product_version }}"
-    owner: "{{ atl_product_user }}"
-    group: "{{ atl_product_user }}"
-    recurse: yes
-
-- name: symlink to the current version
-  ansible.builtin.file:
-    src: "/opt/atlassian/mesh/atlassian-bitbucket-mesh-{{ atl_product_version }}"
-    dest: "/opt/atlassian/mesh/current"
-    state: link
-  when:
-    - mesh_extract.changed | bool
-
-- name: touch the jmx password file
-  ansible.builtin.file:
-    path: "{{ atl_home_base }}/{{ atl_product_edition }}/jmx.access"
-    state: touch
-    owner: "{{ atl_product_user_uid }}"
-    group: "{{ atl_product_user_uid }}"
-    mode: 0600
-
-# - name: template out mesh.properties
-#   ansible.builtin.template:
-#     src: mesh.properties.j2
-#     dest: "{{ atl_home_base }}/{{ atl_product_edition }}/mesh.properties"
-#     owner: "{{ atl_product_user }}"
-#     group: "{{ atl_product_user }}"
-#     mode: 0600

roles/bitbucket_mesh/templates/mesh.properties.j2

@@ -1,10 +0,0 @@
-# Listen for gRPC requests on all interfaces by default. This allows connecting to the node remotely
-grpc.server.address=0.0.0.0
-
-authentication.token={{ ansible_hostname | hash('md5') }}
-node.name={{ ansible_hostname }}
-node.id={{ ansible_hostname }}
-
-jmx.enabled={{ mesh_jmx_enabled | default(false) }}
-management.metrics.export.jmx.domain={{ mesh_jmx_export_domain | default("") }}
-metrics.tags.host={{ ansible_hostname }}

roles/bitbucket_mesh_config/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+bitbucket_mesh_maven_repo: https://packages.atlassian.com/maven-external
+bitbucket_mesh_version: "1.3.1"
+mesh_properties_file: "{{ atl_product_home }}/mesh.properties"
+mesh_node_scheme: "http"
+mesh_node_port: "7777"

roles/bitbucket_mesh_config/handlers/main.yml

@@ -0,0 +1,65 @@
+---
+# Handlers are executed in the order they are defined in the handlers section, not in the order listed in the notify statement.
+
+- name: Restart Product
+  ansible.builtin.service:
+    name: "{{ atl_systemd_service_name }}"
+    state: restarted
+  when:
+    - atl_startup_restart
+    - molecule_yml is not defined
+  no_log: true
+
+- name: Enable Product
+  ansible.builtin.service:
+    name: "{{ atl_systemd_service_name }}"
+    enabled: true
+  when:
+    - atl_startup_enable
+    - molecule_yml is not defined
+  no_log: true
+
+- name: register_mesh_node
+  ansible.builtin.uri:
+    url: '{{ atl_proxy_name }}/rest/api/latest/admin/git/mesh/nodes'
+    method: POST
+    headers:
+      Content-Type: 'application/json'
+      Accept: 'application/json'
+      Authorization: 'Basic {{ mesh_basic_auth }}'
+    timeout: 60
+    body_format: json
+    body:
+      name: "{{ mesh_node_name }}"
+      rpcUrl: "{{ mesh_node_scheme }}://{{ mesh_node_name }}.{{ stack_name }}-{{ ansible_ec2_placement_region }}.{{ atl_hostedzone }}:{{ mesh_node_port }}"
+  register: register_mesh_node_result
+  until: register_mesh_node_result is not failed
+  retries: 5
+  delay: 30
+  no_log: True
+
+- name: update_mesh_node
+  ansible.builtin.uri:
+    url: '{{ atl_proxy_name }}/rest/api/latest/admin/git/mesh/nodes/{{ mesh_node_id }}'
+    method: PUT
+    headers:
+      Content-Type: 'application/json'
+      Accept: 'application/json'
+      Authorization: 'Basic {{ mesh_basic_auth }}'
+    timeout: 60
+    body_format: json
+    body:
+      id: "{{ mesh_node_id }}"
+      name: "{{ mesh_node_name }}"
+      rpcId: "{{ mesh_node_id }}"
+      rpcUrl: "{{ mesh_node_scheme }}://{{ mesh_node_name }}.{{ stack_name }}-{{ ansible_ec2_placement_region }}.{{ atl_hostedzone }}:{{ mesh_node_port }}"
+  register: register_mesh_node_result
+  until: register_mesh_node_result is not failed
+  retries: 5
+  delay: 30
+  no_log: True
+
+- name: Remove orphan_vol lease_lock
+  ansible.builtin.file:
+    path: '{{ atl_shared_mountpoint }}/.{{ orphan_vol }}'
+    state: absent

roles/bitbucket_mesh_config/tasks/bitbucket_api_setup.yml

@@ -0,0 +1,30 @@
+#  handle setting up to call the bitbucket api
+
+- name: set_fact for the MeshRegistrationCredentialsSecret
+  ansible.builtin.set_fact:
+    mesh_registration_credential_secret: "{{ cfnFacts.cloudformation[stack_name].stack_resources.MeshRegistrationCredentialsSecret }}"
+  # NOTE: we decided we want to barf early if the secret doesnt exist as ll the rest of this is somewhat useless if we cant register nodes
+  # when:
+  #   - cfnFacts.cloudformation[stack_name].stack_resources.MeshRegistrationCredentialsSecret is defined
+
+- name: get the mesh registration username
+  ansible.builtin.set_fact:
+    mesh_registration_username: "{{ lookup('amazon.aws.secretsmanager_secret', secretpath, region=ansible_ec2_placement_region, bypath=false, nested=true) }}"
+  ignore_errors: no
+  vars:
+    secretpath: "{{ cfnFacts.cloudformation[stack_name].stack_resources.MeshRegistrationCredentialsSecret }}.username"
+
+- name: get the mesh registration password
+  ansible.builtin.set_fact:
+    mesh_registration_password: "{{ lookup('amazon.aws.secretsmanager_secret', secretpath, region=ansible_ec2_placement_region, bypath=false, nested=true) }}"
+  ignore_errors: no
+  vars:
+    secretpath: "{{ cfnFacts.cloudformation[stack_name].stack_resources.MeshRegistrationCredentialsSecret }}.password"
+  no_log: True
+
+- name: generate the basic auth string
+  ansible.builtin.set_fact:
+    mesh_basic_auth: "{{ auth_string | b64encode }}"
+  vars:
+    auth_string: "{{ mesh_registration_username }}:{{ mesh_registration_password }}"
+  no_log: True

roles/bitbucket_mesh_config/tasks/main.yml

@@ -0,0 +1,134 @@
+---
+
+- name: Create Bitbucket dirs if necessary
+  ansible.builtin.file:
+    path: "{{ item }}"
+    owner: "{{ atl_product_user_uid }}"
+    group: "{{ atl_product_user_uid }}"
+    mode: 0750
+    state: directory
+    recurse: no
+  with_items:
+    - "{{ atl_home_base }}/{{ atl_product_edition }}"
+    - "{{ atl_home_base }}/{{ atl_product_user }}"
+    - "{{ atl_product_installation_base }}"
+
+# the owner/group on the unarchive above isn't thorough
+- name: adjust permissions on the extracted directory
+  ansible.builtin.file:
+    state: directory
+    path: "/opt/atlassian/mesh/atlassian-bitbucket-mesh-{{ atl_product_version }}"
+    owner: "{{ atl_product_user }}"
+    group: "{{ atl_product_user }}"
+    recurse: yes
+
+- name: touch the jmx password file
+  ansible.builtin.file:
+    path: "{{ atl_home_base }}/{{ atl_product_edition }}/jmx.access"
+    state: touch
+    owner: "{{ atl_product_user_uid }}"
+    group: "{{ atl_product_user_uid }}"
+    mode: 0600
+
+# assume bbdc secret exists (else fail) - setup stuff for calling bitbucket rest api and get mesh node report
+# get the registered mesh node info as we need it for determining nodenames
+- name: Import bitbucket api setup pieces and do initial mesh node_report
+  ansible.builtin.import_tasks:
+    file: bitbucket_api_setup.yml
+
+- name: import/run mesh_node_report
+  ansible.builtin.import_tasks:
+    file: mesh_node_report.yml
+
+- name: set fact if this stack is a mesh clone
+  ansible.builtin.set_fact:
+    mesh_clone: "{{ (atl_mesh_snapshot_az1 is defined) | ternary(true, false) }}"
+
+# workout if there is a valid mesh_vol, either mounted or abandoned, set mesh_vol or mesh_snap
+- name: check for valid mesh_vol
+  ansible.builtin.import_tasks:
+    file: meshvol_check.yml
+
+# create/set mesh_vol if still needed
+- name: create/set mesh_vol if still needed
+  ansible.builtin.import_tasks:
+    file: meshvol_create.yml
+  when:
+    - (mesh_vol is not defined) or (mesh_vol | length == 0)
+
+# ensure logs and cache have fs and are mounted
+# action this block when mesh volume is not in mount list
+- name: setup and mount mesh releated filesystems
+  ansible.builtin.import_tasks:
+    file: meshvols_setup.yml
+  when:
+    - atl_product_home not in mounts_list
+    - "'/dev/xvdd' not in devices_list"
+
+# Setup Mesh product
+- name: setup mesh product and related pieces
+  ansible.builtin.import_tasks:
+    file: mesh_setup.yml
+
+# get tag info of mesh_vol
+- name: get info of attached mesh_vol
+  amazon.aws.ec2_vol_info:
+    region: "{{ ansible_ec2_placement_region }}"
+    filters:
+      volume-id: "{{ mesh_vol }}"
+  register: mesh_vol_tag_info
+  tags: notest # doesn't work in molecule
+
+- name: set mesh_node_name_tag var from the mesh_node_name tag of .mesh_vol
+  ansible.builtin.set_fact:
+    mesh_node_name: "{{ mesh_vol_tag_info.volumes[0].tags['mesh_node_name'] }}"
+  ignore_errors: yes
+
+# workout if we are a clone_firstrun
+- name: remove node_id from mesh.properties if new_meshvol_from_snap
+  ansible.builtin.lineinfile:
+    path: "{{ mesh_properties_file }}"
+    regexp: '^node\.id.+$'
+    state: absent
+  when:
+    - new_meshvol_from_snap is defined
+    - new_meshvol_from_snap | bool
+
+- name: set mesh_node_id var from the existing mesh.properties file
+  ansible.builtin.set_fact:
+    mesh_node_id: "{{ lookup('ansible.builtin.ini', 'node.id type=properties file=mesh_properties_file', errors='ignore' )| default('absent', true) }}"
+
+- name: set string to test mesh_clone_firstrun (node_name contains meshnode_{{ mesh_node_id }}")
+  ansible.builtin.set_fact:
+    mesh_clone_teststring: "meshnode_{{ mesh_node_id }}"
+
+# workout if this is mesh_clone_firstrun - either no mounted volume or teststring found in mesh_nodes_report
+- name: work out if this is firstrun of a mesh_clone
+  ansible.builtin.set_fact:
+    mesh_clone_firstrun: false
+
+- name: set firstrun true if mesh_node_id is absent
+  ansible.builtin.set_fact:
+    mesh_clone_firstrun: true
+  when: mesh_node_id == 'absent'
+
+# if meshvol has no petname or this is the first run of a clone, create a new petname, tag the vol and set mesh_node_name
+- name: new petname related processing
+  ansible.builtin.import_tasks:
+    file: mesh_newpet.yml
+  when: (mesh_node_name is not defined) or (mesh_node_name | length == 0) or mesh_clone_firstrun
+
+# bundle all the node_name releated updates together
+- name: mesh_node_name related processing
+  ansible.builtin.import_tasks:
+    file: mesh_node_name.yml
+
+# handle reregistering the correct node if node.id exists
+- name: set mesh_node_id var from the existing mesh.properties file
+  ansible.builtin.set_fact:
+    mesh_node_id: "{{ lookup('ansible.builtin.ini', 'node.id type=properties file=mesh_properties_file') | split('.') | first }}"
+
+# handle functionality relating to starting and registering the mesh service
+- name: mesh service related processing
+  ansible.builtin.import_tasks:
+    file: mesh_service.yml

roles/bitbucket_mesh_config/tasks/mesh_newpet.yml

@@ -0,0 +1,12 @@
+---
+# we just want to set the node.name for now, other values will get auto populated as the mesh node connects
+# this may need to be expanded later, but don't add node.id unless you get it from bitbucket primary cluster
+# first see if it's already set - if it is, use that, else generate a random one
+
+- name: set mesh_node_name var from the existing mesh.properties file
+  ansible.builtin.set_fact:
+    mesh_node_name: "{{ lookup('ansible.builtin.ini', 'node.name type=properties file=mesh_properties_file') | split('.') | first }}"
+
+- name: generate a pet name if mesh_node_name is empty or this is a clone firstrun # noqa jinja[invalid]
+  ansible.builtin.set_fact:
+    mesh_node_name: "{{ lookup('community.general.random_pet', words=3) }}"

roles/bitbucket_mesh_config/tasks/mesh_node_name.yml

@@ -0,0 +1,65 @@
+---
+  # update mesh.properties with the mesh_node_name from the tag in mesh_vol_info
+- name: enforce node.name per volume tag
+  ansible.builtin.lineinfile:
+    line: node.name={{ mesh_node_name }}
+    path: "{{ mesh_properties_file }}"
+    regexp: '^node\.name='
+
+- name: enforce node.rpc-url per node_name
+  ansible.builtin.lineinfile:
+    path: "{{ mesh_properties_file }}"
+    search_string: 'node.rpc-url='
+    line: node.rpc-url={{ mesh_node_scheme }}://{{ mesh_node_name }}.{{ stack_name }}-{{ ansible_ec2_placement_region }}.{{ atl_hostedzone }}:{{ mesh_node_port }}
+
+- name: enforce metrics.tags.host per volume tag
+  ansible.builtin.lineinfile:
+    path: "{{ mesh_properties_file }}"
+    search_string: 'metrics.tags.host='
+    line: metrics.tags.host={{ mesh_node_name }}
+
+# the idea here is that the pet name will persist through clones, and just the stack_name - region part of the url will change (and maybe the hosted zone)
+- name: create r53 entry for this mesh node based on the mesh_node_name
+  amazon.aws.route53:
+    command: create
+    zone: "{{ atl_hostedzone }}"
+    record: "{{ mesh_node_name }}.{{ stack_name }}-{{ ansible_ec2_placement_region }}.{{ atl_hostedzone }}"
+    type: A
+    ttl: 300
+    value: "{{ ansible_default_ipv4.address }}"
+    overwrite: "yes"
+    private_zone: "yes"
+  when:
+    - ansible_default_ipv4.address | ansible.utils.ipaddr
+    - mesh_vol is regex('^vol-.*')
+  tags: notest # doesn't work in molecule
+
+- name: build a dict of tags from the ec2 instance to apply to the ebs volume
+  ansible.builtin.set_fact:
+    ebs_tags: "{{ ebs_tags | default({}) | combine({item.key: item.value}) }}"
+  when:
+    - item.key not in ['Name']
+    - item.key is not match("aws:.*")
+  with_dict: "{{ ec2_instance_tags }}"
+
+- name: apply the tags from the ec2 instance to the ebs volume
+  amazon.aws.ec2_tag:
+    region: "{{ ansible_ec2_placement_region }}"
+    resource: "{{ ((mesh_vol is defined) and (mesh_vol | length > 0)) | ternary(mesh_vol, mesh_vol_info.volume.id) }}"
+    tags: "{{ ebs_tags }}"
+    state: present
+    purge_tags: false
+  tags: notest # doesn't work in molecule
+
+- name: add the mesh_node_name tag to the ebs volume and current ec2
+  amazon.aws.ec2_tag:
+    region: "{{ ansible_ec2_placement_region }}"
+    resource: "{{ item }}"
+    tags:
+      mesh_node_name: "{{ mesh_node_name }}"
+    state: present
+    purge_tags: false
+  tags: notest # doesn't work in molecule
+  with_items:
+    - "{{ ((mesh_vol is defined) and (mesh_vol | length > 0)) | ternary(mesh_vol, mesh_vol_info.volume.id) }}"
+    - "{{ ansible_ec2_instance_id }}"

roles/bitbucket_mesh_config/tasks/mesh_node_report.yml

@@ -0,0 +1,45 @@
+# create a report on existing mesh nodes
+
+- block:
+  - name: try calling the server to get a list of the mesh nodes
+    ansible.builtin.uri:
+      url: '{{ atl_proxy_name }}/rest/api/latest/admin/git/mesh/nodes'
+      method: GET
+      headers:
+        Content-Type: 'application/json'
+        Accept: 'application/json'
+        Authorization: 'Basic {{ mesh_basic_auth }}'
+      timeout: 60
+    register: mesh_nodes_report
+    retries: 5
+    delay: 10
+    no_log: True # http headers contain auth strings
+
+  - name: set bitbucket dc cluster available fact
+    ansible.builtin.set_fact:
+      bitbucket_cluster_available: true
+
+  - name: print out the mesh_nodes_report
+    ansible.builtin.debug:
+      msg: "{{ mesh_nodes_report.json }}"
+    when:
+      - mesh_nodes_report.status == 200
+
+  - name: set mesh_nodenames_list
+    ansible.builtin.set_fact:
+      mesh_nodenames_list: "{{ mesh_nodes_report | json_query('json[*].name') }}"
+    when: mesh_nodes_report.status == 200
+
+  - name: print out the mesh_nodenames_list
+    ansible.builtin.debug:
+      var: mesh_nodenames_list
+    when: mesh_nodes_report.status == 200
+
+  rescue:
+    - name: set bitbucket dc cluster available fact
+      ansible.builtin.set_fact:
+        bitbucket_cluster_available: false
+
+    - name: log a message about bb cluster unavailable
+      ansible.builtin.debug:
+        msg: "Bitbucket DC cluster not currently available."

roles/bitbucket_mesh_config/tasks/mesh_service.yml

@@ -0,0 +1,58 @@
+---
+- name: Enable and Start Mesh Product
+  ansible.builtin.service:
+    name: "{{ atl_systemd_service_name }}"
+    enabled: true
+    state: started
+  when:
+    - molecule_yml is not defined
+  no_log: true
+
+- name: Wait for service to become available
+  ansible.builtin.wait_for:
+    port: "{{ mesh_node_port }}"
+    timeout: 300
+  delegate_to: localhost
+  become: false
+  register: mesh_service_waiter
+
+- name: trigger the handler to update this existing mesh node
+  ansible.builtin.assert:
+    that:
+      - mesh_nodes_report.json | community.general.json_query(query) | length == 0
+  changed_when: true
+  notify: update_mesh_node
+  when:
+    - bitbucket_cluster_available
+    - mesh_nodes_report.json | community.general.json_query(query) | length == 0
+    - (mesh_node_id is defined) and (mesh_node_id|length > 0)
+  vars:
+    query: "[?name=='{{ mesh_node_name }}']"
+
+# if node.id doesnt exist this is new/clean mesh node and we need to register a new node
+- name: if this node is not registered, cleanup any old keys to allow registration of new
+  ansible.builtin.file:
+    path: "{{ atl_home_base }}/{{ atl_product_edition }}/config/{{ item }}"
+    state: absent
+  with_items:
+    - signing-key.pem
+    - control-plane.pem
+  when:
+    - bitbucket_cluster_available
+    - mesh_nodes_report.json | community.general.json_query(query) | length == 0
+    - (mesh_node_id is not defined) or (mesh_node_id|length == 0)
+  vars:
+    query: "[?name=='{{ mesh_node_name }}']"
+
+- name: trigger the handler to register this new mesh node
+  ansible.builtin.assert:
+    that:
+      - mesh_nodes_report.json | community.general.json_query(query) | length == 0
+  changed_when: true
+  notify: register_mesh_node
+  when:
+    - bitbucket_cluster_available
+    - mesh_nodes_report.json | community.general.json_query(query) | length == 0
+    - (mesh_node_id is not defined) or (mesh_node_id|length == 0)
+  vars:
+    query: "[?id=='{{ mesh_node_id }}']"

roles/bitbucket_mesh_config/tasks/mesh_setup.yml

@@ -0,0 +1,96 @@
+---
+- name: Create Bitbucket dirs if necessary
+  ansible.builtin.file:
+    path: "{{ item }}"
+    owner: "{{ atl_product_user }}"
+    group: "{{ atl_product_user }}"
+    mode: 0750
+    state: directory
+    recurse: no
+  with_items:
+    - "{{ atl_home_base }}/{{ atl_product_edition }}"
+    - "{{ mesh_install_dir }}"
+    - "{{ atl_product_home }}"
+    - "{{ atl_product_home }}/caches"
+    - "{{ atl_product_home }}/log"
+
+# - name: download the mesh distribution using maven
+#   community.general.maven_artifact:
+#     group_id: "com.atlassian.bitbucket.mesh"
+#     artifact_id: "mesh-distribution"
+#     extension: "tar.gz"
+#     version: "{{ atl_product_version }}"
+#     repository_url: "{{ bitbucket_mesh_maven_repo }}"
+#     dest: "{{ downloads_dir }}"
+#     keep_name: yes
+#     mode: "0644"
+#     owner: "{{ atl_product_user }}"
+#     group: "{{ atl_product_user }}"
+#   register: maven_download
+
+# - name: extract the downloaded artifact # noqa no-handler
+#   ansible.builtin.unarchive:
+#     src: "{{ maven_download.dest }}"
+#     dest: "/opt/atlassian/mesh/"
+#     creates: "/opt/atlassian/mesh/atlassian-bitbucket-mesh-{{ atl_product_version }}"
+#     mode: 0755
+#     owner: "{{ atl_product_user }}"
+#     group: "{{ atl_product_user }}"
+#   register: mesh_extract
+#   when:
+#     - maven_download.changed | bool
+
+# the owner/group on the unarchive above isn't thorough
+- name: adjust permissions on the extracted directory
+  ansible.builtin.file:
+    state: directory
+    path: "/opt/atlassian/mesh/atlassian-bitbucket-mesh-{{ atl_product_version }}"
+    owner: "{{ atl_product_user }}"
+    group: "{{ atl_product_user }}"
+    recurse: yes
+
+# - name: symlink to the current version # noqa no-handler
+#   ansible.builtin.file:
+#     src: "/opt/atlassian/mesh/atlassian-bitbucket-mesh-{{ atl_product_version }}"
+#     dest: "/opt/atlassian/mesh/current"
+#     state: link
+#   when:
+#     - mesh_extract.changed | bool
+
+- name: touch the jmx password file
+  ansible.builtin.file:
+    path: "{{ atl_home_base }}/{{ atl_product_edition }}/jmx.access"
+    state: touch
+    owner: "{{ atl_product_user }}"
+    group: "{{ atl_product_user }}"
+    mode: 0600
+
+- name: enforce the permissions on the pem files
+  ansible.builtin.file:
+    path: "{{ atl_home_base }}/{{ atl_product_edition }}/config/{{ item }}"
+    state: touch
+    owner: "{{ atl_product_user }}"
+    group: "{{ atl_product_user }}"
+    mode: 0600
+  with_items:
+    - signing-key.pem
+    - control-plane.pem
+  register: pem_chown
+  failed_when:
+    - pem_chown.failed | bool
+    - "'No such file or directory' not in pem_chown.msg"
+
+- name: touch mesh.properties to make sure it exists
+  ansible.builtin.file:
+    state: touch
+    path: "{{ mesh_properties_file }}"
+    owner: "{{ atl_product_user }}"
+    group: "{{ atl_product_user }}"
+    mode: 0600
+
+# check for lines starting with '!' and prefix them with #
+- name: prefix properties files bangs with hashes
+  ansible.builtin.replace:
+    path: "{{ mesh_properties_file }}"
+    regexp: '^(\!.*)$'
+    replace: '# \1'

roles/bitbucket_mesh_config/tasks/meshvol_check.yml

@@ -0,0 +1,117 @@
+---
+# check if meshvol mounted or for an abandoned mesh vol on the current AZ
+# at the end of this module we should have mesh_vol set if there is one to find
+# if we dont have mesh_vol then we need to either hydrate one from snap or make a new/empty one
+
+# first, try to set mesh_vol from currently mounted volume
+- name: create list of current mounts
+  ansible.builtin.set_fact:
+    mounts_list: "{{ ansible_mounts | map(attribute='mount') | list }}"
+
+# - name: discover mesh volume device stat
+#   ansible.builtin.stat:
+#     path: '/dev/xvdd'
+#     follow: no
+#     get_attributes: yes
+#   register: mesh_device_stat
+
+- name: pull info for this_instance from instanceFacts
+  ansible.builtin.set_fact:
+    this_instance: "{{ instanceFacts.instances | first }}"
+
+- name: create list of current block devices
+  ansible.builtin.set_fact:
+    devices_list: "{{ this_instance.block_device_mappings | map(attribute='device_name') | list }}"
+
+- name: discover mesh volume device stat
+  ansible.builtin.stat:
+    path: '/dev/xvdd'
+    follow: no
+    get_attributes: yes
+  register: mesh_device_stat
+
+- name: describe mesh volume true device
+  ansible.builtin.set_fact:
+    mesh_device: "{{ mesh_device_stat.stat['lnk_source'] }}"
+  ignore_errors: yes
+  when: '"lnk_source" in mesh_device_stat.stat'
+
+# lazy guess at mesh_vol - # TODO make this better
+- name: set mesh_vol from currently mounted device
+  ansible.builtin.set_fact:
+    mesh_vol: "{{ this_instance | community.general.json_query(vol_id_query) | first }}"
+  vars:
+    vol_id_query: "block_device_mappings[?(@.device_name=='/dev/xvdd')].ebs.volume_id"
+  when:  (mesh_device is defined) and (mesh_device|length > 0)
+
+- name: shall we check for abandoned?
+  ansible.builtin.set_fact:
+    abandoned_check: "{{ ((mesh_vol is not defined) or (mesh_vol|length == 0)) | ternary(true, false) }}"
+
+# second, if we dont have a mesh_vol yet, try to set it from abandoned
+- block:
+  - name: check for abandoned ebs vols of this stack in current AZ
+    amazon.aws.ec2_vol_info:
+      region: "{{ ansible_ec2_placement_region }}"
+      filters:
+        status: 'available'
+        "tag:service_name": "{{ stack_name }}"
+        "tag:Name": "{{ stack_name }} mesh volume {{ ansible_ec2_placement_availability_zone }}"
+    register: orphan_vol_info
+    tags: notest  # doesn't work in molecule
+
+  - name: Extract orphan volume_id values using json_query
+    ansible.builtin.set_fact:
+      orphan_vol_list: "{{ orphan_vol_info | json_query('volumes[*].id') }}"
+
+  - name: loop to get an orphan_lease_lock (temporary lock on volume cleaned up in handler)
+    ansible.builtin.include_tasks: orphan_lease_lock.yml
+    with_items: "{{ orphan_vol_list }}"
+    loop_control:
+      loop_var: test_orphan_vol
+    when: orphan_vol is not defined
+
+  - name: set mesh_vol from orphan_vol
+    ansible.builtin.set_fact:
+      mesh_vol: "{{ orphan_vol }}"
+    when:
+      - orphan_vol is defined
+      - orphan_vol | length > 0
+
+  - name: attach abandoned mesh vol to this ec2
+    amazon.aws.ec2_vol:
+      instance: "{{ ansible_ec2_instance_id }}"
+      id: "{{ mesh_vol }}"
+      region: "{{ ansible_ec2_placement_region }}"
+      device_name: /dev/xvdd
+      delete_on_termination: no
+    when:
+      - mesh_vol is defined
+      - mesh_vol | length > 0
+
+  when: abandoned_check
+
+# third, if no mesh_vol and this is a clone, set mesh_snap
+- block:
+  - name: create az dict attributes
+    ansible.builtin.set_fact:
+      my_attributes:
+        - key: "{{ atl_mesh_azname_az1 }}"
+          value: "{{ atl_mesh_snapshot_az1 }}"
+        - key: "{{ atl_mesh_azname_az2 }}"
+          value: "{{ atl_mesh_snapshot_az2 }}"
+        - key: "{{ atl_mesh_azname_az3 }}"
+          value: "{{ atl_mesh_snapshot_az3 }}"
+
+  - name: create az dict if this stack is a mesh clone
+    ansible.builtin.set_fact:
+      azsnapdict: "{{ my_attributes | items2dict }}"
+
+  - name: set mesh clone node snapshot fact if this stack is a mesh clone
+    ansible.builtin.set_fact:
+      mesh_snapshot: "{{ azsnapdict[ansible_ec2_placement_availability_zone] }}"
+
+  ignore_errors: yes
+  when:
+   - (mesh_vol is not defined) or (mesh_vol|length == 0)
+   - mesh_clone

roles/bitbucket_mesh_config/tasks/meshvol_create.yml

@@ -0,0 +1,49 @@
+---
+# hydrate mesh volume from snapshot or create empty one if no snapshot set
+- name: create new ebs from snapshot for this mesh node with correct tags
+  amazon.aws.ec2_vol:
+    region: "{{ ansible_ec2_placement_region }}"
+    instance: "{{ ansible_ec2_instance_id }}"
+    volume_type: gp3
+    throughput: 250
+    device_name: /dev/xvdd
+    snapshot: "{{ mesh_snapshot }}"
+    tags:
+      service_name: "{{ stack_name }}"
+      Name: "{{ stack_name }} mesh volume {{ ansible_ec2_placement_availability_zone }}"
+      volume_type: "mesh"
+  register: mesh_vol_info
+  when: (mesh_snapshot is defined) and (mesh_snapshot|length > 0)
+
+- name: set mesh_vol from info of vol created from snapshot
+  ansible.builtin.set_fact:
+    mesh_vol: "{{ mesh_vol_info.volume_id }}"
+    new_meshvol_from_snap: true
+  when: (mesh_snapshot is defined) and (mesh_snapshot|length > 0)
+
+# if we still dont have mesh_vol, create new/empty one
+- name: final choice - create empty mesh_vol
+  amazon.aws.ec2_vol:
+    region: "{{ ansible_ec2_placement_region }}"
+    instance: "{{ ansible_ec2_instance_id }}"
+    volume_size: "{{ atl_mesh_volume_size }}"
+    volume_type: gp3
+    throughput: 250
+    iops: 3072
+    device_name: /dev/xvdd
+    tags:
+      service_name: "{{ stack_name }}"
+      Name: "{{ stack_name }} mesh volume {{ ansible_ec2_placement_availability_zone }}"
+      volume_type: "mesh"
+  register: mesh_vol_info
+  when: (mesh_snapshot is not defined) or (mesh_snapshot|length == 0)
+
+- name: set mesh_vol from info of created
+  ansible.builtin.set_fact:
+    mesh_vol: "{{ mesh_vol_info.volume_id }}"
+  when: (mesh_snapshot is not defined) or (mesh_snapshot|length == 0)
+
+# we should always have mesh_vol by now, if not, Barf !
+- name: we should always have mesh_vol by now
+  ansible.builtin.debug:
+    var: mesh_vol

roles/bitbucket_mesh_config/tasks/meshvols_setup.yml

@@ -0,0 +1,69 @@
+---
+# rerun the mesh volume stat now that we _really_ should have the volume
+- name: discover mesh volume device stat
+  ansible.builtin.stat:
+    path: '/dev/xvdd'
+    follow: no
+    get_attributes: yes
+  register: mesh_device_stat
+  retries: 10
+  delay: 10
+  until: mesh_device_stat.stat['lnk_source'] is defined
+
+- name: describe mesh volume true device
+  ansible.builtin.set_fact:
+    mesh_device: "{{ mesh_device_stat.stat['lnk_source'] }}"
+
+- name: check mesh, caches and logs filetypes
+  community.general.parted:
+    device: "{{ item }}"
+  with_items:
+    - "/dev/nvme1n1"
+    - "/dev/nvme2n1"
+    - "{{ mesh_device }}"
+  register: filesystem_types
+
+- name: set filesystem type fact for mesh volume
+  ansible.builtin.set_fact:
+    mesh_filesystem: "{{ filesystem_types.results | community.general.json_query(query) }}"
+  vars:
+    query: "[?item=='{{ mesh_device }}'].partitions[0].fstype"
+
+- name: ensure mesh volume has a filesystem
+  community.general.filesystem:
+    dev: "{{ mesh_device }}"
+    fstype: "{{ fstype }}"
+  vars:
+    fstype: "{{ (mesh_filesystem|length > 0) | ternary(mesh_filesystem[0], 'xfs') }}"
+
+- name: ensure caches and logs volumes have filesystem
+  community.general.filesystem:
+    dev: "{{ item }}"
+    fstype: "xfs"
+  with_items:
+    - /dev/nvme1n1
+    - /dev/nvme2n1
+
+- name: mount mesh vol to atl_product_home
+  ansible.posix.mount:
+    path: "{{ atl_product_home }}"
+    src: '/dev/xvdd'
+    fstype: "{{ fstype }}"
+    state: mounted
+  register: xfs_mesh_mount
+  vars:
+    fstype: "{{ (mesh_filesystem|length > 0) | ternary(mesh_filesystem[0], 'xfs') }}"
+
+- name: mount cache to atl_product_home/caches
+  ansible.posix.mount:
+    path: "{{ atl_product_home }}/caches"
+    src: '/dev/xvdb'
+    fstype: xfs
+    state: mounted
+
+- name: mount log to atl_product_home/log
+  ansible.posix.mount:
+    path: "{{ atl_product_home }}/log"
+    src: '/dev/xvdc'
+    fstype: xfs
+    state: mounted

roles/bitbucket_mesh_config/tasks/orphan_lease_lock.yml

@@ -0,0 +1,20 @@
+---
+- name: try to create lease_lock directory in efs for this test_orphan_vol
+  ansible.builtin.file:
+    path: '{{ atl_shared_mountpoint }}/.{{ test_orphan_vol }}'
+    state: directory
+    mode: '0755'
+  register: orphan_vol_lease_lock
+  ignore_errors: yes
+  when: orphan_vol is not defined
+
+- name: Set orphan_vol fact on successful lock test
+  ansible.builtin.set_fact:
+    orphan_vol: '{{ test_orphan_vol }}'
+  changed_when: true
+  notify: Remove orphan_vol lease_lock
+  when:
+    - orphan_vol is not defined
+    - orphan_vol_lease_lock is defined
+    - orphan_vol_lease_lock.changed
+  tags: molecule-idempotence-notest