Merge remote-tracking branch 'origin/master' into ITPLT-3960-dcd-ansible---cache-jdk-in-sh

.ansible-lint

@@ -8,4 +8,5 @@ skip_list:
   - name[casing]  # so many of our task names aren't capitalized
   - name[missing]  # some tasks have no name
   - risky-file-permissions
+  - var-naming[no-role-prefix]  # many, many warnings
   - yaml  # many, many warnings

.envrc

@@ -0,0 +1,3 @@
+layout pipenv
+pre-commit install
+ansible-galaxy install -r requirements.yml

.gitignore

@@ -5,7 +5,8 @@ __pycache__
 *.pyc
 /.venv
 \#*\#
-.envrc
+.direnv
 .idea
 .vscode
-ansible_collections
+ansible_collections
+.DS_Store

.pre-commit-config.yaml

@@ -19,9 +19,10 @@ repos:
           - --template={}
           - --pattern=[a-zA-Z0-9]{1,10}-[0-9]{1,6}
   - repo: https://github.com/ansible/ansible-lint.git
-    rev: v24.2.2
+    rev: v24.7.0
     hooks:
       - id: ansible-lint
+        stages: [manual]
         additional_dependencies:
           - .
           - ansible-core==2.16.6

Pipfile

@@ -19,6 +19,7 @@ ansible-compat = "==4.1.11"
 docker = "==7.0.0"
 pytest = "*"
 pytest-testinfra = "*"
+pre-commit = "*"
 
 [requires]
 python_version = "3"

README.md

@@ -31,7 +31,7 @@ helper-script `bin/ansible-with-atl-env` and the corresponding
 ## Customizing your deployment
 
 To customise playbook behaviour, you can fork this repository and edit it as
-needed. However, for one-off tasks you can also override the default and 
+needed. However, for one-off tasks you can also override the default and
 calculated settings with special values. To do this, provide command-line overrides to
 [ansible-playbook](https://docs.ansible.com/ansible/latest/cli/ansible-playbook.html).
 
@@ -77,15 +77,15 @@ to "ansible-core-2.13" and/or manually set the stack's "pinned-ansible-sha" SSM
 
 ### Other customizable parameters
 
-For more deployment customization options, consult the following files for parameters you can 
+For more deployment customization options, consult the following files for parameters you can
 override:
 
 - [`/roles/product_install/defaults/main.yml`](roles/product_install/defaults/main.yml)
 - [`/group_vars/aws_node_local.yml`](group_vars/aws_node_local.yml)
 
-More customizable parameters are defined in specific roles -- specifically, in the 
+More customizable parameters are defined in specific roles -- specifically, in the
 role's `defaults/main.yml` file. Most of these parameters use the `atl_` prefix. You can
-use the following [Bitbucket code search query](https://confluence.atlassian.com/bitbucket/search-873876782.html) 
+use the following [Bitbucket code search query](https://confluence.atlassian.com/bitbucket/search-873876782.html)
 to find them:
 
     repo:dc-deployments-automation repo:dc-deployments-automation path:*/defaults/main.yml atl
@@ -108,7 +108,7 @@ environment and running tests.
 This suite is intended to consist of many small, composable roles that can
 be combined together into playbooks. Wherever possible, roles should be product-agnostic
 (e.g. downloads) and platform-agnostic. Functions that are product-specific or
-platform-specific are split off into separate roles. 
+platform-specific are split off into separate roles.
 
 Roles should be reasonably self-contained, with sensible defaults configured in
 `/roles/<role>/defaults/main.yml`. Like all playbook parameters, you can override
@@ -127,7 +127,7 @@ should be run first.
   more information.
 * Inventory files are under `inv/`. For AWS `cfn-init` the inventory
   `inv/aws_node_local` inventory is probably what you want.
-    * Note that this expects the environment to be setup with infrastructure information. 
+    * Note that this expects the environment to be setup with infrastructure information.
       Refer to the [Usage](#markdown-header-usage) section for more information.
 * Global group vars loaded automatically from `group_vars/<group>.yml`. In
   particular note `group_vars/aws_node_local.yml` which loads infrastructure

aws_bitbucket_dc_node.yml

@@ -1,5 +1,6 @@
 ---
-- hosts: aws_node_local
+- name: aws_bitbucket_dc_node.yml
+  hosts: aws_node_local
   become: true
 
   vars:

aws_bitbucket_mesh_node.yml

@@ -1,5 +1,6 @@
 ---
-- hosts: aws_node_local
+- name: aws_bitbucket_mesh_node.yml
+  hosts: aws_node_local
   become: true
 
   vars:

aws_bitbucket_nfs_node.yml

@@ -1,5 +1,6 @@
 ---
-- hosts: aws_node_local
+- name: aws_bitbucket_nfs_node.yml
+  hosts: aws_node_local
   become: true
 
   vars:

aws_clone_nfs.yml

@@ -1,5 +1,6 @@
 ---
-- hosts: aws_node_local
+- name: aws_clone_nfs.yml
+  hosts: aws_node_local
   become: true
 
   vars:

aws_confluence_dc_node.yml

@@ -1,5 +1,6 @@
 ---
-- hosts: aws_node_local
+- name: aws_confluence_dc_node.yml
+  hosts: aws_node_local
   become: true
 
   vars:

aws_confluence_synchrony_node.yml

@@ -1,6 +1,7 @@
 ---
 
-- hosts: aws_node_local
+- name: aws_confluence_synchrony_node.yml
+  hosts: aws_node_local
   become: true
 
   vars:
@@ -12,7 +13,7 @@
     atl_use_system_jdk: true
     java_major_version: "11"
     atl_download_format: "tarball"
-    
+
     atl_startup_systemd_params:
       - "EnvironmentFile=/etc/atl"
       - "EnvironmentFile=/etc/atl.synchrony"

aws_crowd_dc_node.yml

@@ -1,5 +1,6 @@
 ---
-- hosts: aws_node_local
+- name: aws_crowd_dc_node.yml
+  hosts: aws_node_local
   become: true
 
   vars:

aws_jira_dc_node.yml

@@ -1,5 +1,6 @@
 ---
-- hosts: aws_node_local
+- name: aws_jira_dc_node.yml
+  hosts: aws_node_local
   become: true
 
   vars:

bitbucket-pipelines.yml

@@ -11,12 +11,22 @@ options:
 definitions:
   caches:
     ansible-collections: ansible_collections
+    pre-commit: ~/.cache/pre-commit
   services:
     docker:
       memory: 4096
 
 pipelines:
   default:
+    - step:
+        name: Pre-commit
+        image: python:3.9
+        caches:
+          - pre-commit
+        script:
+          - apt update && apt install -y pipenv
+          - pipenv sync --dev
+          - pipenv run pre-commit run --all-files
     - step:
         name: Pre Parallelization stage
         script:

roles/bitbucket_dataset_restore/tasks/main.yml

@@ -30,7 +30,7 @@
     body_format: json
     return_content: yes
   register: output
-  until:  output.status == 200 
+  until:  output.status == 200
   retries: 6
   delay: 15
   failed_when: output is defined and output.json is defined and output.json.state != 'INITIALISING'
@@ -50,7 +50,7 @@
   delay: 10
 
 - name: create lock file
-  ansible.builtin.file: 
+  ansible.builtin.file:
     path: "{{ atl_product_home_shared }}/data/migration/import/lock.file"
     state: touch
   when: import_status.json.state == 'COMPLETED'

roles/confluence_config/templates/server.xml.j2

@@ -50,7 +50,7 @@
     <Connector port="{{ atl_apptunnel_port }}"
                connectionTimeout="20000"
                maxThreads="200"
-               minSpareThreads="10" 
+               minSpareThreads="10"
                enableLookups="false"
                acceptCount="10"
                URIEncoding="UTF-8"/>

roles/crowd_config/templates/server.xml.j2

@@ -1,39 +1,39 @@
 <?xml version="1.0" encoding="utf-8"?>
 
-<Server port="{{ atl_tomcat_mgmt_port }}" 
+<Server port="{{ atl_tomcat_mgmt_port }}"
         shutdown="SHUTDOWN">
 
     <Service name="Catalina">
 
-        <Connector acceptCount="{{ atl_tomcat_acceptcount }}" 
+        <Connector acceptCount="{{ atl_tomcat_acceptcount }}"
-                   connectionTimeout="{{ atl_tomcat_connectiontimeout }}" 
+                   connectionTimeout="{{ atl_tomcat_connectiontimeout }}"
-                   disableUploadTimeout="true" 
+                   disableUploadTimeout="true"
-                   enableLookups="{{ atl_tomcat_enablelookups }}" 
+                   enableLookups="{{ atl_tomcat_enablelookups }}"
-                   maxHttpHeaderSize="8192" 
+                   maxHttpHeaderSize="8192"
-                   maxThreads="{{ atl_tomcat_maxthreads }}" 
+                   maxThreads="{{ atl_tomcat_maxthreads }}"
-                   minSpareThreads="{{ atl_tomcat_minsparethreads }}" 
+                   minSpareThreads="{{ atl_tomcat_minsparethreads }}"
-                   port="{{ atl_tomcat_port }}" 
+                   port="{{ atl_tomcat_port }}"
-                   redirectPort="{{ atl_tomcat_redirectport }}" 
+                   redirectPort="{{ atl_tomcat_redirectport }}"
-                   useBodyEncodingForURI="true" 
+                   useBodyEncodingForURI="true"
-                   URIEncoding="UTF-8" 
+                   URIEncoding="UTF-8"
-                   compression="on" 
+                   compression="on"
-                   sendReasonPhrase="true" 
+                   sendReasonPhrase="true"
-                   compressableMimeType="text/html,text/xml,application/xml,text/plain,text/css,application/json,application/javascript,application/x-javascript" 
+                   compressableMimeType="text/html,text/xml,application/xml,text/plain,text/css,application/json,application/javascript,application/x-javascript"
-                   secure="{{ atl_tomcat_secure }}" 
+                   secure="{{ atl_tomcat_secure }}"
-                   scheme="{{ atl_tomcat_scheme }}" 
+                   scheme="{{ atl_tomcat_scheme }}"
-                   proxyName="{{ atl_proxy_name }}" 
+                   proxyName="{{ atl_proxy_name }}"
-                   proxyPort="{{ atl_proxy_port }}" 
+                   proxyPort="{{ atl_proxy_port }}"
                    protocol="{{ atl_tomcat_protocol }}">
         </Connector>
 
-        <Engine defaultHost="localhost" 
+        <Engine defaultHost="localhost"
                 name="Catalina">
-            <Host appBase="webapps" 
+            <Host appBase="webapps"
-                  autoDeploy="true" 
+                  autoDeploy="true"
-                  name="localhost" 
+                  name="localhost"
                   unpackWARs="true">
-              <Context path="{{ atl_tomcat_contextpath }}" 
+              <Context path="{{ atl_tomcat_contextpath }}"
-                       docBase="../../crowd-webapp" 
+                       docBase="../../crowd-webapp"
                        debug="0"
                        useHttpOnly="true">
                        <Manager pathname="">
@@ -45,20 +45,20 @@
         <!-- To connect to an external web server (typically Apache) -->
         <!-- Define an AJP 1.3 Connector on port 8009 -->
         <!--
-            <Connector port="8009" 
+            <Connector port="8009"
-                       enableLookups="{{ atl_tomcat_enablelookups }}" 
+                       enableLookups="{{ atl_tomcat_enablelookups }}"
-                       redirectPort="{{ atl_tomcat_redirectport }}" 
+                       redirectPort="{{ atl_tomcat_redirectport }}"
                        protocol=""AJP/1.3"" />
         -->
-        <Connector port="{{ atl_tomcat_redirectport }}" 
+        <Connector port="{{ atl_tomcat_redirectport }}"
-                   protocol="{{ atl_tomcat_protocol }}" 
+                   protocol="{{ atl_tomcat_protocol }}"
-                   relaxedPathChars="[]|" 
+                   relaxedPathChars="[]|"
-                   relaxedQueryChars='[]|{}^\`"&lt;&gt;' 
+                   relaxedQueryChars='[]|{}^\`"&lt;&gt;'
-                   maxHttpHeaderSize="65536" 
+                   maxHttpHeaderSize="65536"
-                   URIEncoding="UTF-8" 
+                   URIEncoding="UTF-8"
-                   connectionTimeout="{{ atl_tomcat_connectiontimeout }}" 
+                   connectionTimeout="{{ atl_tomcat_connectiontimeout }}"
-                   useBodyEncodingForURI="true" 
+                   useBodyEncodingForURI="true"
-                   compression="on" 
+                   compression="on"
                    compressableMimeType="text/html,text/xml,text/plain,text/css,application/json,application/javascript,application/x-javascript">
         </Connector>
     </Service>

roles/jira_config/templates/server.xml.j2

@@ -50,7 +50,7 @@
     <Connector port="{{ atl_apptunnel_port }}"
                connectionTimeout="20000"
                maxThreads="200"
-               minSpareThreads="10" 
+               minSpareThreads="10"
                enableLookups="false"
                acceptCount="10"
                URIEncoding="UTF-8"

roles/product_install/molecule/jira_all/converge.yml

@@ -7,7 +7,7 @@
     atl_product_user: "jira"
     # Version 8.14.0 is chosen deliberately as it is a version where jira and jira-software returns different buildNumber from MPAC
     # See DCD-1216 for context
-    atl_product_version: "8.14.0" 
+    atl_product_version: "8.14.0"
     atl_install_jsd_as_obr: true
     atl_systemd_service_name: "jira.service"
     atl_jdbc_encoding: 'UNICODE'

roles/product_install/tasks/main.yml

@@ -186,13 +186,13 @@
       failed_when: false
       ignore_errors: yes
       no_log: true
-      when: 
+      when:
         - ansible_ec2_placement_region is defined
         - atl_download_secret_name is defined
       tags:
         - runtime_pkg
 
-    # Fetch binary and copy to temp 
+    # Fetch binary and copy to temp
     # optionally use basic_auth creds from secrets_manager
     - name: Fetch binary
       ansible.builtin.get_url:
@@ -202,6 +202,7 @@
         url_username: "{{ download_atlassian_username | default(omit) }}"
         mode: 0755
         force: false
+        http_agent: "ansible-httpget_{{ ansible_play_name }}"
       register: atl_product_completed
 
     # If product installer was fetched make the lock directory