From c94228e82f507f6abca0513e7a82ea964be936d4 Mon Sep 17 00:00:00 2001 From: Lee Goolsbee Date: Fri, 26 Sep 2025 10:10:58 -0500 Subject: [PATCH 1/4] ITPLT-4544 remove snyk scanner from pipelines --- .snyk | 13 ------ bitbucket-pipelines.yml | 41 +++++++------------ .../templates/bitbucket-pipelines.yml.j2 | 16 -------- 3 files changed, 15 insertions(+), 55 deletions(-) delete mode 100644 .snyk diff --git a/.snyk b/.snyk deleted file mode 100644 index cfa80cb..0000000 --- a/.snyk +++ /dev/null @@ -1,13 +0,0 @@ -# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities. -version: v1.14.1 -# ignores vulnerabilities until expiry date; change duration by modifying expiry date -ignore: - 'snyk:lic:pip:ansible:GPL-3.0': - - '*': - reason: 'Not a shipped or linked dependency, only retrieved at run-time.' - expires: 2022-03-01T00:00:00.000Z - 'snyk:lic:pip:ansible-base:GPL-3.0': - - '*': - reason: 'Not a shipped or linked dependency, only retrieved at run-time.' - expires: 2022-03-01T00:00:00.000Z -patch: {} diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index d0744cb..b678300 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -11,26 +11,16 @@ options: definitions: caches: ansible-collections: ansible_collections - pre-commit: ~/.cache/pre-commit services: docker: memory: 4096 pipelines: default: - - step: - name: Pre-commit - image: python:3.9 - caches: - - pre-commit - script: - - apt update && apt install -y pipenv - - pipenv sync --dev - - pipenv run pre-commit run --all-files - step: name: Pre Parallelization stage script: - - echo "Running tests in 38 batches" + - echo "Running tests in 39 batches" - step: name: Check if the template is up-to-date @@ -115,6 +105,20 @@ pipelines: - ./bin/install-ansible --dev - cd roles/bitbucket_config - pipenv run molecule test -s iam_elasticsearch + - step: + name: bitbucket_mesh_config/default + caches: + - ansible-collections + - docker + - pip + services: + - docker + script: + - apt-get update && apt-get install -y rsync + - export ANSIBLE_CONFIG=./ansible.cfg + - ./bin/install-ansible --dev + - cd roles/bitbucket_mesh_config + - pipenv run molecule test -s default - step: name: confluence_config/aurora caches: @@ -578,18 +582,3 @@ pipelines: - cd roles/product_startup - pipenv run molecule test -s synchrony - - step: - name: Run Snyk security scan - caches: - - docker - - pip - - node - services: - - docker - script: - - export ANSIBLE_CONFIG=./ansible.cfg - - ./bin/install-ansible --dev - - apt-get update && apt-get install -y npm - - npm install -g snyk - - snyk auth $SNYK_TOKEN - - pipenv run snyk monitor --severity-threshold=high --project-name=dc-deployments-automation diff --git a/pipeline_generator/templates/bitbucket-pipelines.yml.j2 b/pipeline_generator/templates/bitbucket-pipelines.yml.j2 index 6b0135a..46f6e9c 100644 --- a/pipeline_generator/templates/bitbucket-pipelines.yml.j2 +++ b/pipeline_generator/templates/bitbucket-pipelines.yml.j2 @@ -51,19 +51,3 @@ pipelines: - cd roles/{{ spath.parts[2] }} - pipenv run molecule test -s {{ spath.parts[4] }} {% endfor %} - - - step: - name: Run Snyk security scan - caches: - - docker - - pip - - node - services: - - docker - script: - - export ANSIBLE_CONFIG=./ansible.cfg - - ./bin/install-ansible --dev - - apt-get update && apt-get install -y npm - - npm install -g snyk - - snyk auth $SNYK_TOKEN - - pipenv run snyk monitor --severity-threshold=high --project-name=dc-deployments-automation From dda6d7c93d8c47621cea5e35bf8d439e263119e5 Mon Sep 17 00:00:00 2001 From: Lee Goolsbee Date: Fri, 26 Sep 2025 10:19:35 -0500 Subject: [PATCH 2/4] ITPLT-4544 add back missing (un-templated) pre-commit cache and step --- bitbucket-pipelines.yml | 10 ++++++++++ .../templates/bitbucket-pipelines.yml.j2 | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index b678300..92dff08 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -11,12 +11,22 @@ options: definitions: caches: ansible-collections: ansible_collections + pre-commit: ~/.cache/pre-commit services: docker: memory: 4096 pipelines: default: + - step: + name: Pre-commit + image: python:3.9 + caches: + - pre-commit + script: + - apt update && apt install -y pipenv + - pipenv sync --dev + - pipenv run pre-commit run --all-files - step: name: Pre Parallelization stage script: diff --git a/pipeline_generator/templates/bitbucket-pipelines.yml.j2 b/pipeline_generator/templates/bitbucket-pipelines.yml.j2 index 46f6e9c..c9c3208 100644 --- a/pipeline_generator/templates/bitbucket-pipelines.yml.j2 +++ b/pipeline_generator/templates/bitbucket-pipelines.yml.j2 @@ -11,12 +11,22 @@ options: definitions: caches: ansible-collections: ansible_collections + pre-commit: ~/.cache/pre-commit services: docker: memory: 4096 pipelines: default: + - step: + name: Pre-commit + image: python:3.9 + caches: + - pre-commit + script: + - apt update && apt install -y pipenv + - pipenv sync --dev + - pipenv run pre-commit run --all-files - step: name: Pre Parallelization stage script: From 52c84f656eabca56b2f05dc51da1ba098dee83f6 Mon Sep 17 00:00:00 2001 From: Lee Goolsbee Date: Fri, 26 Sep 2025 10:30:51 -0500 Subject: [PATCH 3/4] ITPLT-4544 fix template check logic --- bitbucket-pipelines.yml | 12 +++++------- .../templates/bitbucket-pipelines.yml.j2 | 12 +++++------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 92dff08..2b46e2a 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -36,13 +36,11 @@ pipelines: name: Check if the template is up-to-date script: - | - actual_scenario_count=$(find ./roles -type f -name "molecule.yml" -exec dirname {} ';' | wc -l | sed -e 's/^[[:space:]]*//') - grep "Running tests in ${actual_scenario_count} batches" bitbucket-pipelines.yml - GREP_RETURN_CODE=$? - if [[ $GREP_RETURN_CODE -ne 0 ]]; then - echo "Mismatch between expected and actual number [${actual_scenario_count}] of scenarios. Please look at https://bitbucket.org/atlassian/dc-deployments-automation/src/master/DEVELOPMENT.md for instructions on how to fix this error." - exit $GREP_RETURN_CODE - fi + actual_scenario_count=$(find ./roles -type f -name "molecule.yml" -exec dirname {} ';' | wc -l | sed -e 's/^[[:space:]]*//') + if ! grep -q "Running tests in ${actual_scenario_count} batches" bitbucket-pipelines.yml; then + echo "Mismatch between expected (39) and actual number (${actual_scenario_count}) of scenarios. Please look at https://bitbucket.org/atlassian/dc-deployments-automation/src/master/DEVELOPMENT.md for instructions on how to fix this error." + exit 1 + fi - parallel: - step: diff --git a/pipeline_generator/templates/bitbucket-pipelines.yml.j2 b/pipeline_generator/templates/bitbucket-pipelines.yml.j2 index c9c3208..7839a1f 100644 --- a/pipeline_generator/templates/bitbucket-pipelines.yml.j2 +++ b/pipeline_generator/templates/bitbucket-pipelines.yml.j2 @@ -36,13 +36,11 @@ pipelines: name: Check if the template is up-to-date script: - | - actual_scenario_count=$(find ./roles -type f -name "molecule.yml" -exec dirname {} ';' | wc -l | sed -e 's/^[[:space:]]*//') - grep "Running tests in ${actual_scenario_count} batches" bitbucket-pipelines.yml - GREP_RETURN_CODE=$? - if [[ $GREP_RETURN_CODE -ne 0 ]]; then - echo "Mismatch between expected and actual number [${actual_scenario_count}] of scenarios. Please look at https://bitbucket.org/atlassian/dc-deployments-automation/src/master/DEVELOPMENT.md for instructions on how to fix this error." - exit $GREP_RETURN_CODE - fi + actual_scenario_count=$(find ./roles -type f -name "molecule.yml" -exec dirname {} ';' | wc -l | sed -e 's/^[[:space:]]*//') + if ! grep -q "Running tests in ${actual_scenario_count} batches" bitbucket-pipelines.yml; then + echo "Mismatch between expected ({{ scenario_paths|length }}) and actual number (${actual_scenario_count}) of scenarios. Please look at https://bitbucket.org/atlassian/dc-deployments-automation/src/master/DEVELOPMENT.md for instructions on how to fix this error." + exit 1 + fi - parallel: {% for spath in scenario_paths %} From c61b69af4556e62c748d76398f449789d056522e Mon Sep 17 00:00:00 2001 From: Lee Goolsbee Date: Fri, 26 Sep 2025 11:03:08 -0500 Subject: [PATCH 4/4] ITPLT-4544 remove WIP role from tests --- bitbucket-pipelines.yml | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 2b46e2a..e258e0e 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -30,7 +30,7 @@ pipelines: - step: name: Pre Parallelization stage script: - - echo "Running tests in 39 batches" + - echo "Running tests in 38 batches" - step: name: Check if the template is up-to-date @@ -38,7 +38,7 @@ pipelines: - | actual_scenario_count=$(find ./roles -type f -name "molecule.yml" -exec dirname {} ';' | wc -l | sed -e 's/^[[:space:]]*//') if ! grep -q "Running tests in ${actual_scenario_count} batches" bitbucket-pipelines.yml; then - echo "Mismatch between expected (39) and actual number (${actual_scenario_count}) of scenarios. Please look at https://bitbucket.org/atlassian/dc-deployments-automation/src/master/DEVELOPMENT.md for instructions on how to fix this error." + echo "Mismatch between expected (38) and actual number (${actual_scenario_count}) of scenarios. Please look at https://bitbucket.org/atlassian/dc-deployments-automation/src/master/DEVELOPMENT.md for instructions on how to fix this error." exit 1 fi @@ -113,20 +113,6 @@ pipelines: - ./bin/install-ansible --dev - cd roles/bitbucket_config - pipenv run molecule test -s iam_elasticsearch - - step: - name: bitbucket_mesh_config/default - caches: - - ansible-collections - - docker - - pip - services: - - docker - script: - - apt-get update && apt-get install -y rsync - - export ANSIBLE_CONFIG=./ansible.cfg - - ./bin/install-ansible --dev - - cd roles/bitbucket_mesh_config - - pipenv run molecule test -s default - step: name: confluence_config/aurora caches: