Merged master into ITSD-487185-opensearch-jira

.snyk

@@ -1,13 +0,0 @@
-# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
-version: v1.14.1
-# ignores vulnerabilities until expiry date; change duration by modifying expiry date
-ignore:
-  'snyk:lic:pip:ansible:GPL-3.0':
-    - '*':
-        reason: 'Not a shipped or linked dependency, only retrieved at run-time.'
-        expires: 2022-03-01T00:00:00.000Z
-  'snyk:lic:pip:ansible-base:GPL-3.0':
-    - '*':
-        reason: 'Not a shipped or linked dependency, only retrieved at run-time.'
-        expires: 2022-03-01T00:00:00.000Z
-patch: {}

bitbucket-pipelines.yml

@@ -37,11 +37,9 @@ pipelines:
         script:
           - |
             actual_scenario_count=$(find ./roles -type f -name "molecule.yml" -exec dirname {} ';' | wc -l | sed -e 's/^[[:space:]]*//')
-              grep  "Running tests in ${actual_scenario_count} batches" bitbucket-pipelines.yml
+            if ! grep -q "Running tests in ${actual_scenario_count} batches" bitbucket-pipelines.yml; then
-              GREP_RETURN_CODE=$?
+              echo "Mismatch between expected (38) and actual number (${actual_scenario_count}) of scenarios. Please look at https://bitbucket.org/atlassian/dc-deployments-automation/src/master/DEVELOPMENT.md for instructions on how to fix this error."
-              if [[ $GREP_RETURN_CODE -ne 0 ]]; then
+              exit 1
-                  echo "Mismatch between expected and actual number [${actual_scenario_count}] of scenarios. Please look at https://bitbucket.org/atlassian/dc-deployments-automation/src/master/DEVELOPMENT.md for instructions on how to fix this error."
-                  exit $GREP_RETURN_CODE
             fi
 
     - parallel:
@@ -578,18 +576,3 @@ pipelines:
               - cd roles/product_startup
               - pipenv run molecule test -s synchrony
 
-        - step:
-            name: Run Snyk security scan
-            caches:
-              - docker
-              - pip
-              - node
-            services:
-              - docker
-            script:
-              - export ANSIBLE_CONFIG=./ansible.cfg
-              - ./bin/install-ansible --dev
-              - apt-get update && apt-get install -y npm
-              - npm install -g snyk
-              - snyk auth $SNYK_TOKEN
-              - pipenv run snyk monitor --severity-threshold=high --project-name=dc-deployments-automation

pipeline_generator/templates/bitbucket-pipelines.yml.j2

@@ -11,12 +11,22 @@ options:
 definitions:
   caches:
     ansible-collections: ansible_collections
+    pre-commit: ~/.cache/pre-commit
   services:
     docker:
       memory: 4096
 
 pipelines:
   default:
+    - step:
+        name: Pre-commit
+        image: python:3.9
+        caches:
+          - pre-commit
+        script:
+          - apt update && apt install -y pipenv
+          - pipenv sync --dev
+          - pipenv run pre-commit run --all-files
     - step:
         name: Pre Parallelization stage
         script:
@@ -27,11 +37,9 @@ pipelines:
         script:
           - |
             actual_scenario_count=$(find ./roles -type f -name "molecule.yml" -exec dirname {} ';' | wc -l | sed -e 's/^[[:space:]]*//')
-              grep  "Running tests in ${actual_scenario_count} batches" bitbucket-pipelines.yml
+            if ! grep -q "Running tests in ${actual_scenario_count} batches" bitbucket-pipelines.yml; then
-              GREP_RETURN_CODE=$?
+              echo "Mismatch between expected ({{ scenario_paths|length }}) and actual number (${actual_scenario_count}) of scenarios. Please look at https://bitbucket.org/atlassian/dc-deployments-automation/src/master/DEVELOPMENT.md for instructions on how to fix this error."
-              if [[ $GREP_RETURN_CODE -ne 0 ]]; then
+              exit 1
-                  echo "Mismatch between expected and actual number [${actual_scenario_count}] of scenarios. Please look at https://bitbucket.org/atlassian/dc-deployments-automation/src/master/DEVELOPMENT.md for instructions on how to fix this error."
-                  exit $GREP_RETURN_CODE
             fi
 
     - parallel:
@@ -51,19 +59,3 @@ pipelines:
               - cd roles/{{ spath.parts[2] }}
               - pipenv run molecule test -s {{ spath.parts[4] }}
        {% endfor %}
-
-        - step:
-            name: Run Snyk security scan
-            caches:
-              - docker
-              - pip
-              - node
-            services:
-              - docker
-            script:
-              - export ANSIBLE_CONFIG=./ansible.cfg
-              - ./bin/install-ansible --dev
-              - apt-get update && apt-get install -y npm
-              - npm install -g snyk
-              - snyk auth $SNYK_TOKEN
-              - pipenv run snyk monitor --severity-threshold=high --project-name=dc-deployments-automation