Merged in DCD-621-parallelize-builds (pull request #20)

DCD-621 parallelize builds

* DCD-621: Parallelizes tests ny invoking the parralelize script

* DCD-621: Removes leading / to start jira command in product_startup role

* DCD-621: Runs a scenario by name instead of using the all parameter

* DCD-621: A better find all scenarios function to ensure we look at relevant molecule scenarios. Better logging too

* DCD-510: Parallelizes tests further, running in 26 batches on 1 each. The pipeline file is now generated using Jinja

* DCD-510: Adds a pre check stage to verify if number of batches match actual number of scenarios. This test will fail (deliberately)

* DCD-510: Adds a pre check stage to verify if number of batches match actual number of scenarios. Fixes 'test' failure

* DCD-590: Adds note to development.md on how to generate a pipeline file. Adds a makefile and updates a few script commands

* DCD-590: Better documentation in development README document. Updates YML with instructions on how to fix issues with the prevalidation stage

Approved-by: Steve Smith <ssmith@atlassian.com>
Approved-by: Ben Partridge <bpartridge@atlassian.com>

pipeline_generator/Makefile

@@ -0,0 +1,2 @@
+generate-pipeline:
+	@python pipeline.py

pipeline_generator/pipeline.py

@@ -0,0 +1,62 @@
+from jinja2 import Template
+from pathlib import Path
+import os
+
+PIPELINE_TEMPLATE_J2_FILE = 'templates/bitbucket-pipelines.yml.j2'
+ROLES_DIR = 'roles/'
+
+
+class Pipeline:
+    def generate_pipeline(self):
+        template_string = self._load_template_as_string()
+        template = Template(template_string)
+        steps = self._build_steps()
+        generated_output = template.render(parallel_steps=steps)
+        print(generated_output)
+
+    def _build_steps(self):
+        return [Step(f"Molecule Test Batch {index}", self._build_script_commands(index)) for index, scenario in
+                enumerate(self._find_all_scenarios(), 1)]
+
+    @staticmethod
+    def _build_script_commands(index):
+        return ScriptCommand(f"./bin/run-tests-in-batches --batch {index}").all_commands()
+
+    @staticmethod
+    def _find_all_scenarios():
+        scenario_dirs = []
+        for root, dirs, files in os.walk(Path(os.path.join(os.path.dirname(__file__), "..", ROLES_DIR))):
+            [scenario_dirs.append(root) for f in files if f.endswith("molecule.yml")]
+        return scenario_dirs
+
+    @staticmethod
+    def _load_template_as_string():
+        path = Path(os.path.join(os.path.dirname(__file__), PIPELINE_TEMPLATE_J2_FILE))
+        return path.read_text()
+
+
+class Step:
+    def __init__(self, name, script_commands=None):
+        if script_commands is None:
+            script_commands = []
+        self.name = name
+        self.scriptCommands = script_commands
+
+
+class ScriptCommand:
+    INSTALL_PACKAGES_COMMAND = "apt-get update && apt-get install -y virtualenv python-dev"
+    INSTALL_ANSIBLE_COMMAND = "./bin/install-ansible"
+
+    def __init__(self, test_command):
+        self.test_command = test_command
+
+    def all_commands(self):
+        return [self.INSTALL_PACKAGES_COMMAND, self.INSTALL_ANSIBLE_COMMAND, self.test_command]
+
+
+def main():
+    Pipeline().generate_pipeline()
+
+
+if __name__ == '__main__':
+    main()

pipeline_generator/templates/bitbucket-pipelines.yml.j2

@@ -0,0 +1,39 @@
+---
+
+image: atlassian/default-image:2
+options:
+  size: 2x
+
+definitions:
+  services:
+    docker:
+      memory: 4096
+
+pipelines:
+  default:
+    - step:
+        name: Pre Parallelization stage
+        script:
+          - echo "Running tests in {{ parallel_steps|length }} batches"
+    - step:
+        name: Check if number of batches match actual number of scenarios
+        script:
+          - |
+              actual_scenario_count=$(find ./roles -type f -name "molecule.yml" -exec dirname {} ';' | wc -l |  sed -e 's/^[[:space:]]*//')
+              grep  "Running tests in ${actual_scenario_count} batches" bitbucket-pipelines.yml
+              GREP_RETURN_CODE=$?
+              if [[ $GREP_RETURN_CODE -ne 0 ]]; then
+                  echo "Mismatch between expected and actual number [${actual_scenario_count}] of scenarios. Please look at https://bitbucket.org/atlassian/dc-deployments-automation/src/master/DEVELOPMENT.md for instructions on how to fix this error."
+                  exit $GREP_RETURN_CODE
+              fi
+    - parallel:
+       {% for parallel_step in parallel_steps -%}
+        - step:
+           name: {{ parallel_step.name }}
+           services:
+             - docker
+           script:
+           {% for scriptCommand in parallel_step.scriptCommands -%}
+              - {{ scriptCommand }}
+           {% endfor %}
+       {% endfor %}