blsmith/dc-deployments-automation
1
0
Fork 0
mirror of https://bitbucket.org/atlassian/dc-deployments-automation.git synced 2025-12-13 16:33:08 -06:00
Code Issues Packages Projects Releases Wiki Activity

DCD-1205: Simplify the pipelines generator.

Browse Source
This commit is contained in:
Steve
2021-01-20 12:55:50 +11:00
parent ad661add8f
commit c250da08ed
3 changed files with 140 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

232
bitbucket-pipelines.yml
View File

@@ -19,8 +19,9 @@ pipelines:
name: Pre Parallelization stage
script:
- echo "Running tests in 38 batches"
- step:
name: Check if number of batches match actual number of scenarios
name: Check if the template is up-to-date
script:
- |
actual_scenario_count=$(find ./roles -type f -name "molecule.yml" -exec dirname {} ';' | wc -l | sed -e 's/^[[:space:]]*//')
@@ -30,309 +31,310 @@ pipelines:
echo "Mismatch between expected and actual number [${actual_scenario_count}] of scenarios. Please look at https://bitbucket.org/atlassian/dc-deployments-automation/src/master/DEVELOPMENT.md for instructions on how to fix this error."
exit $GREP_RETURN_CODE
fi
- parallel:
- step:
name: aws_common/cw-disabled
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 1
- ./bin/install-ansible --dev
- cd roles/aws_common
- molecule test -s cw-disabled
- step:
name: aws_common/default
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 2
- ./bin/install-ansible --dev
- cd roles/aws_common
- molecule test -s default
- step:
name: aws_common/logs-disabled
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 3
- ./bin/install-ansible --dev
- cd roles/aws_common
- molecule test -s logs-disabled
- step:
name: bitbucket_config/default
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 4
- ./bin/install-ansible --dev
- cd roles/bitbucket_config
- molecule test -s default
- step:
name: confluence_config/aurora
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 5
- ./bin/install-ansible --dev
- cd roles/confluence_config
- molecule test -s aurora
- step:
name: confluence_config/default
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 6
- ./bin/install-ansible --dev
- cd roles/confluence_config
- molecule test -s default
- step:
name: confluence_config/password_char_escaping
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 7
- ./bin/install-ansible --dev
- cd roles/confluence_config
- molecule test -s password_char_escaping
- step:
name: confluence_config/system_jdk
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 8
- ./bin/install-ansible --dev
- cd roles/confluence_config
- molecule test -s system_jdk
- step:
name: create_backups/default
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 9
- ./bin/install-ansible --dev
- cd roles/create_backups
- molecule test -s default
- step:
name: diy_backup/default
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 10
- ./bin/install-ansible --dev
- cd roles/diy_backup
- molecule test -s default
- step:
name: jira_config/aurora
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 11
- ./bin/install-ansible --dev
- cd roles/jira_config
- molecule test -s aurora
- step:
name: jira_config/default
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 12
- ./bin/install-ansible --dev
- cd roles/jira_config
- molecule test -s default
- step:
name: jira_config/jira_config_props
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 13
- ./bin/install-ansible --dev
- cd roles/jira_config
- molecule test -s jira_config_props
- step:
name: jira_config/password_char_escaping
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 14
- ./bin/install-ansible --dev
- cd roles/jira_config
- molecule test -s password_char_escaping
- step:
name: linux_common/default
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 15
- ./bin/install-ansible --dev
- cd roles/linux_common
- molecule test -s default
- step:
name: product_common/default
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 16
- ./bin/install-ansible --dev
- cd roles/product_common
- molecule test -s default
- step:
name: product_common/system_jdk
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 17
- ./bin/install-ansible --dev
- cd roles/product_common
- molecule test -s system_jdk
- step:
name: product_install/bitbucket_latest
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 18
- ./bin/install-ansible --dev
- cd roles/product_install
- molecule test -s bitbucket_latest
- step:
name: product_install/confluence_latest
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 19
- ./bin/install-ansible --dev
- cd roles/product_install
- molecule test -s confluence_latest
- step:
name: product_install/crowd_latest
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 20
- ./bin/install-ansible --dev
- cd roles/product_install
- molecule test -s crowd_latest
- step:
name: product_install/default
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 21
- ./bin/install-ansible --dev
- cd roles/product_install
- molecule test -s default
- step:
name: product_install/jira_all
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 22
- ./bin/install-ansible --dev
- cd roles/product_install
- molecule test -s jira_all
- step:
name: product_install/jira_cached_with_downgrade
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 23
- ./bin/install-ansible --dev
- cd roles/product_install
- molecule test -s jira_cached_with_downgrade
- step:
name: product_install/jira_cached_with_upgrade
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 24
- ./bin/install-ansible --dev
- cd roles/product_install
- molecule test -s jira_cached_with_upgrade
- step:
name: product_install/jira_software_latest
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 25
- ./bin/install-ansible --dev
- cd roles/product_install
- molecule test -s jira_software_latest
- step:
name: product_install/jira_tarball
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 26
- ./bin/install-ansible --dev
- cd roles/product_install
- molecule test -s jira_tarball
- step:
name: product_install/jira_version_from_file
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 27
- ./bin/install-ansible --dev
- cd roles/product_install
- molecule test -s jira_version_from_file
- step:
name: product_install/jira_version_latest
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 28
- ./bin/install-ansible --dev
- cd roles/product_install
- molecule test -s jira_version_latest
- step:
name: product_install/jira_version_override
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 29
- ./bin/install-ansible --dev
- cd roles/product_install
- molecule test -s jira_version_override
- step:
name: product_install/servicedesk3
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 30
- ./bin/install-ansible --dev
- cd roles/product_install
- molecule test -s servicedesk3
- step:
name: product_install/servicedesk4
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 31
- ./bin/install-ansible --dev
- cd roles/product_install
- molecule test -s servicedesk4
- step:
name: product_install/servicedesk_latest
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 32
- ./bin/install-ansible --dev
- cd roles/product_install
- molecule test -s servicedesk_latest
- step:
name: product_startup/bitbucket
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 33
- ./bin/install-ansible --dev
- cd roles/product_startup
- molecule test -s bitbucket
- step:
name: product_startup/default
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 34
- ./bin/install-ansible --dev
- cd roles/product_startup
- molecule test -s default
- step:
name: product_startup/synchrony
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 35
- ./bin/install-ansible --dev
- cd roles/product_startup
- molecule test -s synchrony
- step:
name: restore_backups/default
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 36
- ./bin/install-ansible --dev
- cd roles/restore_backups
- molecule test -s default
- step:
name: restore_backups/restore_conf_server
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 37
- ./bin/install-ansible --dev
- cd roles/restore_backups
- molecule test -s restore_conf_server
- step:
name: restore_backups/restore_jira_clustered
services:
- docker
script:
- apt-get update && ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 38
- ./bin/install-ansible --dev
- cd roles/restore_backups
- molecule test -s restore_jira_clustered

60
pipeline_generator/pipeline.py
View File

@@ -6,57 +6,25 @@ PIPELINE_TEMPLATE_J2_FILE = 'templates/bitbucket-pipelines.yml.j2'
ROLES_DIR = 'roles/'
class Pipeline:
def generate_pipeline(self):
template_string = self._load_template_as_string()
template = Template(template_string)
steps = self._build_steps()
generated_output = template.render(parallel_steps=steps)
print(generated_output)
def _build_steps(self):
return [Step(scenario_name, self._build_script_commands(index))
for index, (scenario_rel_path, scenario_name) in
enumerate(self._find_all_scenarios(), 1)]
@staticmethod
def _build_script_commands(index):
return ScriptCommand(f"./bin/run-tests-in-batches --batch {index}").all_commands()
@staticmethod
def _find_all_scenarios():
scenario_dirs = []
for root, dirs, files in os.walk(Path(os.path.join(os.path.dirname(__file__), "..", ROLES_DIR))):
[scenario_dirs.append(root) for f in files if f.endswith("molecule.yml")]
scenarios = map(lambda d: (d, f"{Path(d).parts[2]}/{Path(d).parts[4]}"), scenario_dirs)
return scenarios
@staticmethod
def _load_template_as_string():
path = Path(os.path.join(os.path.dirname(__file__), PIPELINE_TEMPLATE_J2_FILE))
return path.read_text()
def find_all_scenarios():
scenario_dirs = []
for root, dirs, files in os.walk(Path(os.path.join(os.path.dirname(__file__), "..", ROLES_DIR))):
[scenario_dirs.append(Path(root)) for f in files if f.endswith("molecule.yml")]
return scenario_dirs
class Step:
def __init__(self, name, script_commands=None):
if script_commands is None:
script_commands = []
self.name = name
self.scriptCommands = script_commands
class ScriptCommand:
PACKAGE_INSTALL_COMMAND = "apt-get update && ./bin/install-ansible --dev"
def __init__(self, test_command):
self.test_command = test_command
def all_commands(self):
return [self.PACKAGE_INSTALL_COMMAND, self.test_command]
def load_template():
path = Path(os.path.join(os.path.dirname(__file__), PIPELINE_TEMPLATE_J2_FILE))
return Template(path.read_text())
def main():
Pipeline().generate_pipeline()
template = load_template()
scenario_paths = find_all_scenarios()
generated_output = template.render(scenario_paths=scenario_paths)
print(generated_output)
if __name__ == '__main__':

16
pipeline_generator/templates/bitbucket-pipelines.yml.j2
View File

@@ -18,9 +18,10 @@ pipelines:
- step:
name: Pre Parallelization stage
script:
- echo "Running tests in {{ parallel_steps|length }} batches"
- echo "Running tests in {{ scenario_paths|length }} batches"
- step:
name: Check if number of batches match actual number of scenarios
name: Check if the template is up-to-date
script:
- |
actual_scenario_count=$(find ./roles -type f -name "molecule.yml" -exec dirname {} ';' | wc -l | sed -e 's/^[[:space:]]*//')
@@ -30,14 +31,15 @@ pipelines:
echo "Mismatch between expected and actual number [${actual_scenario_count}] of scenarios. Please look at https://bitbucket.org/atlassian/dc-deployments-automation/src/master/DEVELOPMENT.md for instructions on how to fix this error."
exit $GREP_RETURN_CODE
fi
- parallel:
{% for parallel_step in parallel_steps -%}
{% for spath in scenario_paths -%}
- step:
name: {{ parallel_step.name }}
name: {{ spath.parts[2] }}/{{ spath.parts[4] }}
services:
- docker
script:
{% for scriptCommand in parallel_step.scriptCommands -%}
- {{ scriptCommand }}
{% endfor %}
- ./bin/install-ansible --dev
- cd roles/{{ spath.parts[2] }}
- molecule test -s {{ spath.parts[4] }}
{% endfor %}