blsmith/dc-deployments-automation
1
0
Fork 0
mirror of https://bitbucket.org/atlassian/dc-deployments-automation.git synced 2025-12-13 16:33:08 -06:00
Code Issues Packages Projects Releases Wiki Activity

DCD-1205: Simplify the pipelines generator.

Browse Source
This commit is contained in:
Steve
2021-01-20 12:55:50 +11:00
parent ad661add8f
commit c250da08ed
3 changed files with 140 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

232
bitbucket-pipelines.yml
View File

@@ -19,8 +19,9 @@ pipelines:
name: Pre Parallelization stage name: Pre Parallelization stage
script: script:
- echo "Running tests in 38 batches" - echo "Running tests in 38 batches"
- step: - step:
name: Check if number of batches match actual number of scenarios name: Check if the template is up-to-date
script: script:
- | - |
actual_scenario_count=$(find ./roles -type f -name "molecule.yml" -exec dirname {} ';' | wc -l | sed -e 's/^[[:space:]]*//') actual_scenario_count=$(find ./roles -type f -name "molecule.yml" -exec dirname {} ';' | wc -l | sed -e 's/^[[:space:]]*//')
@@ -30,309 +31,310 @@ pipelines:
echo "Mismatch between expected and actual number [${actual_scenario_count}] of scenarios. Please look at https://bitbucket.org/atlassian/dc-deployments-automation/src/master/DEVELOPMENT.md for instructions on how to fix this error." echo "Mismatch between expected and actual number [${actual_scenario_count}] of scenarios. Please look at https://bitbucket.org/atlassian/dc-deployments-automation/src/master/DEVELOPMENT.md for instructions on how to fix this error."
exit $GREP_RETURN_CODE exit $GREP_RETURN_CODE
fi fi
- parallel: - parallel:
- step: - step:
name: aws_common/cw-disabled name: aws_common/cw-disabled
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 1 - cd roles/aws_common
- molecule test -s cw-disabled
- step: - step:
name: aws_common/default name: aws_common/default
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 2 - cd roles/aws_common
- molecule test -s default
- step: - step:
name: aws_common/logs-disabled name: aws_common/logs-disabled
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 3 - cd roles/aws_common
- molecule test -s logs-disabled
- step: - step:
name: bitbucket_config/default name: bitbucket_config/default
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 4 - cd roles/bitbucket_config
- molecule test -s default
- step: - step:
name: confluence_config/aurora name: confluence_config/aurora
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 5 - cd roles/confluence_config
- molecule test -s aurora
- step: - step:
name: confluence_config/default name: confluence_config/default
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 6 - cd roles/confluence_config
- molecule test -s default
- step: - step:
name: confluence_config/password_char_escaping name: confluence_config/password_char_escaping
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 7 - cd roles/confluence_config
- molecule test -s password_char_escaping
- step: - step:
name: confluence_config/system_jdk name: confluence_config/system_jdk
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 8 - cd roles/confluence_config
- molecule test -s system_jdk
- step: - step:
name: create_backups/default name: create_backups/default
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 9 - cd roles/create_backups
- molecule test -s default
- step: - step:
name: diy_backup/default name: diy_backup/default
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 10 - cd roles/diy_backup
- molecule test -s default
- step: - step:
name: jira_config/aurora name: jira_config/aurora
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 11 - cd roles/jira_config
- molecule test -s aurora
- step: - step:
name: jira_config/default name: jira_config/default
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 12 - cd roles/jira_config
- molecule test -s default
- step: - step:
name: jira_config/jira_config_props name: jira_config/jira_config_props
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 13 - cd roles/jira_config
- molecule test -s jira_config_props
- step: - step:
name: jira_config/password_char_escaping name: jira_config/password_char_escaping
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 14 - cd roles/jira_config
- molecule test -s password_char_escaping
- step: - step:
name: linux_common/default name: linux_common/default
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 15 - cd roles/linux_common
- molecule test -s default
- step: - step:
name: product_common/default name: product_common/default
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 16 - cd roles/product_common
- molecule test -s default
- step: - step:
name: product_common/system_jdk name: product_common/system_jdk
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 17 - cd roles/product_common
- molecule test -s system_jdk
- step: - step:
name: product_install/bitbucket_latest name: product_install/bitbucket_latest
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 18 - cd roles/product_install
- molecule test -s bitbucket_latest
- step: - step:
name: product_install/confluence_latest name: product_install/confluence_latest
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 19 - cd roles/product_install
- molecule test -s confluence_latest
- step: - step:
name: product_install/crowd_latest name: product_install/crowd_latest
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 20 - cd roles/product_install
- molecule test -s crowd_latest
- step: - step:
name: product_install/default name: product_install/default
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 21 - cd roles/product_install
- molecule test -s default
- step: - step:
name: product_install/jira_all name: product_install/jira_all
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 22 - cd roles/product_install
- molecule test -s jira_all
- step: - step:
name: product_install/jira_cached_with_downgrade name: product_install/jira_cached_with_downgrade
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 23 - cd roles/product_install
- molecule test -s jira_cached_with_downgrade
- step: - step:
name: product_install/jira_cached_with_upgrade name: product_install/jira_cached_with_upgrade
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 24 - cd roles/product_install
- molecule test -s jira_cached_with_upgrade
- step: - step:
name: product_install/jira_software_latest name: product_install/jira_software_latest
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 25 - cd roles/product_install
- molecule test -s jira_software_latest
- step: - step:
name: product_install/jira_tarball name: product_install/jira_tarball
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 26 - cd roles/product_install
- molecule test -s jira_tarball
- step: - step:
name: product_install/jira_version_from_file name: product_install/jira_version_from_file
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 27 - cd roles/product_install
- molecule test -s jira_version_from_file
- step: - step:
name: product_install/jira_version_latest name: product_install/jira_version_latest
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 28 - cd roles/product_install
- molecule test -s jira_version_latest
- step: - step:
name: product_install/jira_version_override name: product_install/jira_version_override
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 29 - cd roles/product_install
- molecule test -s jira_version_override
- step: - step:
name: product_install/servicedesk3 name: product_install/servicedesk3
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 30 - cd roles/product_install
- molecule test -s servicedesk3
- step: - step:
name: product_install/servicedesk4 name: product_install/servicedesk4
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 31 - cd roles/product_install
- molecule test -s servicedesk4
- step: - step:
name: product_install/servicedesk_latest name: product_install/servicedesk_latest
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 32 - cd roles/product_install
- molecule test -s servicedesk_latest
- step: - step:
name: product_startup/bitbucket name: product_startup/bitbucket
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 33 - cd roles/product_startup
- molecule test -s bitbucket
- step: - step:
name: product_startup/default name: product_startup/default
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 34 - cd roles/product_startup
- molecule test -s default
- step: - step:
name: product_startup/synchrony name: product_startup/synchrony
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 35 - cd roles/product_startup
- molecule test -s synchrony
- step: - step:
name: restore_backups/default name: restore_backups/default
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 36 - cd roles/restore_backups
- molecule test -s default
- step: - step:
name: restore_backups/restore_conf_server name: restore_backups/restore_conf_server
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 37 - cd roles/restore_backups
- molecule test -s restore_conf_server
- step: - step:
name: restore_backups/restore_jira_clustered name: restore_backups/restore_jira_clustered
services: services:
- docker - docker
script: script:
- apt-get update && ./bin/install-ansible --dev - ./bin/install-ansible --dev
- ./bin/run-tests-in-batches --batch 38 - cd roles/restore_backups
- molecule test -s restore_jira_clustered

60
pipeline_generator/pipeline.py
View File

@@ -6,57 +6,25 @@ PIPELINE_TEMPLATE_J2_FILE = 'templates/bitbucket-pipelines.yml.j2'
ROLES_DIR = 'roles/' ROLES_DIR = 'roles/'
class Pipeline: def find_all_scenarios():
def generate_pipeline(self): scenario_dirs = []
template_string = self._load_template_as_string() for root, dirs, files in os.walk(Path(os.path.join(os.path.dirname(__file__), "..", ROLES_DIR))):
template = Template(template_string) [scenario_dirs.append(Path(root)) for f in files if f.endswith("molecule.yml")]
steps = self._build_steps() return scenario_dirs
generated_output = template.render(parallel_steps=steps)
print(generated_output)
def _build_steps(self):
return [Step(scenario_name, self._build_script_commands(index))
for index, (scenario_rel_path, scenario_name) in
enumerate(self._find_all_scenarios(), 1)]
@staticmethod
def _build_script_commands(index):
return ScriptCommand(f"./bin/run-tests-in-batches --batch {index}").all_commands()
@staticmethod
def _find_all_scenarios():
scenario_dirs = []
for root, dirs, files in os.walk(Path(os.path.join(os.path.dirname(__file__), "..", ROLES_DIR))):
[scenario_dirs.append(root) for f in files if f.endswith("molecule.yml")]
scenarios = map(lambda d: (d, f"{Path(d).parts[2]}/{Path(d).parts[4]}"), scenario_dirs)
return scenarios
@staticmethod
def _load_template_as_string():
path = Path(os.path.join(os.path.dirname(__file__), PIPELINE_TEMPLATE_J2_FILE))
return path.read_text()
class Step: def load_template():
def __init__(self, name, script_commands=None): path = Path(os.path.join(os.path.dirname(__file__), PIPELINE_TEMPLATE_J2_FILE))
if script_commands is None: return Template(path.read_text())
script_commands = []
self.name = name
self.scriptCommands = script_commands
class ScriptCommand:
PACKAGE_INSTALL_COMMAND = "apt-get update && ./bin/install-ansible --dev"
def __init__(self, test_command):
self.test_command = test_command
def all_commands(self):
return [self.PACKAGE_INSTALL_COMMAND, self.test_command]
def main(): def main():
Pipeline().generate_pipeline() template = load_template()
scenario_paths = find_all_scenarios()
generated_output = template.render(scenario_paths=scenario_paths)
print(generated_output)
if __name__ == '__main__': if __name__ == '__main__':

16
pipeline_generator/templates/bitbucket-pipelines.yml.j2
View File

@@ -18,9 +18,10 @@ pipelines:
- step: - step:
name: Pre Parallelization stage name: Pre Parallelization stage
script: script:
- echo "Running tests in {{ parallel_steps|length }} batches" - echo "Running tests in {{ scenario_paths|length }} batches"
- step: - step:
name: Check if number of batches match actual number of scenarios name: Check if the template is up-to-date
script: script:
- | - |
actual_scenario_count=$(find ./roles -type f -name "molecule.yml" -exec dirname {} ';' | wc -l | sed -e 's/^[[:space:]]*//') actual_scenario_count=$(find ./roles -type f -name "molecule.yml" -exec dirname {} ';' | wc -l | sed -e 's/^[[:space:]]*//')
@@ -30,14 +31,15 @@ pipelines:
echo "Mismatch between expected and actual number [${actual_scenario_count}] of scenarios. Please look at https://bitbucket.org/atlassian/dc-deployments-automation/src/master/DEVELOPMENT.md for instructions on how to fix this error." echo "Mismatch between expected and actual number [${actual_scenario_count}] of scenarios. Please look at https://bitbucket.org/atlassian/dc-deployments-automation/src/master/DEVELOPMENT.md for instructions on how to fix this error."
exit $GREP_RETURN_CODE exit $GREP_RETURN_CODE
fi fi
- parallel: - parallel:
{% for parallel_step in parallel_steps -%} {% for spath in scenario_paths -%}
- step: - step:
name: {{ parallel_step.name }} name: {{ spath.parts[2] }}/{{ spath.parts[4] }}
services: services:
- docker - docker
script: script:
{% for scriptCommand in parallel_step.scriptCommands -%} - ./bin/install-ansible --dev
- {{ scriptCommand }} - cd roles/{{ spath.parts[2] }}
{% endfor %} - molecule test -s {{ spath.parts[4] }}
{% endfor %} {% endfor %}