From d9e02990af3510bab6cbf9db7cc9fcdd22de2776 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 2 Mar 2021 15:05:09 +1100 Subject: [PATCH 1/9] Add running of Snyk againsts runtime dependencies. --- .../templates/bitbucket-pipelines.yml.j2 | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pipeline_generator/templates/bitbucket-pipelines.yml.j2 b/pipeline_generator/templates/bitbucket-pipelines.yml.j2 index e42e10b..8d29339 100644 --- a/pipeline_generator/templates/bitbucket-pipelines.yml.j2 +++ b/pipeline_generator/templates/bitbucket-pipelines.yml.j2 @@ -43,3 +43,14 @@ pipelines: - cd roles/{{ spath.parts[2] }} - pipenv run molecule test -s {{ spath.parts[4] }} {% endfor %} + + - step: + name: Run Snyk security scan + services: + - docker + script: + - ./bin/install-ansible --dev + - apt-get update && apt-get install -y npm + - npm install -g snyk + - snyk auth $SNYK_TOKEN + - pipenv run snyk test --severity-threshold=high From 470df7a1e9da7ff888b6e324c3c1354bd30e67ce Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 2 Mar 2021 15:07:52 +1100 Subject: [PATCH 2/9] Add ignores for GPL-3.0 licensing as it is a runtime tool-chain, not a distributed or linked. --- .snyk | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .snyk diff --git a/.snyk b/.snyk new file mode 100644 index 0000000..cfa80cb --- /dev/null +++ b/.snyk @@ -0,0 +1,13 @@ +# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities. +version: v1.14.1 +# ignores vulnerabilities until expiry date; change duration by modifying expiry date +ignore: + 'snyk:lic:pip:ansible:GPL-3.0': + - '*': + reason: 'Not a shipped or linked dependency, only retrieved at run-time.' + expires: 2022-03-01T00:00:00.000Z + 'snyk:lic:pip:ansible-base:GPL-3.0': + - '*': + reason: 'Not a shipped or linked dependency, only retrieved at run-time.' + expires: 2022-03-01T00:00:00.000Z +patch: {} From 4a499ef0200f76468627013425522fefbde0c43b Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 2 Mar 2021 15:22:20 +1100 Subject: [PATCH 3/9] Make generated pipeline more PR-friendly and regenerate. --- bitbucket-pipelines.yml | 561 +++++++++++++++++---------------- pipeline_generator/pipeline.py | 4 +- 2 files changed, 288 insertions(+), 277 deletions(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 5b98510..a7552d3 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -34,13 +34,29 @@ pipelines: - parallel: - step: - name: bitbucket_config/iam_elasticsearch + name: aws_common/cw-disabled services: - docker script: - ./bin/install-ansible --dev - - cd roles/bitbucket_config - - pipenv run molecule test -s iam_elasticsearch + - cd roles/aws_common + - pipenv run molecule test -s cw-disabled + - step: + name: aws_common/default + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/aws_common + - pipenv run molecule test -s default + - step: + name: aws_common/logs-disabled + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/aws_common + - pipenv run molecule test -s logs-disabled - step: name: bitbucket_config/default services: @@ -49,6 +65,262 @@ pipelines: - ./bin/install-ansible --dev - cd roles/bitbucket_config - pipenv run molecule test -s default + - step: + name: bitbucket_config/iam_elasticsearch + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/bitbucket_config + - pipenv run molecule test -s iam_elasticsearch + - step: + name: confluence_config/aurora + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/confluence_config + - pipenv run molecule test -s aurora + - step: + name: confluence_config/default + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/confluence_config + - pipenv run molecule test -s default + - step: + name: confluence_config/password_char_escaping + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/confluence_config + - pipenv run molecule test -s password_char_escaping + - step: + name: confluence_config/system_jdk + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/confluence_config + - pipenv run molecule test -s system_jdk + - step: + name: diy_backup/default + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/diy_backup + - pipenv run molecule test -s default + - step: + name: jira_config/aurora + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/jira_config + - pipenv run molecule test -s aurora + - step: + name: jira_config/default + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/jira_config + - pipenv run molecule test -s default + - step: + name: jira_config/jira_config_props + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/jira_config + - pipenv run molecule test -s jira_config_props + - step: + name: jira_config/password_char_escaping + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/jira_config + - pipenv run molecule test -s password_char_escaping + - step: + name: linux_common/default + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/linux_common + - pipenv run molecule test -s default + - step: + name: product_common/default + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/product_common + - pipenv run molecule test -s default + - step: + name: product_common/system_jdk + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/product_common + - pipenv run molecule test -s system_jdk + - step: + name: product_install/bitbucket_latest + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/product_install + - pipenv run molecule test -s bitbucket_latest + - step: + name: product_install/confluence_latest + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/product_install + - pipenv run molecule test -s confluence_latest + - step: + name: product_install/crowd_latest + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/product_install + - pipenv run molecule test -s crowd_latest + - step: + name: product_install/default + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/product_install + - pipenv run molecule test -s default + - step: + name: product_install/jira_all + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/product_install + - pipenv run molecule test -s jira_all + - step: + name: product_install/jira_cached_with_downgrade + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/product_install + - pipenv run molecule test -s jira_cached_with_downgrade + - step: + name: product_install/jira_cached_with_upgrade + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/product_install + - pipenv run molecule test -s jira_cached_with_upgrade + - step: + name: product_install/jira_software_latest + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/product_install + - pipenv run molecule test -s jira_software_latest + - step: + name: product_install/jira_tarball + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/product_install + - pipenv run molecule test -s jira_tarball + - step: + name: product_install/jira_version_from_file + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/product_install + - pipenv run molecule test -s jira_version_from_file + - step: + name: product_install/jira_version_latest + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/product_install + - pipenv run molecule test -s jira_version_latest + - step: + name: product_install/jira_version_override + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/product_install + - pipenv run molecule test -s jira_version_override + - step: + name: product_install/servicedesk3 + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/product_install + - pipenv run molecule test -s servicedesk3 + - step: + name: product_install/servicedesk4 + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/product_install + - pipenv run molecule test -s servicedesk4 + - step: + name: product_install/servicedesk_latest + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/product_install + - pipenv run molecule test -s servicedesk_latest + - step: + name: product_startup/bitbucket + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/product_startup + - pipenv run molecule test -s bitbucket + - step: + name: product_startup/default + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/product_startup + - pipenv run molecule test -s default + - step: + name: product_startup/startup_restart_false + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/product_startup + - pipenv run molecule test -s startup_restart_false + - step: + name: product_startup/synchrony + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/product_startup + - pipenv run molecule test -s synchrony - step: name: restore_backups/default services: @@ -73,276 +345,15 @@ pipelines: - ./bin/install-ansible --dev - cd roles/restore_backups - pipenv run molecule test -s restore_jira_clustered - - step: - name: diy_backup/default - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/diy_backup - - pipenv run molecule test -s default - - step: - name: product_startup/synchrony - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/product_startup - - pipenv run molecule test -s synchrony - - step: - name: product_startup/default - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/product_startup - - pipenv run molecule test -s default - - step: - name: product_startup/bitbucket - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/product_startup - - pipenv run molecule test -s bitbucket - - step: - name: product_startup/startup_restart_false - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/product_startup - - pipenv run molecule test -s startup_restart_false - - step: - name: product_common/default - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/product_common - - pipenv run molecule test -s default - - step: - name: product_common/system_jdk - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/product_common - - pipenv run molecule test -s system_jdk - - step: - name: confluence_config/default - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/confluence_config - - pipenv run molecule test -s default - - step: - name: confluence_config/aurora - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/confluence_config - - pipenv run molecule test -s aurora - - step: - name: confluence_config/system_jdk - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/confluence_config - - pipenv run molecule test -s system_jdk - - step: - name: confluence_config/password_char_escaping - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/confluence_config - - pipenv run molecule test -s password_char_escaping - - step: - name: jira_config/default - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/jira_config - - pipenv run molecule test -s default - - step: - name: jira_config/aurora - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/jira_config - - pipenv run molecule test -s aurora - - step: - name: jira_config/jira_config_props - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/jira_config - - pipenv run molecule test -s jira_config_props - - step: - name: jira_config/password_char_escaping - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/jira_config - - pipenv run molecule test -s password_char_escaping - - step: - name: product_install/jira_version_from_file - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/product_install - - pipenv run molecule test -s jira_version_from_file - - step: - name: product_install/jira_cached_with_upgrade - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/product_install - - pipenv run molecule test -s jira_cached_with_upgrade - - step: - name: product_install/servicedesk4 - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/product_install - - pipenv run molecule test -s servicedesk4 - - step: - name: product_install/servicedesk3 - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/product_install - - pipenv run molecule test -s servicedesk3 - - step: - name: product_install/jira_software_latest - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/product_install - - pipenv run molecule test -s jira_software_latest - - step: - name: product_install/default - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/product_install - - pipenv run molecule test -s default - - step: - name: product_install/bitbucket_latest - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/product_install - - pipenv run molecule test -s bitbucket_latest - - step: - name: product_install/jira_version_override - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/product_install - - pipenv run molecule test -s jira_version_override - - step: - name: product_install/crowd_latest - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/product_install - - pipenv run molecule test -s crowd_latest - - step: - name: product_install/servicedesk_latest - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/product_install - - pipenv run molecule test -s servicedesk_latest - - step: - name: product_install/jira_version_latest - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/product_install - - pipenv run molecule test -s jira_version_latest - - step: - name: product_install/confluence_latest - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/product_install - - pipenv run molecule test -s confluence_latest - - step: - name: product_install/jira_cached_with_downgrade - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/product_install - - pipenv run molecule test -s jira_cached_with_downgrade - - step: - name: product_install/jira_tarball - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/product_install - - pipenv run molecule test -s jira_tarball - - step: - name: product_install/jira_all - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/product_install - - pipenv run molecule test -s jira_all - - step: - name: aws_common/logs-disabled - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/aws_common - - pipenv run molecule test -s logs-disabled - - step: - name: aws_common/cw-disabled - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/aws_common - - pipenv run molecule test -s cw-disabled - - step: - name: aws_common/default - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/aws_common - - pipenv run molecule test -s default - - step: - name: linux_common/default - services: - - docker - script: - - ./bin/install-ansible --dev - - cd roles/linux_common - - pipenv run molecule test -s default + + - step: + name: Run Snyk security scan + services: + - docker + script: + - ./bin/install-ansible --dev + - apt-get update && apt-get install -y npm + - npm install -g snyk + - snyk auth $SNYK_TOKEN + - pipenv run snyk test --severity-threshold=high diff --git a/pipeline_generator/pipeline.py b/pipeline_generator/pipeline.py index b46de5c..9f85056 100644 --- a/pipeline_generator/pipeline.py +++ b/pipeline_generator/pipeline.py @@ -8,9 +8,9 @@ ROLES_DIR = 'roles/' def find_all_scenarios(): scenario_dirs = [] - for root, dirs, files in os.walk(Path(os.path.join(os.path.dirname(__file__), "..", ROLES_DIR))): + for root, dirs, files in os.walk('..'): [scenario_dirs.append(Path(root)) for f in files if f.endswith("molecule.yml")] - return scenario_dirs + return sorted(scenario_dirs) def load_template(): From a59b2b2f0f696086ce09ca9cd1d6b673db2b1012 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 2 Mar 2021 15:30:40 +1100 Subject: [PATCH 4/9] Fix formatting. --- bitbucket-pipelines.yml | 89 +++++++++---------- pipeline_generator/pipeline.py | 14 +-- .../templates/bitbucket-pipelines.yml.j2 | 12 +-- 3 files changed, 58 insertions(+), 57 deletions(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index a7552d3..1d83caa 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -33,7 +33,7 @@ pipelines: fi - parallel: - - step: + - step: name: aws_common/cw-disabled services: - docker @@ -41,7 +41,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/aws_common - pipenv run molecule test -s cw-disabled - - step: + - step: name: aws_common/default services: - docker @@ -49,7 +49,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/aws_common - pipenv run molecule test -s default - - step: + - step: name: aws_common/logs-disabled services: - docker @@ -57,7 +57,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/aws_common - pipenv run molecule test -s logs-disabled - - step: + - step: name: bitbucket_config/default services: - docker @@ -65,7 +65,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/bitbucket_config - pipenv run molecule test -s default - - step: + - step: name: bitbucket_config/iam_elasticsearch services: - docker @@ -73,7 +73,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/bitbucket_config - pipenv run molecule test -s iam_elasticsearch - - step: + - step: name: confluence_config/aurora services: - docker @@ -81,7 +81,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/confluence_config - pipenv run molecule test -s aurora - - step: + - step: name: confluence_config/default services: - docker @@ -89,7 +89,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/confluence_config - pipenv run molecule test -s default - - step: + - step: name: confluence_config/password_char_escaping services: - docker @@ -97,7 +97,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/confluence_config - pipenv run molecule test -s password_char_escaping - - step: + - step: name: confluence_config/system_jdk services: - docker @@ -105,7 +105,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/confluence_config - pipenv run molecule test -s system_jdk - - step: + - step: name: diy_backup/default services: - docker @@ -113,7 +113,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/diy_backup - pipenv run molecule test -s default - - step: + - step: name: jira_config/aurora services: - docker @@ -121,7 +121,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/jira_config - pipenv run molecule test -s aurora - - step: + - step: name: jira_config/default services: - docker @@ -129,7 +129,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/jira_config - pipenv run molecule test -s default - - step: + - step: name: jira_config/jira_config_props services: - docker @@ -137,7 +137,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/jira_config - pipenv run molecule test -s jira_config_props - - step: + - step: name: jira_config/password_char_escaping services: - docker @@ -145,7 +145,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/jira_config - pipenv run molecule test -s password_char_escaping - - step: + - step: name: linux_common/default services: - docker @@ -153,7 +153,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/linux_common - pipenv run molecule test -s default - - step: + - step: name: product_common/default services: - docker @@ -161,7 +161,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/product_common - pipenv run molecule test -s default - - step: + - step: name: product_common/system_jdk services: - docker @@ -169,7 +169,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/product_common - pipenv run molecule test -s system_jdk - - step: + - step: name: product_install/bitbucket_latest services: - docker @@ -177,7 +177,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/product_install - pipenv run molecule test -s bitbucket_latest - - step: + - step: name: product_install/confluence_latest services: - docker @@ -185,7 +185,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/product_install - pipenv run molecule test -s confluence_latest - - step: + - step: name: product_install/crowd_latest services: - docker @@ -193,7 +193,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/product_install - pipenv run molecule test -s crowd_latest - - step: + - step: name: product_install/default services: - docker @@ -201,7 +201,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/product_install - pipenv run molecule test -s default - - step: + - step: name: product_install/jira_all services: - docker @@ -209,7 +209,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/product_install - pipenv run molecule test -s jira_all - - step: + - step: name: product_install/jira_cached_with_downgrade services: - docker @@ -217,7 +217,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/product_install - pipenv run molecule test -s jira_cached_with_downgrade - - step: + - step: name: product_install/jira_cached_with_upgrade services: - docker @@ -225,7 +225,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/product_install - pipenv run molecule test -s jira_cached_with_upgrade - - step: + - step: name: product_install/jira_software_latest services: - docker @@ -233,7 +233,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/product_install - pipenv run molecule test -s jira_software_latest - - step: + - step: name: product_install/jira_tarball services: - docker @@ -241,7 +241,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/product_install - pipenv run molecule test -s jira_tarball - - step: + - step: name: product_install/jira_version_from_file services: - docker @@ -249,7 +249,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/product_install - pipenv run molecule test -s jira_version_from_file - - step: + - step: name: product_install/jira_version_latest services: - docker @@ -257,7 +257,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/product_install - pipenv run molecule test -s jira_version_latest - - step: + - step: name: product_install/jira_version_override services: - docker @@ -265,7 +265,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/product_install - pipenv run molecule test -s jira_version_override - - step: + - step: name: product_install/servicedesk3 services: - docker @@ -273,7 +273,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/product_install - pipenv run molecule test -s servicedesk3 - - step: + - step: name: product_install/servicedesk4 services: - docker @@ -281,7 +281,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/product_install - pipenv run molecule test -s servicedesk4 - - step: + - step: name: product_install/servicedesk_latest services: - docker @@ -289,7 +289,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/product_install - pipenv run molecule test -s servicedesk_latest - - step: + - step: name: product_startup/bitbucket services: - docker @@ -297,7 +297,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/product_startup - pipenv run molecule test -s bitbucket - - step: + - step: name: product_startup/default services: - docker @@ -305,7 +305,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/product_startup - pipenv run molecule test -s default - - step: + - step: name: product_startup/startup_restart_false services: - docker @@ -313,7 +313,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/product_startup - pipenv run molecule test -s startup_restart_false - - step: + - step: name: product_startup/synchrony services: - docker @@ -321,7 +321,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/product_startup - pipenv run molecule test -s synchrony - - step: + - step: name: restore_backups/default services: - docker @@ -329,7 +329,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/restore_backups - pipenv run molecule test -s default - - step: + - step: name: restore_backups/restore_conf_server services: - docker @@ -337,7 +337,7 @@ pipelines: - ./bin/install-ansible --dev - cd roles/restore_backups - pipenv run molecule test -s restore_conf_server - - step: + - step: name: restore_backups/restore_jira_clustered services: - docker @@ -345,15 +345,14 @@ pipelines: - ./bin/install-ansible --dev - cd roles/restore_backups - pipenv run molecule test -s restore_jira_clustered - - step: name: Run Snyk security scan services: - docker script: - - ./bin/install-ansible --dev - - apt-get update && apt-get install -y npm - - npm install -g snyk - - snyk auth $SNYK_TOKEN - - pipenv run snyk test --severity-threshold=high + - ./bin/install-ansible --dev + - apt-get update && apt-get install -y npm + - npm install -g snyk + - snyk auth $SNYK_TOKEN + - pipenv run snyk test --severity-threshold=high diff --git a/pipeline_generator/pipeline.py b/pipeline_generator/pipeline.py index 9f85056..c356a56 100644 --- a/pipeline_generator/pipeline.py +++ b/pipeline_generator/pipeline.py @@ -1,4 +1,4 @@ -from jinja2 import Template +import jinja2 as j2 from pathlib import Path import os @@ -14,14 +14,16 @@ def find_all_scenarios(): def load_template(): - path = Path(os.path.join(os.path.dirname(__file__), PIPELINE_TEMPLATE_J2_FILE)) - return Template(path.read_text()) - + jenv = j2.Environment( + loader=j2.FileSystemLoader('.'), + lstrip_blocks=True, + trim_blocks=True) + return jenv.get_template(PIPELINE_TEMPLATE_J2_FILE) def main(): - template = load_template() - scenario_paths = find_all_scenarios() + + template = load_template() generated_output = template.render(scenario_paths=scenario_paths) print(generated_output) diff --git a/pipeline_generator/templates/bitbucket-pipelines.yml.j2 b/pipeline_generator/templates/bitbucket-pipelines.yml.j2 index 8d29339..77a5e9c 100644 --- a/pipeline_generator/templates/bitbucket-pipelines.yml.j2 +++ b/pipeline_generator/templates/bitbucket-pipelines.yml.j2 @@ -33,7 +33,7 @@ pipelines: fi - parallel: - {% for spath in scenario_paths -%} + {% for spath in scenario_paths %} - step: name: {{ spath.parts[2] }}/{{ spath.parts[4] }} services: @@ -49,8 +49,8 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev - - apt-get update && apt-get install -y npm - - npm install -g snyk - - snyk auth $SNYK_TOKEN - - pipenv run snyk test --severity-threshold=high + - ./bin/install-ansible --dev + - apt-get update && apt-get install -y npm + - npm install -g snyk + - snyk auth $SNYK_TOKEN + - pipenv run snyk test --severity-threshold=high From 241ff40ae8c04c3c69626203eace40ee24e4e28f Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 13 Apr 2021 15:02:14 +1000 Subject: [PATCH 5/9] DCD-1273: Also monitor the project. --- bitbucket-pipelines.yml | 7 ++++--- pipeline_generator/templates/bitbucket-pipelines.yml.j2 | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 1d83caa..f9c7f3f 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -353,6 +353,7 @@ pipelines: script: - ./bin/install-ansible --dev - apt-get update && apt-get install -y npm - - npm install -g snyk - - snyk auth $SNYK_TOKEN - - pipenv run snyk test --severity-threshold=high + - npm install snyk + - npx snyk auth $SNYK_TOKEN + - pipenv run npx snyk test --severity-threshold=high + - pipenv run npx snyk monitor --severity-threshold=high --project-name=dc-deployments-automation diff --git a/pipeline_generator/templates/bitbucket-pipelines.yml.j2 b/pipeline_generator/templates/bitbucket-pipelines.yml.j2 index 77a5e9c..7bdfb26 100644 --- a/pipeline_generator/templates/bitbucket-pipelines.yml.j2 +++ b/pipeline_generator/templates/bitbucket-pipelines.yml.j2 @@ -51,6 +51,7 @@ pipelines: script: - ./bin/install-ansible --dev - apt-get update && apt-get install -y npm - - npm install -g snyk - - snyk auth $SNYK_TOKEN - - pipenv run snyk test --severity-threshold=high + - npm install snyk + - npx snyk auth $SNYK_TOKEN + - pipenv run npx snyk test --severity-threshold=high + - pipenv run npx snyk monitor --severity-threshold=high --project-name=dc-deployments-automation From 0f3aba40d6bcb1bb6d92f3a1aaeb14208ff0cfed Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 13 Apr 2021 15:02:33 +1000 Subject: [PATCH 6/9] DCD-1273: Remove renovate. --- renovate.json | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 renovate.json diff --git a/renovate.json b/renovate.json deleted file mode 100644 index 1927fea..0000000 --- a/renovate.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": [ - "config:base" - ] -} \ No newline at end of file From 8b5598ec6f98a4cc7686dc34a8e59f5ec2c667a0 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 13 Apr 2021 15:20:14 +1000 Subject: [PATCH 7/9] DCD-1273: npx doesn't play well with pipenv for some reason. --- bitbucket-pipelines.yml | 8 ++++---- pipeline_generator/templates/bitbucket-pipelines.yml.j2 | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index f9c7f3f..b77cb38 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -353,7 +353,7 @@ pipelines: script: - ./bin/install-ansible --dev - apt-get update && apt-get install -y npm - - npm install snyk - - npx snyk auth $SNYK_TOKEN - - pipenv run npx snyk test --severity-threshold=high - - pipenv run npx snyk monitor --severity-threshold=high --project-name=dc-deployments-automation + - npm install -g snyk + - snyk auth $SNYK_TOKEN + - pipenv run snyk test --severity-threshold=high + - pipenv run snyk monitor --severity-threshold=high --project-name=dc-deployments-automation diff --git a/pipeline_generator/templates/bitbucket-pipelines.yml.j2 b/pipeline_generator/templates/bitbucket-pipelines.yml.j2 index 7bdfb26..2087084 100644 --- a/pipeline_generator/templates/bitbucket-pipelines.yml.j2 +++ b/pipeline_generator/templates/bitbucket-pipelines.yml.j2 @@ -51,7 +51,7 @@ pipelines: script: - ./bin/install-ansible --dev - apt-get update && apt-get install -y npm - - npm install snyk - - npx snyk auth $SNYK_TOKEN - - pipenv run npx snyk test --severity-threshold=high - - pipenv run npx snyk monitor --severity-threshold=high --project-name=dc-deployments-automation + - npm install -g snyk + - snyk auth $SNYK_TOKEN + - pipenv run snyk test --severity-threshold=high + - pipenv run snyk monitor --severity-threshold=high --project-name=dc-deployments-automation From fb369e6d766fa02ca6b2690802d16c0979a22f5d Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 14 Apr 2021 10:50:30 +1000 Subject: [PATCH 8/9] Updates to package lock. --- Pipfile.lock | 68 ++++++++++++++++++++++++++-------------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 8c67829..5d77427 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "da42952f61acb670cc8542ac3a7cf870522cd2d38a8b5493b8872f0542969e52" + "sha256": "4cc5161ca039ac884905a5453c2aee0c4686c0ece78f1edffd4d8ebad812bcce" }, "pipfile-spec": 6, "requires": { @@ -18,10 +18,10 @@ "default": { "ansible": { "hashes": [ - "sha256:9775229aae31336a624ca5afe5533fea5e49ef4daa96a96791dd9871b2d8b8d1" + "sha256:9ff024500116d53c460cb09ea92e3c9404119f100d1d1ff0de69a9dafca561d5" ], "index": "pypi", - "version": "==2.10.5" + "version": "==2.10.7" }, "ansible-base": { "hashes": [ @@ -32,19 +32,19 @@ }, "boto3": { "hashes": [ - "sha256:3f26aad4c6b238055d17fd662620284ffb4ced542ed9a2f7f9df65d97a3f1190", - "sha256:47151ed571c316458f4931cd2422995ba0c9f6818c5df7d75f49fc845208e42e" + "sha256:a482135c30fa07eaf4370314dd0fb49117222a266d0423b2075aed3835ed1f04", + "sha256:d5ef160442925f5944e4cde88589f0f195f6c284f05613114fc6bbc35e342fa7" ], "index": "pypi", - "version": "==1.16.56" + "version": "==1.17.49" }, "botocore": { "hashes": [ - "sha256:c756d65ffa989c5c0e92178175e41abf7b18ad19b2fe2e82e192f085e264e03a", - "sha256:cf7d108a4d67a0fe670379111927b5d9e0ff1160146c81c326bb9e54c2b8cb19" + "sha256:6a672ba41dd00e5c1c1824ca8143d180d88de8736d78c0b1f96b8d3cb0466561", + "sha256:f7f103fa0651c69dd360c7d0ecd874854303de5cc0869e0cbc2818a52baacc69" ], "index": "pypi", - "version": "==1.19.57" + "version": "==1.20.49" }, "cffi": { "hashes": [ @@ -119,7 +119,7 @@ "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9", "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.0" }, "markupsafe": { @@ -201,7 +201,7 @@ "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.4.7" }, "python-dateutil": { @@ -209,7 +209,7 @@ "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.8.1" }, "pyyaml": { @@ -249,17 +249,17 @@ }, "s3transfer": { "hashes": [ - "sha256:5d48b1fd2232141a9d5fb279709117aaba506cacea7f86f11bc392f06bfa8fc2", - "sha256:c5dadf598762899d8cfaecf68eba649cd25b0ce93b6c954b156aaa3eed160547" + "sha256:35627b86af8ff97e7ac27975fe0a98a312814b46c6333d8a6b889627bcd80994", + "sha256:efa5bd92a897b6a8d5c1383828dca3d52d0790e0756d49740563a3fb6ed03246" ], - "version": "==0.3.6" + "version": "==0.3.7" }, "six": { "hashes": [ "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.15.0" }, "urllib3": { @@ -267,7 +267,7 @@ "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df", "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937" ], - "markers": "python_version != '3.4'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", "version": "==1.26.4" } }, @@ -325,19 +325,19 @@ }, "boto3": { "hashes": [ - "sha256:3f26aad4c6b238055d17fd662620284ffb4ced542ed9a2f7f9df65d97a3f1190", - "sha256:47151ed571c316458f4931cd2422995ba0c9f6818c5df7d75f49fc845208e42e" + "sha256:a482135c30fa07eaf4370314dd0fb49117222a266d0423b2075aed3835ed1f04", + "sha256:d5ef160442925f5944e4cde88589f0f195f6c284f05613114fc6bbc35e342fa7" ], "index": "pypi", - "version": "==1.16.56" + "version": "==1.17.49" }, "botocore": { "hashes": [ - "sha256:c756d65ffa989c5c0e92178175e41abf7b18ad19b2fe2e82e192f085e264e03a", - "sha256:cf7d108a4d67a0fe670379111927b5d9e0ff1160146c81c326bb9e54c2b8cb19" + "sha256:6a672ba41dd00e5c1c1824ca8143d180d88de8736d78c0b1f96b8d3cb0466561", + "sha256:f7f103fa0651c69dd360c7d0ecd874854303de5cc0869e0cbc2818a52baacc69" ], "index": "pypi", - "version": "==1.19.57" + "version": "==1.20.49" }, "cerberus": { "hashes": [ @@ -551,7 +551,7 @@ "sha256:2ec0faae539743ae6aaa84b49a169670a465f7f5d64e6add98388cc29fd1f2f6", "sha256:c9356b657de65c53744046fa8f7358afe0714a1af7d570c00c3835c2d724a7c1" ], - "markers": "python_version < '3.8' and python_version < '3.8'", + "markers": "python_version < '3.8'", "version": "==3.10.1" }, "iniconfig": { @@ -581,7 +581,7 @@ "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9", "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.0" }, "jsonpatch": { @@ -801,7 +801,7 @@ "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.4.7" }, "pyrsistent": { @@ -832,7 +832,7 @@ "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.8.1" }, "python-slugify": { @@ -900,10 +900,10 @@ }, "s3transfer": { "hashes": [ - "sha256:5d48b1fd2232141a9d5fb279709117aaba506cacea7f86f11bc392f06bfa8fc2", - "sha256:c5dadf598762899d8cfaecf68eba649cd25b0ce93b6c954b156aaa3eed160547" + "sha256:35627b86af8ff97e7ac27975fe0a98a312814b46c6333d8a6b889627bcd80994", + "sha256:efa5bd92a897b6a8d5c1383828dca3d52d0790e0756d49740563a3fb6ed03246" ], - "version": "==0.3.6" + "version": "==0.3.7" }, "selinux": { "hashes": [ @@ -926,7 +926,7 @@ "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.15.0" }, "subprocess-tee": { @@ -971,7 +971,7 @@ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.2" }, "typing-extensions": { @@ -980,7 +980,7 @@ "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c", "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f" ], - "markers": "python_version < '3.8' and python_version < '3.8'", + "markers": "python_version < '3.8'", "version": "==3.7.4.3" }, "urllib3": { @@ -988,7 +988,7 @@ "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df", "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937" ], - "markers": "python_version != '3.4'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", "version": "==1.26.4" }, "websocket-client": { From 956f78b4ef047ceefc82722e184c6ac6f6531653 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 14 Apr 2021 11:06:54 +1000 Subject: [PATCH 9/9] DCD-1276: Use snyk monitoring rather than local testing for now. --- bitbucket-pipelines.yml | 1 - pipeline_generator/templates/bitbucket-pipelines.yml.j2 | 1 - 2 files changed, 2 deletions(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index b77cb38..3c618ab 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -355,5 +355,4 @@ pipelines: - apt-get update && apt-get install -y npm - npm install -g snyk - snyk auth $SNYK_TOKEN - - pipenv run snyk test --severity-threshold=high - pipenv run snyk monitor --severity-threshold=high --project-name=dc-deployments-automation diff --git a/pipeline_generator/templates/bitbucket-pipelines.yml.j2 b/pipeline_generator/templates/bitbucket-pipelines.yml.j2 index 2087084..bd05ff8 100644 --- a/pipeline_generator/templates/bitbucket-pipelines.yml.j2 +++ b/pipeline_generator/templates/bitbucket-pipelines.yml.j2 @@ -53,5 +53,4 @@ pipelines: - apt-get update && apt-get install -y npm - npm install -g snyk - snyk auth $SNYK_TOKEN - - pipenv run snyk test --severity-threshold=high - pipenv run snyk monitor --severity-threshold=high --project-name=dc-deployments-automation