From 58632c27f986a5481707a87ff32ace2f66579c57 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 3 Oct 2019 11:24:31 +1000 Subject: [PATCH 01/27] DCD-686: Initial manifest download logic. --- group_vars/aws_node_local.yml | 2 ++ .../molecule/default/Dockerfile.j2 | 14 ++++++++++ .../molecule/default/molecule.yml | 20 ++++++++++++++ .../molecule/default/playbook.yml | 12 +++++++++ .../molecule/default/tests/test_default.py | 19 ++++++++++++++ roles/restore_metadata/tasks/main.yml | 26 +++++++++++++++++++ 6 files changed, 93 insertions(+) create mode 100644 roles/restore_metadata/molecule/default/Dockerfile.j2 create mode 100644 roles/restore_metadata/molecule/default/molecule.yml create mode 100644 roles/restore_metadata/molecule/default/playbook.yml create mode 100644 roles/restore_metadata/molecule/default/tests/test_default.py create mode 100644 roles/restore_metadata/tasks/main.yml diff --git a/group_vars/aws_node_local.yml b/group_vars/aws_node_local.yml index cd6ea3b..6db2a6d 100644 --- a/group_vars/aws_node_local.yml +++ b/group_vars/aws_node_local.yml @@ -126,3 +126,5 @@ atl_rds_instance_class: "{{ lookup('env', 'ATL_RDS_INSTANCE_CLASS') }}" atl_rds_multi_az: "{{ lookup('env', 'ATL_RDS_MULTI_AZ') }}" atl_rds_subnet_group_name: "{{ lookup('env', 'ATL_RDS_SUBNET_GROUP_NAME') }}" atl_rds_security_group: "{{ lookup('env', 'ATL_RDS_SECURITY_GROUP') }}" + +atl_backup_manifest_url: "{{ lookup('env', 'ATL_BACKUP_MANIFEST_URL) }}" diff --git a/roles/restore_metadata/molecule/default/Dockerfile.j2 b/roles/restore_metadata/molecule/default/Dockerfile.j2 new file mode 100644 index 0000000..e6aa95d --- /dev/null +++ b/roles/restore_metadata/molecule/default/Dockerfile.j2 @@ -0,0 +1,14 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi diff --git a/roles/restore_metadata/molecule/default/molecule.yml b/roles/restore_metadata/molecule/default/molecule.yml new file mode 100644 index 0000000..c6a6d26 --- /dev/null +++ b/roles/restore_metadata/molecule/default/molecule.yml @@ -0,0 +1,20 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: amazon_linux2 + image: amazonlinux:2 + - name: ubuntu_lts + image: ubuntu:bionic +provisioner: + name: ansible + lint: + name: ansible-lint +verifier: + name: testinfra + lint: + name: flake8 diff --git a/roles/restore_metadata/molecule/default/playbook.yml b/roles/restore_metadata/molecule/default/playbook.yml new file mode 100644 index 0000000..6e24bad --- /dev/null +++ b/roles/restore_metadata/molecule/default/playbook.yml @@ -0,0 +1,12 @@ +--- +- name: Converge + hosts: all + vars: + atl_product_user: "testuser" + atl_product_home: "/opt/atlassian/product" + atl_product_installation_base: "/opt/atlassian/product/install" + atl_installer_temp: "/opt/atlassian/temp" + atl_product_home_shared: "/media/atl/jira/shared" + atl_product_shared_plugins: "/media/atl/jira/shared/plugins/" + roles: + - role: restore_metadata diff --git a/roles/restore_metadata/molecule/default/tests/test_default.py b/roles/restore_metadata/molecule/default/tests/test_default.py new file mode 100644 index 0000000..b29ef83 --- /dev/null +++ b/roles/restore_metadata/molecule/default/tests/test_default.py @@ -0,0 +1,19 @@ +import os +import pytest + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +def test_user_created(host): + user = host.user('testuser') + assert user.exists + + +@pytest.mark.parametrize('exe', [ + '/usr/bin/git' +]) +def test_package_exes(host, exe): + assert host.file(exe).exists diff --git a/roles/restore_metadata/tasks/main.yml b/roles/restore_metadata/tasks/main.yml new file mode 100644 index 0000000..a6bd85f --- /dev/null +++ b/roles/restore_metadata/tasks/main.yml @@ -0,0 +1,26 @@ +--- + +- block: + + - name: Parse the manifest URL + set_fact: + atl_backup_manifest_scheme: "{{ atl_backup_manifest_url | urlsplit('scheme') }}" + atl_backup_manifest_bucket: "{{ atl_backup_manifest_url | urlsplit('hostname') }}" + atl_backup_manifest_path: "{{ atl_backup_manifest_url | urlsplit('path') }}" + atl_backup_manifest_filename: "{{ atl_backup_manifest_path | basename}}" + atl_backup_manifest_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest_filename }}" + + - name: Fetch the manifest from S3 + aws_s3: + bucket: "{{ atl_backup_manifest | urlsplit('hostname' }}" + object: "{{ atl_backup_manifest | urlsplit('path' }}" + dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest | urlsplit('path' | }}" + when: atl_backup_manifest_scheme == 's3' + + - name: Fetch the manifest from remote host + get_url: + url: "{{ atl_backup_manifest_url }}" + dest: "{{ atl_backup_manifest_dest }}" + when: atl_backup_manifest_scheme != 's3' + + when: atl_backup_manifest_url is defined and atl_backup_manifest_url != '' From 24d8a8f16cf5b6a0f0c7100bccabeb192848856e Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 3 Oct 2019 11:34:25 +1000 Subject: [PATCH 02/27] DCD-686: Fix missing quote. --- group_vars/aws_node_local.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/group_vars/aws_node_local.yml b/group_vars/aws_node_local.yml index 6db2a6d..69285f9 100644 --- a/group_vars/aws_node_local.yml +++ b/group_vars/aws_node_local.yml @@ -127,4 +127,4 @@ atl_rds_multi_az: "{{ lookup('env', 'ATL_RDS_MULTI_AZ') }}" atl_rds_subnet_group_name: "{{ lookup('env', 'ATL_RDS_SUBNET_GROUP_NAME') }}" atl_rds_security_group: "{{ lookup('env', 'ATL_RDS_SECURITY_GROUP') }}" -atl_backup_manifest_url: "{{ lookup('env', 'ATL_BACKUP_MANIFEST_URL) }}" +atl_backup_manifest_url: "{{ lookup('env', 'ATL_BACKUP_MANIFEST_URL') }}" From cfca695c31107175e4954de5d365e8114e90f614 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 3 Oct 2019 11:36:49 +1000 Subject: [PATCH 03/27] DCD-686: set_fact variables don't stack. --- roles/restore_metadata/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/restore_metadata/tasks/main.yml b/roles/restore_metadata/tasks/main.yml index a6bd85f..848e1de 100644 --- a/roles/restore_metadata/tasks/main.yml +++ b/roles/restore_metadata/tasks/main.yml @@ -8,7 +8,7 @@ atl_backup_manifest_bucket: "{{ atl_backup_manifest_url | urlsplit('hostname') }}" atl_backup_manifest_path: "{{ atl_backup_manifest_url | urlsplit('path') }}" atl_backup_manifest_filename: "{{ atl_backup_manifest_path | basename}}" - atl_backup_manifest_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest_filename }}" + atl_backup_manifest_dest: "{{ atl_installer_temp }}/{{ {{ atl_backup_manifest_path | basename}}" - name: Fetch the manifest from S3 aws_s3: From 38b1441d4a95faa193f27a9be7ceb818401cc4a2 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 3 Oct 2019 11:37:51 +1000 Subject: [PATCH 04/27] DCD-686: Typo --- roles/restore_metadata/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/restore_metadata/tasks/main.yml b/roles/restore_metadata/tasks/main.yml index 848e1de..7f7c30f 100644 --- a/roles/restore_metadata/tasks/main.yml +++ b/roles/restore_metadata/tasks/main.yml @@ -8,7 +8,7 @@ atl_backup_manifest_bucket: "{{ atl_backup_manifest_url | urlsplit('hostname') }}" atl_backup_manifest_path: "{{ atl_backup_manifest_url | urlsplit('path') }}" atl_backup_manifest_filename: "{{ atl_backup_manifest_path | basename}}" - atl_backup_manifest_dest: "{{ atl_installer_temp }}/{{ {{ atl_backup_manifest_path | basename}}" + atl_backup_manifest_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest_path | basename}}" - name: Fetch the manifest from S3 aws_s3: From 79fd9e5db940c224dd2dc5924225701f82dc4c07 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 3 Oct 2019 11:47:10 +1000 Subject: [PATCH 05/27] DCD-686: More tweaks to manifest handling. --- roles/restore_metadata/tasks/main.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/roles/restore_metadata/tasks/main.yml b/roles/restore_metadata/tasks/main.yml index 7f7c30f..c532608 100644 --- a/roles/restore_metadata/tasks/main.yml +++ b/roles/restore_metadata/tasks/main.yml @@ -4,17 +4,19 @@ - name: Parse the manifest URL set_fact: - atl_backup_manifest_scheme: "{{ atl_backup_manifest_url | urlsplit('scheme') }}" - atl_backup_manifest_bucket: "{{ atl_backup_manifest_url | urlsplit('hostname') }}" - atl_backup_manifest_path: "{{ atl_backup_manifest_url | urlsplit('path') }}" - atl_backup_manifest_filename: "{{ atl_backup_manifest_path | basename}}" - atl_backup_manifest_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest_path | basename}}" + atl_backup_manifest_url: "{{ atl_backup_manifest_url | urlsplit }}" + + - name: Extract manifest file information + set_fact: + atl_backup_manifest_bucket: "{{ atl_backup_manifest_url.hostname }}" + atl_backup_manifest_path: "{{ atl_backup_manifest_url.path }}" + atl_backup_manifest_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest_url.path | basename }}" - name: Fetch the manifest from S3 aws_s3: - bucket: "{{ atl_backup_manifest | urlsplit('hostname' }}" - object: "{{ atl_backup_manifest | urlsplit('path' }}" - dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest | urlsplit('path' | }}" + bucket: "{{ atl_backup_manifest_bucket }}" + object: "{{ atl_backup_manifest_path }}" + dest: "{{ atl_backup_manifest_dest }}" when: atl_backup_manifest_scheme == 's3' - name: Fetch the manifest from remote host From 347bdab24db209922397797e5ff3f029ea190023 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 3 Oct 2019 11:48:47 +1000 Subject: [PATCH 06/27] DCD-686: Correct scheme lookup. --- roles/restore_metadata/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/restore_metadata/tasks/main.yml b/roles/restore_metadata/tasks/main.yml index c532608..74de3d2 100644 --- a/roles/restore_metadata/tasks/main.yml +++ b/roles/restore_metadata/tasks/main.yml @@ -17,12 +17,12 @@ bucket: "{{ atl_backup_manifest_bucket }}" object: "{{ atl_backup_manifest_path }}" dest: "{{ atl_backup_manifest_dest }}" - when: atl_backup_manifest_scheme == 's3' + when: atl_backup_manifest_url.scheme == 's3' - name: Fetch the manifest from remote host get_url: url: "{{ atl_backup_manifest_url }}" dest: "{{ atl_backup_manifest_dest }}" - when: atl_backup_manifest_scheme != 's3' + when: atl_backup_manifest_url.scheme != 's3' when: atl_backup_manifest_url is defined and atl_backup_manifest_url != '' From 8c3d7ce9f4d17dd00de86239bd65572a957744d6 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 3 Oct 2019 11:50:14 +1000 Subject: [PATCH 07/27] DCD-686: Add file permissions. --- roles/restore_metadata/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/restore_metadata/tasks/main.yml b/roles/restore_metadata/tasks/main.yml index 74de3d2..5c13946 100644 --- a/roles/restore_metadata/tasks/main.yml +++ b/roles/restore_metadata/tasks/main.yml @@ -23,6 +23,7 @@ get_url: url: "{{ atl_backup_manifest_url }}" dest: "{{ atl_backup_manifest_dest }}" + mode: 0600 when: atl_backup_manifest_url.scheme != 's3' when: atl_backup_manifest_url is defined and atl_backup_manifest_url != '' From a3b3460f97b26f33dad746a3d15fa09481ae1410 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 3 Oct 2019 11:51:14 +1000 Subject: [PATCH 08/27] DCD-686: Add file permissions to s3 fetch. --- roles/restore_metadata/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/restore_metadata/tasks/main.yml b/roles/restore_metadata/tasks/main.yml index 5c13946..8afd08b 100644 --- a/roles/restore_metadata/tasks/main.yml +++ b/roles/restore_metadata/tasks/main.yml @@ -17,6 +17,7 @@ bucket: "{{ atl_backup_manifest_bucket }}" object: "{{ atl_backup_manifest_path }}" dest: "{{ atl_backup_manifest_dest }}" + mode: 0600 when: atl_backup_manifest_url.scheme == 's3' - name: Fetch the manifest from remote host From 89038fb7c055b03b3a980b569bc6691a19c56667 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 3 Oct 2019 11:52:34 +1000 Subject: [PATCH 09/27] DCD-686: Fix S3 download. --- roles/restore_metadata/tasks/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/restore_metadata/tasks/main.yml b/roles/restore_metadata/tasks/main.yml index 8afd08b..f23f1f0 100644 --- a/roles/restore_metadata/tasks/main.yml +++ b/roles/restore_metadata/tasks/main.yml @@ -14,17 +14,16 @@ - name: Fetch the manifest from S3 aws_s3: + mode: get bucket: "{{ atl_backup_manifest_bucket }}" object: "{{ atl_backup_manifest_path }}" dest: "{{ atl_backup_manifest_dest }}" - mode: 0600 when: atl_backup_manifest_url.scheme == 's3' - name: Fetch the manifest from remote host get_url: url: "{{ atl_backup_manifest_url }}" dest: "{{ atl_backup_manifest_dest }}" - mode: 0600 when: atl_backup_manifest_url.scheme != 's3' when: atl_backup_manifest_url is defined and atl_backup_manifest_url != '' From 14e2fdf25b673fc6548c4e07b198a44bf32edf9f Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 3 Oct 2019 13:24:07 +1000 Subject: [PATCH 10/27] DCD-686: Ensure temp directory is present before downloads. --- roles/restore_metadata/tasks/main.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/roles/restore_metadata/tasks/main.yml b/roles/restore_metadata/tasks/main.yml index f23f1f0..5264854 100644 --- a/roles/restore_metadata/tasks/main.yml +++ b/roles/restore_metadata/tasks/main.yml @@ -2,6 +2,15 @@ - block: + - name: Ensure temp directory is present + file: + path: "{{ atl_installer_temp }}" + state: directory + mode: 0750 + owner: "{{ atl_product_user }}" + group: "{{ atl_product_user }}" + changed_when: false # For Molecule idempotence check + - name: Parse the manifest URL set_fact: atl_backup_manifest_url: "{{ atl_backup_manifest_url | urlsplit }}" From 46adc9af3f9212f2914004adc72ee16b8eb30b22 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 3 Oct 2019 13:29:21 +1000 Subject: [PATCH 11/27] DCD-686: Move to Python 3.7 and update packages to support AWS operations. --- Pipfile | 4 +- Pipfile.lock | 501 +++++++++++++++++++------------------------- bin/install-ansible | 9 +- 3 files changed, 225 insertions(+), 289 deletions(-) diff --git a/Pipfile b/Pipfile index 0baa144..4d8129d 100644 --- a/Pipfile +++ b/Pipfile @@ -5,6 +5,8 @@ name = "pypi" [packages] ansible = "==2.7.11" +boto3 = "==1.9.241" +botocore = "==1.12.241" [dev-packages] molecule = "==2.20.1" @@ -14,4 +16,4 @@ taskcat = "*" Jinja2 = "*" [requires] -python_version = "2.7" +python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock index 12753dd..976eed7 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "d19b07115cf0a0e5ea9ce4283e43ee9c0efa52683080b730b8bf943ed87861e8" + "sha256": "fe304fca8752522c4a630677978735c57d39f161e7d0046ea128a21c7c28e373" }, "pipfile-spec": 6, "requires": { - "python_version": "2.7" + "python_version": "3.7" }, "sources": [ { @@ -25,34 +25,47 @@ }, "asn1crypto": { "hashes": [ - "sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87", - "sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49" + "sha256:d02bf8ea1b964a5ff04ac7891fe3a39150045d1e5e4fe99273ba677d11b92a04", + "sha256:f822954b90c4c44f002e2cd46d636ab630f1fe4df22c816a82b66505c404eb2a" ], - "version": "==0.24.0" + "version": "==1.0.0" }, "bcrypt": { "hashes": [ - "sha256:0ba875eb67b011add6d8c5b76afbd92166e98b1f1efab9433d5dc0fafc76e203", - "sha256:21ed446054c93e209434148ef0b362432bb82bbdaf7beef70a32c221f3e33d1c", - "sha256:28a0459381a8021f57230954b9e9a65bb5e3d569d2c253c5cac6cb181d71cf23", - "sha256:2aed3091eb6f51c26b7c2fad08d6620d1c35839e7a362f706015b41bd991125e", - "sha256:2fa5d1e438958ea90eaedbf8082c2ceb1a684b4f6c75a3800c6ec1e18ebef96f", - "sha256:3a73f45484e9874252002793518da060fb11eaa76c30713faa12115db17d1430", - "sha256:3e489787638a36bb466cd66780e15715494b6d6905ffdbaede94440d6d8e7dba", - "sha256:44636759d222baa62806bbceb20e96f75a015a6381690d1bc2eda91c01ec02ea", - "sha256:678c21b2fecaa72a1eded0cf12351b153615520637efcadc09ecf81b871f1596", - "sha256:75460c2c3786977ea9768d6c9d8957ba31b5fbeb0aae67a5c0e96aab4155f18c", - "sha256:8ac06fb3e6aacb0a95b56eba735c0b64df49651c6ceb1ad1cf01ba75070d567f", - "sha256:8fdced50a8b646fff8fa0e4b1c5fd940ecc844b43d1da5a980cb07f2d1b1132f", - "sha256:9b2c5b640a2da533b0ab5f148d87fb9989bf9bcb2e61eea6a729102a6d36aef9", - "sha256:a9083e7fa9adb1a4de5ac15f9097eb15b04e2c8f97618f1b881af40abce382e1", - "sha256:b7e3948b8b1a81c5a99d41da5fb2dc03ddb93b5f96fcd3fd27e643f91efa33e1", - "sha256:b998b8ca979d906085f6a5d84f7b5459e5e94a13fc27c28a3514437013b6c2f6", - "sha256:dd08c50bc6f7be69cd7ba0769acca28c846ec46b7a8ddc2acf4b9ac6f8a7457e", - "sha256:de5badee458544ab8125e63e39afeedfcf3aef6a6e2282ac159c95ae7472d773", - "sha256:ede2a87333d24f55a4a7338a6ccdccf3eaa9bed081d1737e0db4dbd1a4f7e6b6" + "sha256:0258f143f3de96b7c14f762c770f5fc56ccd72f8a1857a451c1cd9a655d9ac89", + "sha256:0b0069c752ec14172c5f78208f1863d7ad6755a6fae6fe76ec2c80d13be41e42", + "sha256:19a4b72a6ae5bb467fea018b825f0a7d917789bcfe893e53f15c92805d187294", + "sha256:5432dd7b34107ae8ed6c10a71b4397f1c853bd39a4d6ffa7e35f40584cffd161", + "sha256:69361315039878c0680be456640f8705d76cb4a3a3fe1e057e0f261b74be4b31", + "sha256:6fe49a60b25b584e2f4ef175b29d3a83ba63b3a4df1b4c0605b826668d1b6be5", + "sha256:74a015102e877d0ccd02cdeaa18b32aa7273746914a6c5d0456dd442cb65b99c", + "sha256:763669a367869786bb4c8fcf731f4175775a5b43f070f50f46f0b59da45375d0", + "sha256:8b10acde4e1919d6015e1df86d4c217d3b5b01bb7744c36113ea43d529e1c3de", + "sha256:9fe92406c857409b70a38729dbdf6578caf9228de0aef5bc44f859ffe971a39e", + "sha256:a190f2a5dbbdbff4b74e3103cef44344bc30e61255beb27310e2aec407766052", + "sha256:a595c12c618119255c90deb4b046e1ca3bcfad64667c43d1166f2b04bc72db09", + "sha256:c9457fa5c121e94a58d6505cadca8bed1c64444b83b3204928a866ca2e599105", + "sha256:cb93f6b2ab0f6853550b74e051d297c27a638719753eb9ff66d1e4072be67133", + "sha256:d7bdc26475679dd073ba0ed2766445bb5b20ca4793ca0db32b399dccc6bc84b7", + "sha256:ff032765bb8716d9387fd5376d987a937254b0619eff0972779515b5c98820bc" ], - "version": "==3.1.6" + "version": "==3.1.7" + }, + "boto3": { + "hashes": [ + "sha256:60e711f1113be926bcec1cfe62fa336438d021ce834f4a5228beead3b4bc5142", + "sha256:8c9b9b2422c1baa84c0f331ee86ac4d265e1e7d321ce7ba58dbb863585c2191f" + ], + "index": "pypi", + "version": "==1.9.241" + }, + "botocore": { + "hashes": [ + "sha256:897415ec68b2cbb65a7d32965c456d332bb2eb936e533c9ad6064cd15e67c0c1", + "sha256:e35c2e6b8946be9063d7988b19dea2b6136b80c0e3469b6a076c574d5abca6b3" + ], + "index": "pypi", + "version": "==1.12.241" }, "cffi": { "hashes": [ @@ -108,23 +121,13 @@ ], "version": "==2.7" }, - "enum34": { + "docutils": { "hashes": [ - "sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850", - "sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a", - "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", - "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" + "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0", + "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827", + "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99" ], - "markers": "python_version < '3'", - "version": "==1.1.6" - }, - "ipaddress": { - "hashes": [ - "sha256:64b28eec5e78e7510698f6d4da08800a5c575caa4a286c93d651c5d3ff7b6794", - "sha256:b146c751ea45cad6188dd6cf2d9b757f6f4f8d6ffb96a023e6f2e26eea02a72c" - ], - "markers": "python_version < '3'", - "version": "==1.0.22" + "version": "==0.15.2" }, "jinja2": { "hashes": [ @@ -133,6 +136,13 @@ ], "version": "==2.10.1" }, + "jmespath": { + "hashes": [ + "sha256:3720a4b1bd659dd2eecad0666459b9788813e032b83e7ba58578e48254e0a0e6", + "sha256:bde2aef6f44302dfb30320115b17d030798de8c4110e28d5cf6cf91a7a31074c" + ], + "version": "==0.9.4" + }, "markupsafe": { "hashes": [ "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", @@ -168,10 +178,10 @@ }, "paramiko": { "hashes": [ - "sha256:69c219df239775800a2589ee60159aa7cfd87175809b6557da7fb9dcb44ca430", - "sha256:9f081281064b5180dc0ef60e256224a280ff16f603a99f3dd4ba6334ebb65f7e" + "sha256:99f0179bdc176281d21961a003ffdb2ec369daac1a1007241f53374e376576cf", + "sha256:f4b2edfa0d226b70bd4ca31ea7e389325990283da23465d572ed1f70a7583041" ], - "version": "==2.5.0" + "version": "==2.6.0" }, "pycparser": { "hashes": [ @@ -203,21 +213,38 @@ ], "version": "==1.3.0" }, + "python-dateutil": { + "hashes": [ + "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", + "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" + ], + "markers": "python_version >= '2.7'", + "version": "==2.8.0" + }, "pyyaml": { "hashes": [ - "sha256:57acc1d8533cbe51f6662a55434f0dbecfa2b9eaf115bede8f6fd00115a0c0d3", - "sha256:588c94b3d16b76cfed8e0be54932e5729cc185caffaa5a451e7ad2f7ed8b4043", - "sha256:68c8dd247f29f9a0d09375c9c6b8fdc64b60810ebf07ba4cdd64ceee3a58c7b7", - "sha256:70d9818f1c9cd5c48bb87804f2efc8692f1023dac7f1a1a5c61d454043c1d265", - "sha256:86a93cccd50f8c125286e637328ff4eef108400dd7089b46a7be3445eecfa391", - "sha256:a0f329125a926876f647c9fa0ef32801587a12328b4a3c741270464e3e4fa778", - "sha256:a3c252ab0fa1bb0d5a3f6449a4826732f3eb6c0270925548cac342bc9b22c225", - "sha256:b4bb4d3f5e232425e25dda21c070ce05168a786ac9eda43768ab7f3ac2770955", - "sha256:cd0618c5ba5bda5f4039b9398bb7fb6a317bb8298218c3de25c47c4740e4b95e", - "sha256:ceacb9e5f8474dcf45b940578591c7f3d960e82f926c707788a570b51ba59190", - "sha256:fe6a88094b64132c4bb3b631412e90032e8cfe9745a58370462240b8cb7553cd" + "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", + "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4", + "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8", + "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696", + "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34", + "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9", + "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73", + "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299", + "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b", + "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae", + "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681", + "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41", + "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8" ], - "version": "==5.1.1" + "version": "==5.1.2" + }, + "s3transfer": { + "hashes": [ + "sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d", + "sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba" + ], + "version": "==0.2.1" }, "six": { "hashes": [ @@ -225,6 +252,14 @@ "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" ], "version": "==1.12.0" + }, + "urllib3": { + "hashes": [ + "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398", + "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86" + ], + "markers": "python_version >= '3.4'", + "version": "==1.25.6" } }, "develop": { @@ -250,17 +285,17 @@ }, "arrow": { "hashes": [ - "sha256:03404b624e89ac5e4fc19c52045fa0f3203419fd4dd64f6e8958c522580a574a", - "sha256:41be7ea4c53c2cf57bf30f2d614f60c411160133f7a0a8c49111c30fb7e725b5" + "sha256:10257c5daba1a88db34afa284823382f4963feca7733b9107956bed041aff24f", + "sha256:c2325911fcd79972cf493cfd957072f9644af8ad25456201ae1ede3316576eb4" ], - "version": "==0.14.2" + "version": "==0.15.2" }, "asn1crypto": { "hashes": [ - "sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87", - "sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49" + "sha256:d02bf8ea1b964a5ff04ac7891fe3a39150045d1e5e4fe99273ba677d11b92a04", + "sha256:f822954b90c4c44f002e2cd46d636ab630f1fe4df22c816a82b66505c404eb2a" ], - "version": "==0.24.0" + "version": "==1.0.0" }, "atomicwrites": { "hashes": [ @@ -271,31 +306,16 @@ }, "attrs": { "hashes": [ - "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", - "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" + "sha256:ec20e7a4825331c1b5ebf261d111e16fa9612c1f7a5e1f884f12bd53a664dfd2", + "sha256:f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396" ], - "version": "==19.1.0" + "version": "==19.2.0" }, "aws-sam-translator": { "hashes": [ - "sha256:db872c43bdfbbae9fc8c9201e6a7aeb9a661cda116a94708ab0577b46a38b962" + "sha256:3c615bff465fcf6a7990b9f84d002d55c75cd3e52d98e727d24959756ab0f0b1" ], - "version": "==1.11.0" - }, - "backports.functools-lru-cache": { - "hashes": [ - "sha256:9d98697f088eb1b0fa451391f91afb5e3ebde16bbdb272819fd091151fda4f1a", - "sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd" - ], - "markers": "python_version == '2.7'", - "version": "==1.5" - }, - "backports.ssl-match-hostname": { - "hashes": [ - "sha256:bb82e60f9fbf4c080eabd957c39f0641f0fc247d9a16e31e26d594d8f42b9fd2" - ], - "markers": "python_version < '3.5'", - "version": "==3.7.0.1" + "version": "==1.14.0" }, "binaryornot": { "hashes": [ @@ -306,17 +326,19 @@ }, "boto3": { "hashes": [ - "sha256:794a9a4b6a9e40c1ac57a377de609872d28d62afe4295c48cdc1b1c92f96ab8e", - "sha256:962b078568cc520869ea2842f307864c9abc30ad5ed160e12b2a89debf220161" + "sha256:60e711f1113be926bcec1cfe62fa336438d021ce834f4a5228beead3b4bc5142", + "sha256:8c9b9b2422c1baa84c0f331ee86ac4d265e1e7d321ce7ba58dbb863585c2191f" ], - "version": "==1.9.168" + "index": "pypi", + "version": "==1.9.241" }, "botocore": { "hashes": [ - "sha256:675f2b66af486dd02f5825601bb0c8378773999f8705c6f75450849ca41fed80", - "sha256:c3fc314c0e0aa13aa024d272d991e23d37550050abf96b3c7dea889ed1743723" + "sha256:897415ec68b2cbb65a7d32965c456d332bb2eb936e533c9ad6064cd15e67c0c1", + "sha256:e35c2e6b8946be9063d7988b19dea2b6136b80c0e3469b6a076c574d5abca6b3" ], - "version": "==1.12.168" + "index": "pypi", + "version": "==1.12.241" }, "cerberus": { "hashes": [ @@ -326,10 +348,10 @@ }, "certifi": { "hashes": [ - "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", - "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" + "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", + "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" ], - "version": "==2019.3.9" + "version": "==2019.9.11" }, "cffi": { "hashes": [ @@ -366,10 +388,10 @@ }, "cfn-lint": { "hashes": [ - "sha256:16500272b5e2a3e9eb94e6b42c0a652b1a084fa96f8c5efb07ff4adde3b448ec", - "sha256:ce4bf8c0e6d5b8ad3f1b4cd8261e1eca795d61fb3723e3dce85c78eff95ab120" + "sha256:32a3e1597c681c9411205bff48b421db60908c304c472f4644d5a32bc9ecdad3", + "sha256:623cf0f6ed4c7b3fb4563549e25ac68119478900d89ca976639f11c5d85063a6" ], - "version": "==0.21.5" + "version": "==0.24.3" }, "chardet": { "hashes": [ @@ -398,22 +420,6 @@ ], "version": "==0.3.9" }, - "configparser": { - "hashes": [ - "sha256:8be81d89d6e7b4c0d4e44bcc525845f6da25821de80cb5e06e7e0238a2899e32", - "sha256:da60d0014fd8c55eb48c1c5354352e363e2d30bbf7057e5e171a468390184c75" - ], - "markers": "python_version < '3.2'", - "version": "==3.7.4" - }, - "contextlib2": { - "hashes": [ - "sha256:509f9419ee91cdd00ba34443217d5ca51f5a364a404e1dce9e8979cea969ca48", - "sha256:f5260a6e679d2ff42ec91ec5252f4eeffdcf21053db9113bd0a8e4d953769c00" - ], - "markers": "python_version < '3'", - "version": "==0.5.5" - }, "cookiecutter": { "hashes": [ "sha256:1316a52e1c1f08db0c9efbf7d876dbc01463a74b155a0d83e722be88beda9a3e", @@ -452,11 +458,11 @@ }, "docutils": { "hashes": [ - "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6", - "sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274", - "sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6" + "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0", + "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827", + "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99" ], - "version": "==0.14" + "version": "==0.15.2" }, "entrypoints": { "hashes": [ @@ -465,16 +471,6 @@ ], "version": "==0.3" }, - "enum34": { - "hashes": [ - "sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850", - "sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a", - "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", - "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" - ], - "markers": "python_version < '3'", - "version": "==1.1.6" - }, "fasteners": { "hashes": [ "sha256:007e4d2b2d4a10093f67e932e5166722d2eab83b77724156e92ad013c6226574", @@ -484,26 +480,10 @@ }, "flake8": { "hashes": [ - "sha256:859996073f341f2670741b51ec1e67a01da142831aa1fdc6242dbf88dffbe661", - "sha256:a796a115208f5c03b18f332f7c11729812c8c3ded6c46319c59b53efd3819da8" + "sha256:19241c1cbc971b9962473e4438a2ca19749a7dd002dd1a946eaba171b4114548", + "sha256:8e9dfa3cecb2400b3738a42c54c3043e821682b9c840b0448c0503f781130696" ], - "version": "==3.7.7" - }, - "funcsigs": { - "hashes": [ - "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca", - "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50" - ], - "markers": "python_version < '3.0'", - "version": "==1.0.2" - }, - "functools32": { - "hashes": [ - "sha256:89d824aa6c358c421a234d7f9ee0bd75933a67c29588ce50aaa3acdf4d403fa0", - "sha256:f6253dfbe0538ad2e387bd8fdfd9293c925d63553f5813c4e587745416501e6d" - ], - "markers": "python_version == '2.7'", - "version": "==3.2.3.post2" + "version": "==3.7.8" }, "future": { "hashes": [ @@ -511,14 +491,6 @@ ], "version": "==0.17.1" }, - "futures": { - "hashes": [ - "sha256:9ec02aa7d674acb8618afb127e27fde7fc68994c0437ad759fa094a574adb265", - "sha256:ec0a6cb848cc212002b9828c3e34c675e0c9ff6741dc445cab6fdd4e1085d1f1" - ], - "markers": "python_version == '2.6' or python_version == '2.7'", - "version": "==3.2.0" - }, "git-url-parse": { "hashes": [ "sha256:4655ee22f1d8bf7a1eb1066c1da16529b186966c6d8331f7f55686a76a9f7aef", @@ -536,18 +508,11 @@ }, "importlib-metadata": { "hashes": [ - "sha256:6dfd58dfe281e8d240937776065dd3624ad5469c835248219bd16cf2e12dbeb7", - "sha256:cb6ee23b46173539939964df59d3d72c3e0c1b5d54b84f1d8a7e912fe43612db" + "sha256:aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26", + "sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af" ], - "version": "==0.18" - }, - "ipaddress": { - "hashes": [ - "sha256:64b28eec5e78e7510698f6d4da08800a5c575caa4a286c93d651c5d3ff7b6794", - "sha256:b146c751ea45cad6188dd6cf2d9b757f6f4f8d6ffb96a023e6f2e26eea02a72c" - ], - "markers": "python_version < '3'", - "version": "==1.0.22" + "markers": "python_version < '3.8'", + "version": "==0.23" }, "jinja2": { "hashes": [ @@ -572,10 +537,10 @@ }, "jsonpatch": { "hashes": [ - "sha256:49f29cab70e9068db3b1dc6b656cbe2ee4edf7dfe9bf5a0055f17a4b6804a4b9", - "sha256:8bf92fa26bc42c346c03bd4517722a8e4f429225dbe775ac774b2c70d95dbd33" + "sha256:83f29a2978c13da29bfdf89da9d65542d62576479caf215df19632d7dc04c6e6", + "sha256:cbb72f8bf35260628aea6b508a107245f757d1ec839a19c34349985e2c05645a" ], - "version": "==1.23" + "version": "==1.24" }, "jsonpointer": { "hashes": [ @@ -586,10 +551,10 @@ }, "jsonschema": { "hashes": [ - "sha256:000e68abd33c972a5248544925a0cae7d1125f9bf6c58280d37546b946769a08", - "sha256:6ff5f3180870836cae40f06fa10419f557208175f13ad7bc26caa77beb1f6e02" + "sha256:5f9c0a719ca2ce14c5de2fd350a64fd2d13e8539db29836a86adc990bb1a068f", + "sha256:8d4a2b7b6c2237e0199c8ea1a6d3e05bf118e289ae2b9d7ba444182a2959560d" ], - "version": "==2.6.0" + "version": "==3.0.2" }, "markupsafe": { "hashes": [ @@ -655,26 +620,17 @@ }, "more-itertools": { "hashes": [ - "sha256:38a936c0a6d98a38bcc2d03fdaaedaba9f412879461dd2ceff8d37564d6522e4", - "sha256:c0a5785b1109a6bd7fac76d6837fd1feca158e54e521ccd2ae8bfe393cc9d4fc", - "sha256:fe7a7cae1ccb57d33952113ff4fa1bc5f879963600ed74918f1236e212ee50b9" + "sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832", + "sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4" ], - "markers": "python_version <= '2.7'", - "version": "==5.0.0" + "version": "==7.2.0" }, "packaging": { "hashes": [ - "sha256:0c98a5d0be38ed775798ece1b9727178c4469d9c3b4ada66e8e6b7849f8732af", - "sha256:9e1cbf8c12b1f1ce0bb5344b8d7ecf66a6f8a6e91bcb0c84593ed6d3ab5c4ab3" + "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", + "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108" ], - "version": "==19.0" - }, - "pathlib2": { - "hashes": [ - "sha256:25199318e8cc3c25dcb45cbe084cc061051336d5a9ea2a12448d3d8cb748f742", - "sha256:5887121d7f7df3603bca2f710e7219f3eca0eb69e0b7cc6e0a022e155ac931a7" - ], - "version": "==2.3.3" + "version": "==19.2" }, "pathspec": { "hashes": [ @@ -698,17 +654,17 @@ }, "pluggy": { "hashes": [ - "sha256:0825a152ac059776623854c1543d65a4ad408eb3d33ee114dff91e57ec6ae6fc", - "sha256:b9817417e95936bf75d85d3f8767f7df6cdde751fc40aed3bb3074cbcb77757c" + "sha256:0db4b7601aae1d35b4a033282da476845aa19185c1e6964b25cf324b5e4ec3e6", + "sha256:fa5fa1622fa6dd5c030e9cad086fa19ef6a0cf6d7a2d12318e10cb49d6d68f34" ], - "version": "==0.12.0" + "version": "==0.13.0" }, "poyo": { "hashes": [ - "sha256:c34a5413191210ed564640510e9c4a4ba3b698746d6b454d46eb5bfb30edcd1d", - "sha256:d1c317054145a6b1ca0608b5e676b943ddc3bfd671f886a2fe09288b98221edb" + "sha256:3e2ca8e33fdc3c411cd101ca395668395dd5dc7ac775b8e809e3def9f9fe041a", + "sha256:e26956aa780c45f011ca9886f044590e2d8fd8b61db7b1c1cf4e0869f48ed4dd" ], - "version": "==0.4.2" + "version": "==0.5.0" }, "psutil": { "hashes": [ @@ -753,13 +709,6 @@ ], "version": "==2.19" }, - "pyfiglet": { - "hashes": [ - "sha256:c6c2321755d09267b438ec7b936825a4910fec696292139e664ca8670e103639", - "sha256:d555bcea17fbeaf70eaefa48bb119352487e629c9b56f30f383e2c62dd67a01c" - ], - "version": "==0.8.post1" - }, "pyflakes": { "hashes": [ "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0", @@ -769,17 +718,23 @@ }, "pyparsing": { "hashes": [ - "sha256:1873c03321fc118f4e9746baf201ff990ceb915f433f23b395f5580d1840cb2a", - "sha256:9b6323ef4ab914af344ba97510e966d64ba91055d6b9afa6b30799340e89cc03" + "sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80", + "sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4" ], - "version": "==2.4.0" + "version": "==2.4.2" + }, + "pyrsistent": { + "hashes": [ + "sha256:34b47fa169d6006b32e99d4b3c4031f155e6e68ebcc107d6454852e8e0ee6533" + ], + "version": "==0.15.4" }, "pytest": { "hashes": [ - "sha256:4a784f1d4f2ef198fe9b7aef793e9fa1a3b2f84e822d9b3a64a181293a572d45", - "sha256:926855726d8ae8371803f7b2e6ec0a69953d9c6311fa7c3b6c1b929ff92d27da" + "sha256:13c1c9b22127a77fc684eee24791efafcef343335d855e3573791c68588fe1a5", + "sha256:d8ba7be9466f55ef96ba203fc0f90d0cf212f2f927e69186e1353e30bc7f62e5" ], - "version": "==4.6.3" + "version": "==5.2.0" }, "python-dateutil": { "hashes": [ @@ -799,19 +754,21 @@ }, "pyyaml": { "hashes": [ - "sha256:57acc1d8533cbe51f6662a55434f0dbecfa2b9eaf115bede8f6fd00115a0c0d3", - "sha256:588c94b3d16b76cfed8e0be54932e5729cc185caffaa5a451e7ad2f7ed8b4043", - "sha256:68c8dd247f29f9a0d09375c9c6b8fdc64b60810ebf07ba4cdd64ceee3a58c7b7", - "sha256:70d9818f1c9cd5c48bb87804f2efc8692f1023dac7f1a1a5c61d454043c1d265", - "sha256:86a93cccd50f8c125286e637328ff4eef108400dd7089b46a7be3445eecfa391", - "sha256:a0f329125a926876f647c9fa0ef32801587a12328b4a3c741270464e3e4fa778", - "sha256:a3c252ab0fa1bb0d5a3f6449a4826732f3eb6c0270925548cac342bc9b22c225", - "sha256:b4bb4d3f5e232425e25dda21c070ce05168a786ac9eda43768ab7f3ac2770955", - "sha256:cd0618c5ba5bda5f4039b9398bb7fb6a317bb8298218c3de25c47c4740e4b95e", - "sha256:ceacb9e5f8474dcf45b940578591c7f3d960e82f926c707788a570b51ba59190", - "sha256:fe6a88094b64132c4bb3b631412e90032e8cfe9745a58370462240b8cb7553cd" + "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", + "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4", + "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8", + "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696", + "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34", + "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9", + "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73", + "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299", + "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b", + "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae", + "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681", + "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41", + "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8" ], - "version": "==5.1.1" + "version": "==5.1.2" }, "requests": { "hashes": [ @@ -820,40 +777,36 @@ ], "version": "==2.22.0" }, - "ruamel.ordereddict": { - "hashes": [ - "sha256:08b4b19fe518d32251a5338e039c4dc9eb0876f2919f94c9b8d2f9446ea80806", - "sha256:150ce8e6c514a2a2b62753622a75874962561f8e5eeec81a3172ab952807bf0b", - "sha256:45541836cbfdde630033cae7bbbe35acbac87a0ceec79f944b7a3bedd940fe78", - "sha256:854dd4a524811b16111b1107d8a751e4ca064d2bb103d3d91deab75de36b6620", - "sha256:aee2fa23e884249b4284b728888c553d551e5bfd4de2731f10153fd7813ec55f", - "sha256:bf0a198c8ce5d973c24e5dba12d3abc254996788ca6ad8448eabc6aa710db149" - ], - "markers": "platform_python_implementation == 'CPython' and python_version <= '2.7'", - "version": "==0.4.13" - }, "ruamel.yaml": { "hashes": [ - "sha256:17dbf6b7362e7aee8494f7a0f5cffd44902a6331fe89ef0853b855a7930ab845", - "sha256:23731c9efb79f3f5609dedffeb6c5c47a68125fd3d4b157d9fc71b1cd49076a9", - "sha256:2bbdd598ae57bac20968cf9028cc67d37d83bdb7942a94b9478110bc72193148", - "sha256:34586084cdd60845a3e1bece2b58f0a889be25450db8cc0ea143ddf0f40557a2", - "sha256:35957fedbb287b01313bb5c556ffdc70c0277c3500213b5e73dfd8716f748d77", - "sha256:414cb87a40974a575830b406ffab4ab8c6cbd82eeb73abd2a9d1397c1f0223e1", - "sha256:428775be75db68d908b17e4e8dda424c410222f170dc173246aa63e972d094b3", - "sha256:514f670f7d36519bda504d507edfe63e3c20489f86c86d42bc4d9a6dbdf82c7b", - "sha256:5cb962c1ac6887c5da29138fbbe3b4b7705372eb54e599907fa63d4cd743246d", - "sha256:5f6e30282cf70fb7754e1a5f101e27b5240009766376e131b31ab49f14fe81be", - "sha256:86f8e010af6af0b4f42de2d0d9b19cb441e61d3416082186f9dd03c8552d13ad", - "sha256:8d47ed1e557d546bd2dfe54f504d7274274602ff7a0652cde84c258ad6c2d96d", - "sha256:98668876720bce1ac08562d8b93a564a80e3397e442c7ea19cebdcdf73da7f74", - "sha256:9e1f0ddc18d8355dcf5586a5d90417df56074f237812b8682a93b62cca9d2043", - "sha256:a7bc812a72a79d6b7dbb96fa5bee3950464b65ec055d3abc4db6572f2373a95c", - "sha256:b72e13f9f206ee103247b07afd5a39c8b1aa98e8eba80ddba184d030337220ba", - "sha256:bcff8ea9d916789e85e24beed8830c157fb8bc7c313e554733a8151540e66c01", - "sha256:c76e78b3bab652069b8d6f7889b0e72f3455c2b854b2e0a8818393d149ad0a0d" + "sha256:0db639b1b2742dae666c6fc009b8d1931ef15c9276ef31c0673cc6dcf766cf40", + "sha256:412a6f5cfdc0525dee6a27c08f5415c7fd832a7afcb7a0ed7319628aed23d408" ], - "version": "==0.15.97" + "version": "==0.16.5" + }, + "ruamel.yaml.clib": { + "hashes": [ + "sha256:1e77424825caba5553bbade750cec2277ef130647d685c2b38f68bc03453bac6", + "sha256:392b7c371312abf27fb549ec2d5e0092f7ef6e6c9f767bfb13e83cb903aca0fd", + "sha256:4d55386129291b96483edcb93b381470f7cd69f97585829b048a3d758d31210a", + "sha256:550168c02d8de52ee58c3d8a8193d5a8a9491a5e7b2462d27ac5bf63717574c9", + "sha256:57933a6986a3036257ad7bf283529e7c19c2810ff24c86f4a0cfeb49d2099919", + "sha256:615b0396a7fad02d1f9a0dcf9f01202bf9caefee6265198f252c865f4227fcc6", + "sha256:77556a7aa190be9a2bd83b7ee075d3df5f3c5016d395613671487e79b082d784", + "sha256:7aee724e1ff424757b5bd8f6c5bbdb033a570b2b4683b17ace4dbe61a99a657b", + "sha256:8073c8b92b06b572e4057b583c3d01674ceaf32167801fe545a087d7a1e8bf52", + "sha256:9c6d040d0396c28d3eaaa6cb20152cb3b2f15adf35a0304f4f40a3cf9f1d2448", + "sha256:a0ff786d2a7dbe55f9544b3f6ebbcc495d7e730df92a08434604f6f470b899c5", + "sha256:b1b7fcee6aedcdc7e62c3a73f238b3d080c7ba6650cd808bce8d7761ec484070", + "sha256:b66832ea8077d9b3f6e311c4a53d06273db5dc2db6e8a908550f3c14d67e718c", + "sha256:d0d3ac228c9bbab08134b4004d748cf9f8743504875b3603b3afbb97e3472947", + "sha256:d10e9dd744cf85c219bf747c75194b624cc7a94f0c80ead624b06bfa9f61d3bc", + "sha256:ea4362548ee0cbc266949d8a441238d9ad3600ca9910c3fe4e82ee3a50706973", + "sha256:ed5b3698a2bb241b7f5cbbe277eaa7fe48b07a58784fba4f75224fd066d253ad", + "sha256:f9dcc1ae73f36e8059589b601e8e4776b9976effd76c21ad6a855a74318efd6e" + ], + "markers": "platform_python_implementation == 'CPython' and python_version < '3.8'", + "version": "==0.2.0" }, "s3transfer": { "hashes": [ @@ -862,23 +815,6 @@ ], "version": "==0.2.1" }, - "scandir": { - "hashes": [ - "sha256:2586c94e907d99617887daed6c1d102b5ca28f1085f90446554abf1faf73123e", - "sha256:2ae41f43797ca0c11591c0c35f2f5875fa99f8797cb1a1fd440497ec0ae4b022", - "sha256:2b8e3888b11abb2217a32af0766bc06b65cc4a928d8727828ee68af5a967fa6f", - "sha256:2c712840c2e2ee8dfaf36034080108d30060d759c7b73a01a52251cc8989f11f", - "sha256:4d4631f6062e658e9007ab3149a9b914f3548cb38bfb021c64f39a025ce578ae", - "sha256:67f15b6f83e6507fdc6fca22fedf6ef8b334b399ca27c6b568cbfaa82a364173", - "sha256:7d2d7a06a252764061a020407b997dd036f7bd6a175a5ba2b345f0a357f0b3f4", - "sha256:8c5922863e44ffc00c5c693190648daa6d15e7c1207ed02d6f46a8dcc2869d32", - "sha256:92c85ac42f41ffdc35b6da57ed991575bdbe69db895507af88b9f499b701c188", - "sha256:b24086f2375c4a094a6b51e78b4cf7ca16c721dcee2eddd7aa6494b42d6d519d", - "sha256:cb925555f43060a1745d0a321cca94bcea927c50114b623d73179189a4e100ac" - ], - "markers": "python_version < '3.5'", - "version": "==1.10.0" - }, "sh": { "hashes": [ "sha256:ae3258c5249493cebe73cb4e18253a41ed69262484bad36fdb3efcb8ad8870bb", @@ -901,10 +837,10 @@ }, "taskcat": { "hashes": [ - "sha256:af4149d6b951cbc4974e5a03f8eb8c137e1f81ac98a348715eb6dc287f728b2a" + "sha256:e84eb198c74ca677b589889d4e6877568e25858235d51cdd99a8128d525b63b2" ], "index": "pypi", - "version": "==0.8.35" + "version": "==0.8.47" }, "testinfra": { "hashes": [ @@ -920,22 +856,13 @@ ], "version": "==0.1.2" }, - "typing": { - "hashes": [ - "sha256:4027c5f6127a6267a435201981ba156de91ad0d1d98e9ddc2aa173453453492d", - "sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4", - "sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a" - ], - "markers": "python_version < '3.5'", - "version": "==3.6.6" - }, "urllib3": { "hashes": [ - "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", - "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232" + "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398", + "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86" ], - "markers": "python_version == '2.7'", - "version": "==1.25.3" + "markers": "python_version >= '3.4'", + "version": "==1.25.6" }, "wcwidth": { "hashes": [ @@ -953,30 +880,30 @@ }, "whichcraft": { "hashes": [ - "sha256:7533870f751901a0ce43c93cc9850186e9eba7fe58c924dfb435968ba9c9fa4e", - "sha256:fecddd531f237ffc5db8b215409afb18fa30300699064cca4817521b4fc81815" + "sha256:acdbb91b63d6a15efbd6430d1d7b2d36e44a71697e93e19b7ded477afd9fce87", + "sha256:deda9266fbb22b8c64fd3ee45c050d61139cd87419765f588e37c8d23e236dd9" ], - "version": "==0.5.2" + "version": "==0.6.1" }, "yamllint": { "hashes": [ - "sha256:9a4fec2d40804979de5f54453fd1551bc1f8b59a7ad4a26fd7f26aeca34a83af", - "sha256:f97cd763fe7b588444a94cc44fd3764b832a613b5250baa2bfe8b84c91e4c330" + "sha256:67173339f28868260ce5912abfefa10e115ceb1d2ac1c4d8c7acc8c4ef6c9a8a", + "sha256:70a6f8316851254e197a6231c35577be29fa2fbe2c77390a54c9a50217cdaa13" ], - "version": "==1.16.0" + "version": "==1.17.0" }, "yattag": { "hashes": [ - "sha256:d7214d100315093e3ddc34da9840acbfa65c79ec84b48a8191ddf535353c2e3f" + "sha256:47d1c842e0da596bac081fcc047f2d6fd778b16d20745a28c00ce99d80831fbc" ], - "version": "==1.11.2" + "version": "==1.12.2" }, "zipp": { "hashes": [ - "sha256:8c1019c6aad13642199fbe458275ad6a84907634cc9f0989877ccc4a2840139d", - "sha256:ca943a7e809cc12257001ccfb99e3563da9af99d52f261725e96dfe0f9275bc3" + "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e", + "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335" ], - "version": "==0.5.1" + "version": "==0.6.0" } } } diff --git a/bin/install-ansible b/bin/install-ansible index a6da6bf..8a32ac5 100755 --- a/bin/install-ansible +++ b/bin/install-ansible @@ -10,7 +10,14 @@ set -e # Luckily AmazonLinux2 and Ubuntu use the same package name for # pip. This may need some logic if other distros are added. Note: # Parsing /etc/os-release is probably a good starting point for that. -./bin/pacapt install --noconfirm python-pip +# +# Additionally we need to install boto3 and botocore, as the Ansible +# AWS modules manage to escape the virtualenv and invoke the native python. + +./bin/pacapt install --noconfirm \ + python-pip \ + python-boto3 \ + python-botocore export PATH=$PATH:/usr/local/bin # See Pipfile and Pipfile.lock. From f5ebeaedb0836c6737f5f5dbc258284fae0b7768 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 3 Oct 2019 14:23:05 +1000 Subject: [PATCH 12/27] DCD-686: Amazon Linux 2 has updated its Ansible packages so we can remove pipenv from the installation phase. --- Pipfile | 2 +- Pipfile.lock | 62 ++++------------------------------------ bin/ansible-with-atl-env | 11 ++++--- bin/install-ansible | 23 ++++----------- 4 files changed, 17 insertions(+), 81 deletions(-) diff --git a/Pipfile b/Pipfile index 4d8129d..55724ff 100644 --- a/Pipfile +++ b/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -ansible = "==2.7.11" +ansible = "==2.8.2" boto3 = "==1.9.241" botocore = "==1.12.241" diff --git a/Pipfile.lock b/Pipfile.lock index 976eed7..4c43753 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "fe304fca8752522c4a630677978735c57d39f161e7d0046ea128a21c7c28e373" + "sha256": "8f8923741e447b125ad9cb5c3912ba86e2239e753c7211ce1f074097892e1b6f" }, "pipfile-spec": 6, "requires": { @@ -18,10 +18,10 @@ "default": { "ansible": { "hashes": [ - "sha256:e7e6de461b7d07cb4d8b2dd2a32b231af7c56e6bf39b851024671aaa52fd377e" + "sha256:1e5ba829ca0602c55b33da399b06f99b135a34014b661d1c36d8892a1e2d3730" ], "index": "pypi", - "version": "==2.7.11" + "version": "==2.8.2" }, "asn1crypto": { "hashes": [ @@ -30,27 +30,6 @@ ], "version": "==1.0.0" }, - "bcrypt": { - "hashes": [ - "sha256:0258f143f3de96b7c14f762c770f5fc56ccd72f8a1857a451c1cd9a655d9ac89", - "sha256:0b0069c752ec14172c5f78208f1863d7ad6755a6fae6fe76ec2c80d13be41e42", - "sha256:19a4b72a6ae5bb467fea018b825f0a7d917789bcfe893e53f15c92805d187294", - "sha256:5432dd7b34107ae8ed6c10a71b4397f1c853bd39a4d6ffa7e35f40584cffd161", - "sha256:69361315039878c0680be456640f8705d76cb4a3a3fe1e057e0f261b74be4b31", - "sha256:6fe49a60b25b584e2f4ef175b29d3a83ba63b3a4df1b4c0605b826668d1b6be5", - "sha256:74a015102e877d0ccd02cdeaa18b32aa7273746914a6c5d0456dd442cb65b99c", - "sha256:763669a367869786bb4c8fcf731f4175775a5b43f070f50f46f0b59da45375d0", - "sha256:8b10acde4e1919d6015e1df86d4c217d3b5b01bb7744c36113ea43d529e1c3de", - "sha256:9fe92406c857409b70a38729dbdf6578caf9228de0aef5bc44f859ffe971a39e", - "sha256:a190f2a5dbbdbff4b74e3103cef44344bc30e61255beb27310e2aec407766052", - "sha256:a595c12c618119255c90deb4b046e1ca3bcfad64667c43d1166f2b04bc72db09", - "sha256:c9457fa5c121e94a58d6505cadca8bed1c64444b83b3204928a866ca2e599105", - "sha256:cb93f6b2ab0f6853550b74e051d297c27a638719753eb9ff66d1e4072be67133", - "sha256:d7bdc26475679dd073ba0ed2766445bb5b20ca4793ca0db32b399dccc6bc84b7", - "sha256:ff032765bb8716d9387fd5376d987a937254b0619eff0972779515b5c98820bc" - ], - "version": "==3.1.7" - }, "boto3": { "hashes": [ "sha256:60e711f1113be926bcec1cfe62fa336438d021ce834f4a5228beead3b4bc5142", @@ -176,43 +155,12 @@ ], "version": "==1.1.1" }, - "paramiko": { - "hashes": [ - "sha256:99f0179bdc176281d21961a003ffdb2ec369daac1a1007241f53374e376576cf", - "sha256:f4b2edfa0d226b70bd4ca31ea7e389325990283da23465d572ed1f70a7583041" - ], - "version": "==2.6.0" - }, "pycparser": { "hashes": [ "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" ], "version": "==2.19" }, - "pynacl": { - "hashes": [ - "sha256:05c26f93964373fc0abe332676cb6735f0ecad27711035b9472751faa8521255", - "sha256:0c6100edd16fefd1557da078c7a31e7b7d7a52ce39fdca2bec29d4f7b6e7600c", - "sha256:0d0a8171a68edf51add1e73d2159c4bc19fc0718e79dec51166e940856c2f28e", - "sha256:1c780712b206317a746ace34c209b8c29dbfd841dfbc02aa27f2084dd3db77ae", - "sha256:2424c8b9f41aa65bbdbd7a64e73a7450ebb4aa9ddedc6a081e7afcc4c97f7621", - "sha256:2d23c04e8d709444220557ae48ed01f3f1086439f12dbf11976e849a4926db56", - "sha256:30f36a9c70450c7878053fa1344aca0145fd47d845270b43a7ee9192a051bf39", - "sha256:37aa336a317209f1bb099ad177fef0da45be36a2aa664507c5d72015f956c310", - "sha256:4943decfc5b905748f0756fdd99d4f9498d7064815c4cf3643820c9028b711d1", - "sha256:57ef38a65056e7800859e5ba9e6091053cd06e1038983016effaffe0efcd594a", - "sha256:5bd61e9b44c543016ce1f6aef48606280e45f892a928ca7068fba30021e9b786", - "sha256:6482d3017a0c0327a49dddc8bd1074cc730d45db2ccb09c3bac1f8f32d1eb61b", - "sha256:7d3ce02c0784b7cbcc771a2da6ea51f87e8716004512493a2b69016326301c3b", - "sha256:a14e499c0f5955dcc3991f785f3f8e2130ed504fa3a7f44009ff458ad6bdd17f", - "sha256:a39f54ccbcd2757d1d63b0ec00a00980c0b382c62865b61a505163943624ab20", - "sha256:aabb0c5232910a20eec8563503c153a8e78bbf5459490c49ab31f6adf3f3a415", - "sha256:bd4ecb473a96ad0f90c20acba4f0bf0df91a4e03a1f4dd6a4bdc9ca75aa3a715", - "sha256:e2da3c13307eac601f3de04887624939aca8ee3c9488a0bb0eca4fb9401fc6b1", - "sha256:f67814c38162f4deb31f68d590771a29d5ae3b1bd64b75cf232308e5c74777e0" - ], - "version": "==1.3.0" - }, "python-dateutil": { "hashes": [ "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", @@ -265,10 +213,10 @@ "develop": { "ansible": { "hashes": [ - "sha256:e7e6de461b7d07cb4d8b2dd2a32b231af7c56e6bf39b851024671aaa52fd377e" + "sha256:1e5ba829ca0602c55b33da399b06f99b135a34014b661d1c36d8892a1e2d3730" ], "index": "pypi", - "version": "==2.7.11" + "version": "==2.8.2" }, "ansible-lint": { "hashes": [ diff --git a/bin/ansible-with-atl-env b/bin/ansible-with-atl-env index 29d5fee..580b4d4 100755 --- a/bin/ansible-with-atl-env +++ b/bin/ansible-with-atl-env @@ -14,9 +14,8 @@ source $ENV_FILE set +a # Use Ansible from virtualenv if provided -pipenv run \ - ansible-playbook -v \ - $ATL_DEPLOYMENT_REPOSITORY_CUSTOM_PARAMS \ - -i $INV \ - $PLAYBOOK \ - 2>&1 | tee --append $LOG_FILE +ansible-playbook -v \ + $ATL_DEPLOYMENT_REPOSITORY_CUSTOM_PARAMS \ + -i $INV \ + $PLAYBOOK \ + 2>&1 | tee --append $LOG_FILE diff --git a/bin/install-ansible b/bin/install-ansible index 8a32ac5..ac95ed8 100755 --- a/bin/install-ansible +++ b/bin/install-ansible @@ -2,24 +2,13 @@ set -e -# The Amazon Linux 2 Ansible package is 2.4, which has issue -# interacting with RDS, so use pipenv to install a known-good version. -# Another alternative here would be nix, however that has issues -# installing as root, and can be slow in practice. - -# Luckily AmazonLinux2 and Ubuntu use the same package name for -# pip. This may need some logic if other distros are added. Note: -# Parsing /etc/os-release is probably a good starting point for that. -# -# Additionally we need to install boto3 and botocore, as the Ansible -# AWS modules manage to escape the virtualenv and invoke the native python. +# Amazon Linux 2 packages Ansible separately, so enable the repo +. /etc/os-release +if [[ $ID == 'amzn' ]]; then + amazon-linux-extras enable ansible2 +fi ./bin/pacapt install --noconfirm \ - python-pip \ + ansible \ python-boto3 \ python-botocore -export PATH=$PATH:/usr/local/bin - -# See Pipfile and Pipfile.lock. -pip install pipenv -pipenv sync From 0c5f82c438608cdffa7ab96366c5a3c359fc08a8 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 3 Oct 2019 14:57:20 +1000 Subject: [PATCH 13/27] DCD-686: Move git version to where it is used. --- roles/linux_common/defaults/main.yml | 1 + roles/product_common/defaults/main.yml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/linux_common/defaults/main.yml b/roles/linux_common/defaults/main.yml index 561baf2..cb751b1 100644 --- a/roles/linux_common/defaults/main.yml +++ b/roles/linux_common/defaults/main.yml @@ -1,3 +1,4 @@ --- atl_product_user_uid: '2001' +git_version: "2.14.4" diff --git a/roles/product_common/defaults/main.yml b/roles/product_common/defaults/main.yml index cb807b0..ce62ba4 100644 --- a/roles/product_common/defaults/main.yml +++ b/roles/product_common/defaults/main.yml @@ -2,7 +2,6 @@ java_version: "1.8.0" java_major_version: "8" postgres_version: "9.6" -git_version: "2.14.4" # Disable these when using the product installer, otherwise we end up # fighting with it. From 2a0a1814c8facbdc791ab4aed9b8a2cc563efac8 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 3 Oct 2019 14:58:14 +1000 Subject: [PATCH 14/27] DCD-686: Remove molecule as the manifest fetch isn't really testable. --- .../molecule/default/Dockerfile.j2 | 14 ------------- .../molecule/default/molecule.yml | 20 ------------------- .../molecule/default/playbook.yml | 12 ----------- .../molecule/default/tests/test_default.py | 19 ------------------ 4 files changed, 65 deletions(-) delete mode 100644 roles/restore_metadata/molecule/default/Dockerfile.j2 delete mode 100644 roles/restore_metadata/molecule/default/molecule.yml delete mode 100644 roles/restore_metadata/molecule/default/playbook.yml delete mode 100644 roles/restore_metadata/molecule/default/tests/test_default.py diff --git a/roles/restore_metadata/molecule/default/Dockerfile.j2 b/roles/restore_metadata/molecule/default/Dockerfile.j2 deleted file mode 100644 index e6aa95d..0000000 --- a/roles/restore_metadata/molecule/default/Dockerfile.j2 +++ /dev/null @@ -1,14 +0,0 @@ -# Molecule managed - -{% if item.registry is defined %} -FROM {{ item.registry.url }}/{{ item.image }} -{% else %} -FROM {{ item.image }} -{% endif %} - -RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ - elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \ - elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ - elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \ - elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ - elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi diff --git a/roles/restore_metadata/molecule/default/molecule.yml b/roles/restore_metadata/molecule/default/molecule.yml deleted file mode 100644 index c6a6d26..0000000 --- a/roles/restore_metadata/molecule/default/molecule.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -dependency: - name: galaxy -driver: - name: docker -lint: - name: yamllint -platforms: - - name: amazon_linux2 - image: amazonlinux:2 - - name: ubuntu_lts - image: ubuntu:bionic -provisioner: - name: ansible - lint: - name: ansible-lint -verifier: - name: testinfra - lint: - name: flake8 diff --git a/roles/restore_metadata/molecule/default/playbook.yml b/roles/restore_metadata/molecule/default/playbook.yml deleted file mode 100644 index 6e24bad..0000000 --- a/roles/restore_metadata/molecule/default/playbook.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- name: Converge - hosts: all - vars: - atl_product_user: "testuser" - atl_product_home: "/opt/atlassian/product" - atl_product_installation_base: "/opt/atlassian/product/install" - atl_installer_temp: "/opt/atlassian/temp" - atl_product_home_shared: "/media/atl/jira/shared" - atl_product_shared_plugins: "/media/atl/jira/shared/plugins/" - roles: - - role: restore_metadata diff --git a/roles/restore_metadata/molecule/default/tests/test_default.py b/roles/restore_metadata/molecule/default/tests/test_default.py deleted file mode 100644 index b29ef83..0000000 --- a/roles/restore_metadata/molecule/default/tests/test_default.py +++ /dev/null @@ -1,19 +0,0 @@ -import os -import pytest - -import testinfra.utils.ansible_runner - -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') - - -def test_user_created(host): - user = host.user('testuser') - assert user.exists - - -@pytest.mark.parametrize('exe', [ - '/usr/bin/git' -]) -def test_package_exes(host, exe): - assert host.file(exe).exists From 3dca4691778840b24b6e20e811833a5e3dc440e1 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 3 Oct 2019 16:11:58 +1000 Subject: [PATCH 15/27] DCD-686: Load the downloaded manifest into a var. --- roles/restore_metadata/tasks/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/restore_metadata/tasks/main.yml b/roles/restore_metadata/tasks/main.yml index 5264854..477aa8d 100644 --- a/roles/restore_metadata/tasks/main.yml +++ b/roles/restore_metadata/tasks/main.yml @@ -35,4 +35,9 @@ dest: "{{ atl_backup_manifest_dest }}" when: atl_backup_manifest_url.scheme != 's3' + - name: Load parameters from manifest + include_vars: + file: "{{ atl_backup_manifest_dest }}" + name: atl_backup_manifest + when: atl_backup_manifest_url is defined and atl_backup_manifest_url != '' From 0f6bbe0f89149f78a912754e1ec7fb6cd186ae20 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Fri, 4 Oct 2019 08:19:33 +1000 Subject: [PATCH 16/27] DCD-686: Add comment describing role. --- roles/restore_metadata/tasks/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/restore_metadata/tasks/main.yml b/roles/restore_metadata/tasks/main.yml index 477aa8d..8a62e9d 100644 --- a/roles/restore_metadata/tasks/main.yml +++ b/roles/restore_metadata/tasks/main.yml @@ -1,5 +1,9 @@ --- +# This role will attempt to fetch and load the backup manifest from a +# remote HTTP or S3 URL. On successful completion the contents of JSON +# or YAML document will be in the var `atl_backup_manifest`. + - block: - name: Ensure temp directory is present From 6a940a718354688ee3c94bb2ad8b538905ab825c Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Fri, 4 Oct 2019 09:04:50 +1000 Subject: [PATCH 17/27] DCD-686: Rename manifest fetching role to be more descriptive. --- roles/{restore_metadata => load_backup_manifest}/tasks/main.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename roles/{restore_metadata => load_backup_manifest}/tasks/main.yml (100%) diff --git a/roles/restore_metadata/tasks/main.yml b/roles/load_backup_manifest/tasks/main.yml similarity index 100% rename from roles/restore_metadata/tasks/main.yml rename to roles/load_backup_manifest/tasks/main.yml From 5e85f71ae8580c011da703d38e15a04f85eed8c6 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Fri, 4 Oct 2019 11:39:43 +1000 Subject: [PATCH 18/27] DCD-686: Update pipelines config. --- bin/install-ansible | 9 ++ bitbucket-pipelines.yml | 86 +++++++------------ pipeline_generator/Makefile | 2 +- pipeline_generator/pipeline.py | 5 +- .../templates/bitbucket-pipelines.yml.j2 | 4 +- 5 files changed, 43 insertions(+), 63 deletions(-) diff --git a/bin/install-ansible b/bin/install-ansible index ac95ed8..f94539d 100755 --- a/bin/install-ansible +++ b/bin/install-ansible @@ -12,3 +12,12 @@ fi ansible \ python-boto3 \ python-botocore + +if [[ $1 == "--dev" ]]; then + ./bin/pacapt install --noconfirm \ + python-dev python-pip \ + python3-dev python3-pip + + pip install pipenv + pipenv sync --dev +fi diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index a63c197..7cccd33 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -1,6 +1,6 @@ --- -image: atlassian/default-image:2 +image: ubuntu:disco options: size: 2x @@ -32,8 +32,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 1 - step: @@ -41,8 +40,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 2 - step: @@ -50,8 +48,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 3 - step: @@ -59,8 +56,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 4 - step: @@ -68,8 +64,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 5 - step: @@ -77,8 +72,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 6 - step: @@ -86,8 +80,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 7 - step: @@ -95,8 +88,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 8 - step: @@ -104,8 +96,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 9 - step: @@ -113,8 +104,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 10 - step: @@ -122,8 +112,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 11 - step: @@ -131,8 +120,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 12 - step: @@ -140,8 +128,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 13 - step: @@ -149,8 +136,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 14 - step: @@ -158,8 +144,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 15 - step: @@ -167,8 +152,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 16 - step: @@ -176,8 +160,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 17 - step: @@ -185,8 +168,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 18 - step: @@ -194,8 +176,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 19 - step: @@ -203,8 +184,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 20 - step: @@ -212,8 +192,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 21 - step: @@ -221,8 +200,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 22 - step: @@ -230,8 +208,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 23 - step: @@ -239,8 +216,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 24 - step: @@ -248,8 +224,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 25 - step: @@ -257,8 +232,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 26 - step: @@ -266,8 +240,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 27 - step: @@ -275,8 +248,7 @@ pipelines: services: - docker script: - - apt-get update && apt-get install -y virtualenv python-dev - - ./bin/install-ansible + - ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 28 diff --git a/pipeline_generator/Makefile b/pipeline_generator/Makefile index 5b6e164..aed9280 100644 --- a/pipeline_generator/Makefile +++ b/pipeline_generator/Makefile @@ -1,2 +1,2 @@ generate-pipeline: - @python pipeline.py + @python3 pipeline.py diff --git a/pipeline_generator/pipeline.py b/pipeline_generator/pipeline.py index 62b1408..c225bbe 100644 --- a/pipeline_generator/pipeline.py +++ b/pipeline_generator/pipeline.py @@ -46,14 +46,13 @@ class Step: class ScriptCommand: - INSTALL_PACKAGES_COMMAND = "apt-get update && apt-get install -y virtualenv python-dev" - INSTALL_ANSIBLE_COMMAND = "./bin/install-ansible" + PACKAGE_INSTALL_COMMAND = "./bin/install-ansible --dev" def __init__(self, test_command): self.test_command = test_command def all_commands(self): - return [self.INSTALL_PACKAGES_COMMAND, self.INSTALL_ANSIBLE_COMMAND, self.test_command] + return [self.PACKAGE_INSTALL_COMMAND, self.test_command] def main(): diff --git a/pipeline_generator/templates/bitbucket-pipelines.yml.j2 b/pipeline_generator/templates/bitbucket-pipelines.yml.j2 index 9b7fe42..4432857 100644 --- a/pipeline_generator/templates/bitbucket-pipelines.yml.j2 +++ b/pipeline_generator/templates/bitbucket-pipelines.yml.j2 @@ -1,6 +1,6 @@ --- -image: atlassian/default-image:2 +image: ubuntu:disco options: size: 2x @@ -36,4 +36,4 @@ pipelines: {% for scriptCommand in parallel_step.scriptCommands -%} - {{ scriptCommand }} {% endfor %} - {% endfor %} \ No newline at end of file + {% endfor %} From b82ab6327a5e1c6aadc801324d39f01169822362 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Fri, 4 Oct 2019 11:43:19 +1000 Subject: [PATCH 19/27] DCD-686: Need to force update on Ubuntu. --- bitbucket-pipelines.yml | 56 +++++++++++++++++----------------- pipeline_generator/pipeline.py | 2 +- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 7cccd33..83f6b71 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -32,7 +32,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 1 - step: @@ -40,7 +40,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 2 - step: @@ -48,7 +48,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 3 - step: @@ -56,7 +56,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 4 - step: @@ -64,7 +64,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 5 - step: @@ -72,7 +72,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 6 - step: @@ -80,7 +80,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 7 - step: @@ -88,7 +88,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 8 - step: @@ -96,7 +96,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 9 - step: @@ -104,7 +104,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 10 - step: @@ -112,7 +112,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 11 - step: @@ -120,7 +120,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 12 - step: @@ -128,7 +128,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 13 - step: @@ -136,7 +136,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 14 - step: @@ -144,7 +144,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 15 - step: @@ -152,7 +152,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 16 - step: @@ -160,7 +160,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 17 - step: @@ -168,7 +168,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 18 - step: @@ -176,7 +176,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 19 - step: @@ -184,7 +184,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 20 - step: @@ -192,7 +192,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 21 - step: @@ -200,7 +200,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 22 - step: @@ -208,7 +208,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 23 - step: @@ -216,7 +216,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 24 - step: @@ -224,7 +224,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 25 - step: @@ -232,7 +232,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 26 - step: @@ -240,7 +240,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 27 - step: @@ -248,7 +248,7 @@ pipelines: services: - docker script: - - ./bin/install-ansible --dev + - apt-get update && ./bin/install-ansible --dev - ./bin/run-tests-in-batches --batch 28 diff --git a/pipeline_generator/pipeline.py b/pipeline_generator/pipeline.py index c225bbe..fd9edbb 100644 --- a/pipeline_generator/pipeline.py +++ b/pipeline_generator/pipeline.py @@ -46,7 +46,7 @@ class Step: class ScriptCommand: - PACKAGE_INSTALL_COMMAND = "./bin/install-ansible --dev" + PACKAGE_INSTALL_COMMAND = "apt-get update && ./bin/install-ansible --dev" def __init__(self, test_command): self.test_command = test_command From ba25ab8f73bf3b191cd2775df326cfd34e1dd35f Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Fri, 4 Oct 2019 11:57:54 +1000 Subject: [PATCH 20/27] DCD-686: Update to Debian Buster for better compability with Amazon Linux 2. --- Pipfile | 4 +- Pipfile.lock | 212 ++++++++++++------ bin/install-ansible | 3 +- bitbucket-pipelines.yml | 2 +- .../templates/bitbucket-pipelines.yml.j2 | 2 +- 5 files changed, 150 insertions(+), 73 deletions(-) diff --git a/Pipfile b/Pipfile index 55724ff..e8b700a 100644 --- a/Pipfile +++ b/Pipfile @@ -5,8 +5,6 @@ name = "pypi" [packages] ansible = "==2.8.2" -boto3 = "==1.9.241" -botocore = "==1.12.241" [dev-packages] molecule = "==2.20.1" @@ -16,4 +14,4 @@ taskcat = "*" Jinja2 = "*" [requires] -python_version = "3.7" +python_version = "2.7" diff --git a/Pipfile.lock b/Pipfile.lock index 4c43753..0508d9e 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "8f8923741e447b125ad9cb5c3912ba86e2239e753c7211ce1f074097892e1b6f" + "sha256": "32443872340b7d7f286060fb7ce8c930db8eb565aab1702fca75daaec230177a" }, "pipfile-spec": 6, "requires": { - "python_version": "3.7" + "python_version": "2.7" }, "sources": [ { @@ -30,22 +30,6 @@ ], "version": "==1.0.0" }, - "boto3": { - "hashes": [ - "sha256:60e711f1113be926bcec1cfe62fa336438d021ce834f4a5228beead3b4bc5142", - "sha256:8c9b9b2422c1baa84c0f331ee86ac4d265e1e7d321ce7ba58dbb863585c2191f" - ], - "index": "pypi", - "version": "==1.9.241" - }, - "botocore": { - "hashes": [ - "sha256:897415ec68b2cbb65a7d32965c456d332bb2eb936e533c9ad6064cd15e67c0c1", - "sha256:e35c2e6b8946be9063d7988b19dea2b6136b80c0e3469b6a076c574d5abca6b3" - ], - "index": "pypi", - "version": "==1.12.241" - }, "cffi": { "hashes": [ "sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774", @@ -100,13 +84,23 @@ ], "version": "==2.7" }, - "docutils": { + "enum34": { "hashes": [ - "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0", - "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827", - "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99" + "sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850", + "sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a", + "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", + "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" ], - "version": "==0.15.2" + "markers": "python_version < '3'", + "version": "==1.1.6" + }, + "ipaddress": { + "hashes": [ + "sha256:64b28eec5e78e7510698f6d4da08800a5c575caa4a286c93d651c5d3ff7b6794", + "sha256:b146c751ea45cad6188dd6cf2d9b757f6f4f8d6ffb96a023e6f2e26eea02a72c" + ], + "markers": "python_version < '3'", + "version": "==1.0.22" }, "jinja2": { "hashes": [ @@ -115,13 +109,6 @@ ], "version": "==2.10.1" }, - "jmespath": { - "hashes": [ - "sha256:3720a4b1bd659dd2eecad0666459b9788813e032b83e7ba58578e48254e0a0e6", - "sha256:bde2aef6f44302dfb30320115b17d030798de8c4110e28d5cf6cf91a7a31074c" - ], - "version": "==0.9.4" - }, "markupsafe": { "hashes": [ "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", @@ -161,14 +148,6 @@ ], "version": "==2.19" }, - "python-dateutil": { - "hashes": [ - "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", - "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" - ], - "markers": "python_version >= '2.7'", - "version": "==2.8.0" - }, "pyyaml": { "hashes": [ "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", @@ -187,27 +166,12 @@ ], "version": "==5.1.2" }, - "s3transfer": { - "hashes": [ - "sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d", - "sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba" - ], - "version": "==0.2.1" - }, "six": { "hashes": [ "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" ], "version": "==1.12.0" - }, - "urllib3": { - "hashes": [ - "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398", - "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86" - ], - "markers": "python_version >= '3.4'", - "version": "==1.25.6" } }, "develop": { @@ -261,9 +225,24 @@ }, "aws-sam-translator": { "hashes": [ - "sha256:3c615bff465fcf6a7990b9f84d002d55c75cd3e52d98e727d24959756ab0f0b1" + "sha256:6563aa3b534e7ad672d580ecd3dfa92021e81b4e5983604c0df7ee0a07b3ed99" ], - "version": "==1.14.0" + "version": "==1.15.0" + }, + "backports.functools-lru-cache": { + "hashes": [ + "sha256:9d98697f088eb1b0fa451391f91afb5e3ebde16bbdb272819fd091151fda4f1a", + "sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd" + ], + "markers": "python_version == '2.7'", + "version": "==1.5" + }, + "backports.ssl-match-hostname": { + "hashes": [ + "sha256:bb82e60f9fbf4c080eabd957c39f0641f0fc247d9a16e31e26d594d8f42b9fd2" + ], + "markers": "python_version < '3.5'", + "version": "==3.7.0.1" }, "binaryornot": { "hashes": [ @@ -274,19 +253,17 @@ }, "boto3": { "hashes": [ - "sha256:60e711f1113be926bcec1cfe62fa336438d021ce834f4a5228beead3b4bc5142", - "sha256:8c9b9b2422c1baa84c0f331ee86ac4d265e1e7d321ce7ba58dbb863585c2191f" + "sha256:4189e1ffed768bd0efd754a0abedebce19495ba2aa6b2f5e20f29ba80f81f9cb", + "sha256:fa4e28166922feeb9b7b56134c1acc817a1bca36284a0035bc08a3dab1853a9f" ], - "index": "pypi", - "version": "==1.9.241" + "version": "==1.9.242" }, "botocore": { "hashes": [ - "sha256:897415ec68b2cbb65a7d32965c456d332bb2eb936e533c9ad6064cd15e67c0c1", - "sha256:e35c2e6b8946be9063d7988b19dea2b6136b80c0e3469b6a076c574d5abca6b3" + "sha256:7af52e0aabaf4ba045e1a5832308e70e1ea4b499b71624857f09aed2ba5e667c", + "sha256:dd62d63bcd3176c92775c52d3e879288f89bf0ac0039df14ea31f25d693acd6d" ], - "index": "pypi", - "version": "==1.12.241" + "version": "==1.12.242" }, "cerberus": { "hashes": [ @@ -368,6 +345,22 @@ ], "version": "==0.3.9" }, + "configparser": { + "hashes": [ + "sha256:254c1d9c79f60c45dfde850850883d5aaa7f19a23f13561243a050d5a7c3fe4c", + "sha256:c7d282687a5308319bf3d2e7706e575c635b0a470342641c93bea0ea3b5331df" + ], + "markers": "python_version < '3'", + "version": "==4.0.2" + }, + "contextlib2": { + "hashes": [ + "sha256:7197aa736777caac513dbd800944c209a49765bf1979b12b037dce0277077ed3", + "sha256:9d2c67f18c1f9b6db1b46317f7f784aa82789d2ee5dea5d9c0f0f2a764eb862e" + ], + "markers": "python_version < '3'", + "version": "==0.6.0" + }, "cookiecutter": { "hashes": [ "sha256:1316a52e1c1f08db0c9efbf7d876dbc01463a74b155a0d83e722be88beda9a3e", @@ -419,6 +412,16 @@ ], "version": "==0.3" }, + "enum34": { + "hashes": [ + "sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850", + "sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a", + "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", + "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" + ], + "markers": "python_version < '3'", + "version": "==1.1.6" + }, "fasteners": { "hashes": [ "sha256:007e4d2b2d4a10093f67e932e5166722d2eab83b77724156e92ad013c6226574", @@ -433,12 +436,36 @@ ], "version": "==3.7.8" }, + "funcsigs": { + "hashes": [ + "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca", + "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50" + ], + "markers": "python_version < '3.3'", + "version": "==1.0.2" + }, + "functools32": { + "hashes": [ + "sha256:89d824aa6c358c421a234d7f9ee0bd75933a67c29588ce50aaa3acdf4d403fa0", + "sha256:f6253dfbe0538ad2e387bd8fdfd9293c925d63553f5813c4e587745416501e6d" + ], + "markers": "python_version < '3.2'", + "version": "==3.2.3.post2" + }, "future": { "hashes": [ "sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8" ], "version": "==0.17.1" }, + "futures": { + "hashes": [ + "sha256:49b3f5b064b6e3afc3316421a3f25f66c137ae88f068abbf72830170033c5e16", + "sha256:7e033af76a5e35f58e56da7a91e687706faf4e7bdfb2cbc3f2cca6b9bcda9794" + ], + "markers": "python_version == '2.6' or python_version == '2.7'", + "version": "==3.3.0" + }, "git-url-parse": { "hashes": [ "sha256:4655ee22f1d8bf7a1eb1066c1da16529b186966c6d8331f7f55686a76a9f7aef", @@ -462,6 +489,14 @@ "markers": "python_version < '3.8'", "version": "==0.23" }, + "ipaddress": { + "hashes": [ + "sha256:64b28eec5e78e7510698f6d4da08800a5c575caa4a286c93d651c5d3ff7b6794", + "sha256:b146c751ea45cad6188dd6cf2d9b757f6f4f8d6ffb96a023e6f2e26eea02a72c" + ], + "markers": "python_version < '3'", + "version": "==1.0.22" + }, "jinja2": { "hashes": [ "sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013", @@ -580,11 +615,19 @@ ], "version": "==19.2" }, + "pathlib2": { + "hashes": [ + "sha256:0ec8205a157c80d7acc301c0b18fbd5d44fe655968f5d947b6ecef5290fc35db", + "sha256:6cd9a47b597b37cc57de1c05e56fb1a1c9cc9fab04fe78c29acd090418529868" + ], + "markers": "python_version == '3.4.*' or python_version < '3'", + "version": "==2.3.5" + }, "pathspec": { "hashes": [ - "sha256:54a5eab895d89f342b52ba2bffe70930ef9f8d96e398cccf530d21fa0516a873" + "sha256:e285ccc8b0785beadd4c18e5708b12bb8fcf529a1e61215b3feff1d1e559ea5c" ], - "version": "==0.5.9" + "version": "==0.6.0" }, "pbr": { "hashes": [ @@ -725,6 +768,18 @@ ], "version": "==2.22.0" }, + "ruamel.ordereddict": { + "hashes": [ + "sha256:281051d26eb2b18ef3d920e1e260716a52bd058a6b1a2f324102fc6a15cb8d4a", + "sha256:36fe0af3a02a0e1199447d050e6c3a1f5bd4c7d68e4c260f6a7a058fb4da71cb", + "sha256:4375a70d5d217069a8349bf5fbc27aa4cf1aedfbf03ce94df113b75d22d1a1e2", + "sha256:4cd0ec38dac57a4054dda14b0a5eea1a877dcc73106131ef08513fb89ba95a22", + "sha256:4f641c4de9082866b9e88497ad8050dca38c5ddbb8cb7ae9316da9db257092b2", + "sha256:7324310945c6b47218255b5d75ccbc74d435221c44652ec4406b1a871ddc3bc3" + ], + "markers": "platform_python_implementation == 'CPython' and python_version <= '2.7'", + "version": "==0.4.14" + }, "ruamel.yaml": { "hashes": [ "sha256:0db639b1b2742dae666c6fc009b8d1931ef15c9276ef31c0673cc6dcf766cf40", @@ -763,6 +818,23 @@ ], "version": "==0.2.1" }, + "scandir": { + "hashes": [ + "sha256:2586c94e907d99617887daed6c1d102b5ca28f1085f90446554abf1faf73123e", + "sha256:2ae41f43797ca0c11591c0c35f2f5875fa99f8797cb1a1fd440497ec0ae4b022", + "sha256:2b8e3888b11abb2217a32af0766bc06b65cc4a928d8727828ee68af5a967fa6f", + "sha256:2c712840c2e2ee8dfaf36034080108d30060d759c7b73a01a52251cc8989f11f", + "sha256:4d4631f6062e658e9007ab3149a9b914f3548cb38bfb021c64f39a025ce578ae", + "sha256:67f15b6f83e6507fdc6fca22fedf6ef8b334b399ca27c6b568cbfaa82a364173", + "sha256:7d2d7a06a252764061a020407b997dd036f7bd6a175a5ba2b345f0a357f0b3f4", + "sha256:8c5922863e44ffc00c5c693190648daa6d15e7c1207ed02d6f46a8dcc2869d32", + "sha256:92c85ac42f41ffdc35b6da57ed991575bdbe69db895507af88b9f499b701c188", + "sha256:b24086f2375c4a094a6b51e78b4cf7ca16c721dcee2eddd7aa6494b42d6d519d", + "sha256:cb925555f43060a1745d0a321cca94bcea927c50114b623d73179189a4e100ac" + ], + "markers": "python_version < '3.5'", + "version": "==1.10.0" + }, "sh": { "hashes": [ "sha256:ae3258c5249493cebe73cb4e18253a41ed69262484bad36fdb3efcb8ad8870bb", @@ -804,12 +876,20 @@ ], "version": "==0.1.2" }, + "typing": { + "hashes": [ + "sha256:91dfe6f3f706ee8cc32d38edbbf304e9b7583fb37108fef38229617f8b3eba23", + "sha256:c8cabb5ab8945cd2f54917be357d134db9cc1eb039e59d1606dc1e60cb1d9d36", + "sha256:f38d83c5a7a7086543a0f649564d661859c5146a85775ab90c0d2f93ffaa9714" + ], + "markers": "python_version < '3.5'", + "version": "==3.7.4.1" + }, "urllib3": { "hashes": [ "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398", "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86" ], - "markers": "python_version >= '3.4'", "version": "==1.25.6" }, "wcwidth": { diff --git a/bin/install-ansible b/bin/install-ansible index f94539d..4654763 100755 --- a/bin/install-ansible +++ b/bin/install-ansible @@ -15,8 +15,7 @@ fi if [[ $1 == "--dev" ]]; then ./bin/pacapt install --noconfirm \ - python-dev python-pip \ - python3-dev python3-pip + python-dev python-pip pip install pipenv pipenv sync --dev diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 83f6b71..ec993c6 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -1,6 +1,6 @@ --- -image: ubuntu:disco +image: debian:buster options: size: 2x diff --git a/pipeline_generator/templates/bitbucket-pipelines.yml.j2 b/pipeline_generator/templates/bitbucket-pipelines.yml.j2 index 4432857..6b89e62 100644 --- a/pipeline_generator/templates/bitbucket-pipelines.yml.j2 +++ b/pipeline_generator/templates/bitbucket-pipelines.yml.j2 @@ -1,6 +1,6 @@ --- -image: ubuntu:disco +image: debian:buster options: size: 2x From 60b823cd02dac1c4fb791225a84c74f80145281b Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Fri, 4 Oct 2019 12:10:07 +1000 Subject: [PATCH 21/27] DCD-686: Try with python3. --- Pipfile | 2 +- Pipfile.lock | 142 +------------------------------------------- bin/install-ansible | 4 +- 3 files changed, 5 insertions(+), 143 deletions(-) diff --git a/Pipfile b/Pipfile index e8b700a..766c833 100644 --- a/Pipfile +++ b/Pipfile @@ -14,4 +14,4 @@ taskcat = "*" Jinja2 = "*" [requires] -python_version = "2.7" +python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock index 0508d9e..6a3e5b4 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "32443872340b7d7f286060fb7ce8c930db8eb565aab1702fca75daaec230177a" + "sha256": "dae68e0cb0d94bd8016536c7ed22b2e4cf8292aafd988c421aa31a851763a83e" }, "pipfile-spec": 6, "requires": { - "python_version": "2.7" + "python_version": "3.7" }, "sources": [ { @@ -84,24 +84,6 @@ ], "version": "==2.7" }, - "enum34": { - "hashes": [ - "sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850", - "sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a", - "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", - "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" - ], - "markers": "python_version < '3'", - "version": "==1.1.6" - }, - "ipaddress": { - "hashes": [ - "sha256:64b28eec5e78e7510698f6d4da08800a5c575caa4a286c93d651c5d3ff7b6794", - "sha256:b146c751ea45cad6188dd6cf2d9b757f6f4f8d6ffb96a023e6f2e26eea02a72c" - ], - "markers": "python_version < '3'", - "version": "==1.0.22" - }, "jinja2": { "hashes": [ "sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013", @@ -229,21 +211,6 @@ ], "version": "==1.15.0" }, - "backports.functools-lru-cache": { - "hashes": [ - "sha256:9d98697f088eb1b0fa451391f91afb5e3ebde16bbdb272819fd091151fda4f1a", - "sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd" - ], - "markers": "python_version == '2.7'", - "version": "==1.5" - }, - "backports.ssl-match-hostname": { - "hashes": [ - "sha256:bb82e60f9fbf4c080eabd957c39f0641f0fc247d9a16e31e26d594d8f42b9fd2" - ], - "markers": "python_version < '3.5'", - "version": "==3.7.0.1" - }, "binaryornot": { "hashes": [ "sha256:359501dfc9d40632edc9fac890e19542db1a287bbcfa58175b66658392018061", @@ -345,22 +312,6 @@ ], "version": "==0.3.9" }, - "configparser": { - "hashes": [ - "sha256:254c1d9c79f60c45dfde850850883d5aaa7f19a23f13561243a050d5a7c3fe4c", - "sha256:c7d282687a5308319bf3d2e7706e575c635b0a470342641c93bea0ea3b5331df" - ], - "markers": "python_version < '3'", - "version": "==4.0.2" - }, - "contextlib2": { - "hashes": [ - "sha256:7197aa736777caac513dbd800944c209a49765bf1979b12b037dce0277077ed3", - "sha256:9d2c67f18c1f9b6db1b46317f7f784aa82789d2ee5dea5d9c0f0f2a764eb862e" - ], - "markers": "python_version < '3'", - "version": "==0.6.0" - }, "cookiecutter": { "hashes": [ "sha256:1316a52e1c1f08db0c9efbf7d876dbc01463a74b155a0d83e722be88beda9a3e", @@ -412,16 +363,6 @@ ], "version": "==0.3" }, - "enum34": { - "hashes": [ - "sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850", - "sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a", - "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", - "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" - ], - "markers": "python_version < '3'", - "version": "==1.1.6" - }, "fasteners": { "hashes": [ "sha256:007e4d2b2d4a10093f67e932e5166722d2eab83b77724156e92ad013c6226574", @@ -436,36 +377,12 @@ ], "version": "==3.7.8" }, - "funcsigs": { - "hashes": [ - "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca", - "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50" - ], - "markers": "python_version < '3.3'", - "version": "==1.0.2" - }, - "functools32": { - "hashes": [ - "sha256:89d824aa6c358c421a234d7f9ee0bd75933a67c29588ce50aaa3acdf4d403fa0", - "sha256:f6253dfbe0538ad2e387bd8fdfd9293c925d63553f5813c4e587745416501e6d" - ], - "markers": "python_version < '3.2'", - "version": "==3.2.3.post2" - }, "future": { "hashes": [ "sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8" ], "version": "==0.17.1" }, - "futures": { - "hashes": [ - "sha256:49b3f5b064b6e3afc3316421a3f25f66c137ae88f068abbf72830170033c5e16", - "sha256:7e033af76a5e35f58e56da7a91e687706faf4e7bdfb2cbc3f2cca6b9bcda9794" - ], - "markers": "python_version == '2.6' or python_version == '2.7'", - "version": "==3.3.0" - }, "git-url-parse": { "hashes": [ "sha256:4655ee22f1d8bf7a1eb1066c1da16529b186966c6d8331f7f55686a76a9f7aef", @@ -489,14 +406,6 @@ "markers": "python_version < '3.8'", "version": "==0.23" }, - "ipaddress": { - "hashes": [ - "sha256:64b28eec5e78e7510698f6d4da08800a5c575caa4a286c93d651c5d3ff7b6794", - "sha256:b146c751ea45cad6188dd6cf2d9b757f6f4f8d6ffb96a023e6f2e26eea02a72c" - ], - "markers": "python_version < '3'", - "version": "==1.0.22" - }, "jinja2": { "hashes": [ "sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013", @@ -615,14 +524,6 @@ ], "version": "==19.2" }, - "pathlib2": { - "hashes": [ - "sha256:0ec8205a157c80d7acc301c0b18fbd5d44fe655968f5d947b6ecef5290fc35db", - "sha256:6cd9a47b597b37cc57de1c05e56fb1a1c9cc9fab04fe78c29acd090418529868" - ], - "markers": "python_version == '3.4.*' or python_version < '3'", - "version": "==2.3.5" - }, "pathspec": { "hashes": [ "sha256:e285ccc8b0785beadd4c18e5708b12bb8fcf529a1e61215b3feff1d1e559ea5c" @@ -732,7 +633,6 @@ "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" ], - "markers": "python_version >= '2.7'", "version": "==2.8.0" }, "python-gilt": { @@ -768,18 +668,6 @@ ], "version": "==2.22.0" }, - "ruamel.ordereddict": { - "hashes": [ - "sha256:281051d26eb2b18ef3d920e1e260716a52bd058a6b1a2f324102fc6a15cb8d4a", - "sha256:36fe0af3a02a0e1199447d050e6c3a1f5bd4c7d68e4c260f6a7a058fb4da71cb", - "sha256:4375a70d5d217069a8349bf5fbc27aa4cf1aedfbf03ce94df113b75d22d1a1e2", - "sha256:4cd0ec38dac57a4054dda14b0a5eea1a877dcc73106131ef08513fb89ba95a22", - "sha256:4f641c4de9082866b9e88497ad8050dca38c5ddbb8cb7ae9316da9db257092b2", - "sha256:7324310945c6b47218255b5d75ccbc74d435221c44652ec4406b1a871ddc3bc3" - ], - "markers": "platform_python_implementation == 'CPython' and python_version <= '2.7'", - "version": "==0.4.14" - }, "ruamel.yaml": { "hashes": [ "sha256:0db639b1b2742dae666c6fc009b8d1931ef15c9276ef31c0673cc6dcf766cf40", @@ -818,23 +706,6 @@ ], "version": "==0.2.1" }, - "scandir": { - "hashes": [ - "sha256:2586c94e907d99617887daed6c1d102b5ca28f1085f90446554abf1faf73123e", - "sha256:2ae41f43797ca0c11591c0c35f2f5875fa99f8797cb1a1fd440497ec0ae4b022", - "sha256:2b8e3888b11abb2217a32af0766bc06b65cc4a928d8727828ee68af5a967fa6f", - "sha256:2c712840c2e2ee8dfaf36034080108d30060d759c7b73a01a52251cc8989f11f", - "sha256:4d4631f6062e658e9007ab3149a9b914f3548cb38bfb021c64f39a025ce578ae", - "sha256:67f15b6f83e6507fdc6fca22fedf6ef8b334b399ca27c6b568cbfaa82a364173", - "sha256:7d2d7a06a252764061a020407b997dd036f7bd6a175a5ba2b345f0a357f0b3f4", - "sha256:8c5922863e44ffc00c5c693190648daa6d15e7c1207ed02d6f46a8dcc2869d32", - "sha256:92c85ac42f41ffdc35b6da57ed991575bdbe69db895507af88b9f499b701c188", - "sha256:b24086f2375c4a094a6b51e78b4cf7ca16c721dcee2eddd7aa6494b42d6d519d", - "sha256:cb925555f43060a1745d0a321cca94bcea927c50114b623d73179189a4e100ac" - ], - "markers": "python_version < '3.5'", - "version": "==1.10.0" - }, "sh": { "hashes": [ "sha256:ae3258c5249493cebe73cb4e18253a41ed69262484bad36fdb3efcb8ad8870bb", @@ -876,15 +747,6 @@ ], "version": "==0.1.2" }, - "typing": { - "hashes": [ - "sha256:91dfe6f3f706ee8cc32d38edbbf304e9b7583fb37108fef38229617f8b3eba23", - "sha256:c8cabb5ab8945cd2f54917be357d134db9cc1eb039e59d1606dc1e60cb1d9d36", - "sha256:f38d83c5a7a7086543a0f649564d661859c5146a85775ab90c0d2f93ffaa9714" - ], - "markers": "python_version < '3.5'", - "version": "==3.7.4.1" - }, "urllib3": { "hashes": [ "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398", diff --git a/bin/install-ansible b/bin/install-ansible index 4654763..953663f 100755 --- a/bin/install-ansible +++ b/bin/install-ansible @@ -15,8 +15,8 @@ fi if [[ $1 == "--dev" ]]; then ./bin/pacapt install --noconfirm \ - python-dev python-pip + python3-dev python3-pip - pip install pipenv + pip3 install pipenv pipenv sync --dev fi From 0209ad22b7aaf0941c195845510c48f0343c2748 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Fri, 4 Oct 2019 12:24:59 +1000 Subject: [PATCH 22/27] DCD-686: Another combination of dev dependencies to work around compatability issues. --- Pipfile | 6 ++-- Pipfile.lock | 82 +++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 71 insertions(+), 17 deletions(-) diff --git a/Pipfile b/Pipfile index 766c833..c0d22ba 100644 --- a/Pipfile +++ b/Pipfile @@ -4,11 +4,11 @@ verify_ssl = true name = "pypi" [packages] -ansible = "==2.8.2" +ansible = "==2.7.11" [dev-packages] -molecule = "==2.20.1" -docker = "==4.0.1" +molecule = "==2.20.2" +docker = "==4.1.0" six = "*" taskcat = "*" Jinja2 = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 6a3e5b4..d5ce753 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "dae68e0cb0d94bd8016536c7ed22b2e4cf8292aafd988c421aa31a851763a83e" + "sha256": "3e6fecddc35743d370fbdbd68b57b6edc588fe026f58e67e6f6343c7dafc2ee6" }, "pipfile-spec": 6, "requires": { @@ -18,10 +18,10 @@ "default": { "ansible": { "hashes": [ - "sha256:1e5ba829ca0602c55b33da399b06f99b135a34014b661d1c36d8892a1e2d3730" + "sha256:e7e6de461b7d07cb4d8b2dd2a32b231af7c56e6bf39b851024671aaa52fd377e" ], "index": "pypi", - "version": "==2.8.2" + "version": "==2.7.11" }, "asn1crypto": { "hashes": [ @@ -30,6 +30,27 @@ ], "version": "==1.0.0" }, + "bcrypt": { + "hashes": [ + "sha256:0258f143f3de96b7c14f762c770f5fc56ccd72f8a1857a451c1cd9a655d9ac89", + "sha256:0b0069c752ec14172c5f78208f1863d7ad6755a6fae6fe76ec2c80d13be41e42", + "sha256:19a4b72a6ae5bb467fea018b825f0a7d917789bcfe893e53f15c92805d187294", + "sha256:5432dd7b34107ae8ed6c10a71b4397f1c853bd39a4d6ffa7e35f40584cffd161", + "sha256:69361315039878c0680be456640f8705d76cb4a3a3fe1e057e0f261b74be4b31", + "sha256:6fe49a60b25b584e2f4ef175b29d3a83ba63b3a4df1b4c0605b826668d1b6be5", + "sha256:74a015102e877d0ccd02cdeaa18b32aa7273746914a6c5d0456dd442cb65b99c", + "sha256:763669a367869786bb4c8fcf731f4175775a5b43f070f50f46f0b59da45375d0", + "sha256:8b10acde4e1919d6015e1df86d4c217d3b5b01bb7744c36113ea43d529e1c3de", + "sha256:9fe92406c857409b70a38729dbdf6578caf9228de0aef5bc44f859ffe971a39e", + "sha256:a190f2a5dbbdbff4b74e3103cef44344bc30e61255beb27310e2aec407766052", + "sha256:a595c12c618119255c90deb4b046e1ca3bcfad64667c43d1166f2b04bc72db09", + "sha256:c9457fa5c121e94a58d6505cadca8bed1c64444b83b3204928a866ca2e599105", + "sha256:cb93f6b2ab0f6853550b74e051d297c27a638719753eb9ff66d1e4072be67133", + "sha256:d7bdc26475679dd073ba0ed2766445bb5b20ca4793ca0db32b399dccc6bc84b7", + "sha256:ff032765bb8716d9387fd5376d987a937254b0619eff0972779515b5c98820bc" + ], + "version": "==3.1.7" + }, "cffi": { "hashes": [ "sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774", @@ -124,12 +145,43 @@ ], "version": "==1.1.1" }, + "paramiko": { + "hashes": [ + "sha256:99f0179bdc176281d21961a003ffdb2ec369daac1a1007241f53374e376576cf", + "sha256:f4b2edfa0d226b70bd4ca31ea7e389325990283da23465d572ed1f70a7583041" + ], + "version": "==2.6.0" + }, "pycparser": { "hashes": [ "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" ], "version": "==2.19" }, + "pynacl": { + "hashes": [ + "sha256:05c26f93964373fc0abe332676cb6735f0ecad27711035b9472751faa8521255", + "sha256:0c6100edd16fefd1557da078c7a31e7b7d7a52ce39fdca2bec29d4f7b6e7600c", + "sha256:0d0a8171a68edf51add1e73d2159c4bc19fc0718e79dec51166e940856c2f28e", + "sha256:1c780712b206317a746ace34c209b8c29dbfd841dfbc02aa27f2084dd3db77ae", + "sha256:2424c8b9f41aa65bbdbd7a64e73a7450ebb4aa9ddedc6a081e7afcc4c97f7621", + "sha256:2d23c04e8d709444220557ae48ed01f3f1086439f12dbf11976e849a4926db56", + "sha256:30f36a9c70450c7878053fa1344aca0145fd47d845270b43a7ee9192a051bf39", + "sha256:37aa336a317209f1bb099ad177fef0da45be36a2aa664507c5d72015f956c310", + "sha256:4943decfc5b905748f0756fdd99d4f9498d7064815c4cf3643820c9028b711d1", + "sha256:57ef38a65056e7800859e5ba9e6091053cd06e1038983016effaffe0efcd594a", + "sha256:5bd61e9b44c543016ce1f6aef48606280e45f892a928ca7068fba30021e9b786", + "sha256:6482d3017a0c0327a49dddc8bd1074cc730d45db2ccb09c3bac1f8f32d1eb61b", + "sha256:7d3ce02c0784b7cbcc771a2da6ea51f87e8716004512493a2b69016326301c3b", + "sha256:a14e499c0f5955dcc3991f785f3f8e2130ed504fa3a7f44009ff458ad6bdd17f", + "sha256:a39f54ccbcd2757d1d63b0ec00a00980c0b382c62865b61a505163943624ab20", + "sha256:aabb0c5232910a20eec8563503c153a8e78bbf5459490c49ab31f6adf3f3a415", + "sha256:bd4ecb473a96ad0f90c20acba4f0bf0df91a4e03a1f4dd6a4bdc9ca75aa3a715", + "sha256:e2da3c13307eac601f3de04887624939aca8ee3c9488a0bb0eca4fb9401fc6b1", + "sha256:f67814c38162f4deb31f68d590771a29d5ae3b1bd64b75cf232308e5c74777e0" + ], + "version": "==1.3.0" + }, "pyyaml": { "hashes": [ "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", @@ -159,10 +211,10 @@ "develop": { "ansible": { "hashes": [ - "sha256:1e5ba829ca0602c55b33da399b06f99b135a34014b661d1c36d8892a1e2d3730" + "sha256:e7e6de461b7d07cb4d8b2dd2a32b231af7c56e6bf39b851024671aaa52fd377e" ], "index": "pypi", - "version": "==2.8.2" + "version": "==2.7.11" }, "ansible-lint": { "hashes": [ @@ -342,11 +394,11 @@ }, "docker": { "hashes": [ - "sha256:3db499d4d25847fed86acf8e100c989f7bc0f75a6fff6c52855726ada1d124f6", - "sha256:f61c37d721b489b7d55ef631b241be2d6a5884c3ffe63dc8f7dd9a3c3cd60489" + "sha256:6e06c5e70ba4fad73e35f00c55a895a448398f3ada7faae072e2bb01348bafc1", + "sha256:8f93775b8bdae3a2df6bc9a5312cce564cade58d6555f2c2570165a1270cd8a7" ], "index": "pypi", - "version": "==4.0.1" + "version": "==4.1.0" }, "docutils": { "hashes": [ @@ -497,11 +549,11 @@ }, "molecule": { "hashes": [ - "sha256:0e9ef6845cdf2a01f6c386445e4e54add3f515a033ee16b7b658e6122c8f0d76", - "sha256:621797c54299775f284bbb010d5bb9be485500eecaaa14a476cbc0df285d0da7" + "sha256:5fa56e52602364716dd5aa55e1dd70400f2094b8cc3c458869e5382e84149065", + "sha256:9dc29b9ef172b26532752784687faca2e868c84e2d90f0b4f018d81d76a8b30a" ], "index": "pypi", - "version": "==2.20.1" + "version": "==2.20.2" }, "monotonic": { "hashes": [ @@ -633,6 +685,7 @@ "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" ], + "markers": "python_version >= '2.7'", "version": "==2.8.0" }, "python-gilt": { @@ -735,10 +788,10 @@ }, "testinfra": { "hashes": [ - "sha256:8dbbf25039674d419598f576c5652947cebdf7cbbea8f23acacc80271009c6cb", - "sha256:d13dda899d5a051465f041a821363e2ebdd079391fbeae04089a2df7d35e3d54" + "sha256:16201d64659ec0c2d25f65d6ce1f5367668b7b4eb102450efd4f8983a399d7d0", + "sha256:5cebf61fee13c2e83b5e177431e751e243fc779293377c5e0c3b43910bb7e870" ], - "version": "==1.19.0" + "version": "==3.2.0" }, "tree-format": { "hashes": [ @@ -752,6 +805,7 @@ "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398", "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86" ], + "markers": "python_version >= '3.4'", "version": "==1.25.6" }, "wcwidth": { From 606ac960d90ec57ca39a441a910c4b921125720f Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Fri, 4 Oct 2019 13:35:17 +1000 Subject: [PATCH 23/27] DCD-686: Ansible 2.8 added a timeout to the yum module which broke everything. --- roles/aws_common/tasks/amazon.yml | 1 + roles/linux_common/tasks/amazon.yml | 1 + roles/nfs_server/tasks/amazon.yml | 1 + roles/nfs_server/tasks/ubuntu.yml | 2 +- roles/product_common/tasks/amazon.yml | 2 ++ 5 files changed, 6 insertions(+), 1 deletion(-) diff --git a/roles/aws_common/tasks/amazon.yml b/roles/aws_common/tasks/amazon.yml index a6592bf..f19485a 100644 --- a/roles/aws_common/tasks/amazon.yml +++ b/roles/aws_common/tasks/amazon.yml @@ -6,6 +6,7 @@ - ec2-utils - amazon-ssm-agent - amazon-efs-utils + lock_timeout: 30 - name: Install CloudWatch Agent yum: diff --git a/roles/linux_common/tasks/amazon.yml b/roles/linux_common/tasks/amazon.yml index 3be04db..ad70b98 100644 --- a/roles/linux_common/tasks/amazon.yml +++ b/roles/linux_common/tasks/amazon.yml @@ -6,3 +6,4 @@ - shadow-utils - libxml2 - git-{{ git_version }} + lock_timeout: 30 diff --git a/roles/nfs_server/tasks/amazon.yml b/roles/nfs_server/tasks/amazon.yml index ad2adee..67b71d0 100644 --- a/roles/nfs_server/tasks/amazon.yml +++ b/roles/nfs_server/tasks/amazon.yml @@ -4,3 +4,4 @@ yum: name: - nfs-utils + lock_timeout: 30 diff --git a/roles/nfs_server/tasks/ubuntu.yml b/roles/nfs_server/tasks/ubuntu.yml index 5bb5dcb..becb1d8 100644 --- a/roles/nfs_server/tasks/ubuntu.yml +++ b/roles/nfs_server/tasks/ubuntu.yml @@ -1,7 +1,7 @@ --- - name: Install Ubuntu-specific NFS packages - yum: + apt: name: - nfs-kernel-server - libnfs-utils diff --git a/roles/product_common/tasks/amazon.yml b/roles/product_common/tasks/amazon.yml index c02f864..0175922 100644 --- a/roles/product_common/tasks/amazon.yml +++ b/roles/product_common/tasks/amazon.yml @@ -4,9 +4,11 @@ yum: name: - java-{{ java_version }}-openjdk-devel + lock_timeout: 30 when: atl_use_system_jdk - name: Install other base packages on Amazon Linux yum: name: - dejavu-fonts-common # Required by the installer + lock_timeout: 30 From bbff7f94bb34bbdc4eddbbecc59297413d802ed8 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Fri, 4 Oct 2019 13:49:43 +1000 Subject: [PATCH 24/27] DCD-686: Revert lockfiles as we should move to fine control over Ansible versions (again). --- roles/aws_common/tasks/amazon.yml | 1 - roles/linux_common/tasks/amazon.yml | 1 - roles/nfs_server/tasks/amazon.yml | 1 - roles/product_common/tasks/amazon.yml | 2 -- 4 files changed, 5 deletions(-) diff --git a/roles/aws_common/tasks/amazon.yml b/roles/aws_common/tasks/amazon.yml index f19485a..a6592bf 100644 --- a/roles/aws_common/tasks/amazon.yml +++ b/roles/aws_common/tasks/amazon.yml @@ -6,7 +6,6 @@ - ec2-utils - amazon-ssm-agent - amazon-efs-utils - lock_timeout: 30 - name: Install CloudWatch Agent yum: diff --git a/roles/linux_common/tasks/amazon.yml b/roles/linux_common/tasks/amazon.yml index ad70b98..3be04db 100644 --- a/roles/linux_common/tasks/amazon.yml +++ b/roles/linux_common/tasks/amazon.yml @@ -6,4 +6,3 @@ - shadow-utils - libxml2 - git-{{ git_version }} - lock_timeout: 30 diff --git a/roles/nfs_server/tasks/amazon.yml b/roles/nfs_server/tasks/amazon.yml index 67b71d0..ad2adee 100644 --- a/roles/nfs_server/tasks/amazon.yml +++ b/roles/nfs_server/tasks/amazon.yml @@ -4,4 +4,3 @@ yum: name: - nfs-utils - lock_timeout: 30 diff --git a/roles/product_common/tasks/amazon.yml b/roles/product_common/tasks/amazon.yml index 0175922..c02f864 100644 --- a/roles/product_common/tasks/amazon.yml +++ b/roles/product_common/tasks/amazon.yml @@ -4,11 +4,9 @@ yum: name: - java-{{ java_version }}-openjdk-devel - lock_timeout: 30 when: atl_use_system_jdk - name: Install other base packages on Amazon Linux yum: name: - dejavu-fonts-common # Required by the installer - lock_timeout: 30 From 87ac31ea3a101b50b4dddad610ef12f85f1a03d7 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Fri, 4 Oct 2019 13:55:37 +1000 Subject: [PATCH 25/27] DCD-686: Move back to pipenv-based Ansible installation to work around bugs (again). --- Pipfile | 4 +- Pipfile.lock | 106 ++++++++++++++++++++------------------- bin/ansible-with-atl-env | 11 ++-- bin/install-ansible | 18 ++----- 4 files changed, 69 insertions(+), 70 deletions(-) diff --git a/Pipfile b/Pipfile index c0d22ba..e78f929 100644 --- a/Pipfile +++ b/Pipfile @@ -4,7 +4,9 @@ verify_ssl = true name = "pypi" [packages] -ansible = "==2.7.11" +ansible = "==2.8.5" +boto3 = "==1.9.242" +botocore = "==1.12.242" [dev-packages] molecule = "==2.20.2" diff --git a/Pipfile.lock b/Pipfile.lock index d5ce753..725f9f6 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "3e6fecddc35743d370fbdbd68b57b6edc588fe026f58e67e6f6343c7dafc2ee6" + "sha256": "4cec168800858d0bce3beaae422011cb6458d548e9a8fc1807f39bd7c8eb24e6" }, "pipfile-spec": 6, "requires": { @@ -18,10 +18,10 @@ "default": { "ansible": { "hashes": [ - "sha256:e7e6de461b7d07cb4d8b2dd2a32b231af7c56e6bf39b851024671aaa52fd377e" + "sha256:8e9403e755ce8ef27b6066cdd7a4c567aa80ebe2fd90d0ff8efa0a725d246986" ], "index": "pypi", - "version": "==2.7.11" + "version": "==2.8.5" }, "asn1crypto": { "hashes": [ @@ -30,26 +30,21 @@ ], "version": "==1.0.0" }, - "bcrypt": { + "boto3": { "hashes": [ - "sha256:0258f143f3de96b7c14f762c770f5fc56ccd72f8a1857a451c1cd9a655d9ac89", - "sha256:0b0069c752ec14172c5f78208f1863d7ad6755a6fae6fe76ec2c80d13be41e42", - "sha256:19a4b72a6ae5bb467fea018b825f0a7d917789bcfe893e53f15c92805d187294", - "sha256:5432dd7b34107ae8ed6c10a71b4397f1c853bd39a4d6ffa7e35f40584cffd161", - "sha256:69361315039878c0680be456640f8705d76cb4a3a3fe1e057e0f261b74be4b31", - "sha256:6fe49a60b25b584e2f4ef175b29d3a83ba63b3a4df1b4c0605b826668d1b6be5", - "sha256:74a015102e877d0ccd02cdeaa18b32aa7273746914a6c5d0456dd442cb65b99c", - "sha256:763669a367869786bb4c8fcf731f4175775a5b43f070f50f46f0b59da45375d0", - "sha256:8b10acde4e1919d6015e1df86d4c217d3b5b01bb7744c36113ea43d529e1c3de", - "sha256:9fe92406c857409b70a38729dbdf6578caf9228de0aef5bc44f859ffe971a39e", - "sha256:a190f2a5dbbdbff4b74e3103cef44344bc30e61255beb27310e2aec407766052", - "sha256:a595c12c618119255c90deb4b046e1ca3bcfad64667c43d1166f2b04bc72db09", - "sha256:c9457fa5c121e94a58d6505cadca8bed1c64444b83b3204928a866ca2e599105", - "sha256:cb93f6b2ab0f6853550b74e051d297c27a638719753eb9ff66d1e4072be67133", - "sha256:d7bdc26475679dd073ba0ed2766445bb5b20ca4793ca0db32b399dccc6bc84b7", - "sha256:ff032765bb8716d9387fd5376d987a937254b0619eff0972779515b5c98820bc" + "sha256:4189e1ffed768bd0efd754a0abedebce19495ba2aa6b2f5e20f29ba80f81f9cb", + "sha256:fa4e28166922feeb9b7b56134c1acc817a1bca36284a0035bc08a3dab1853a9f" ], - "version": "==3.1.7" + "index": "pypi", + "version": "==1.9.242" + }, + "botocore": { + "hashes": [ + "sha256:7af52e0aabaf4ba045e1a5832308e70e1ea4b499b71624857f09aed2ba5e667c", + "sha256:dd62d63bcd3176c92775c52d3e879288f89bf0ac0039df14ea31f25d693acd6d" + ], + "index": "pypi", + "version": "==1.12.242" }, "cffi": { "hashes": [ @@ -105,6 +100,14 @@ ], "version": "==2.7" }, + "docutils": { + "hashes": [ + "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0", + "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827", + "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99" + ], + "version": "==0.15.2" + }, "jinja2": { "hashes": [ "sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013", @@ -112,6 +115,13 @@ ], "version": "==2.10.1" }, + "jmespath": { + "hashes": [ + "sha256:3720a4b1bd659dd2eecad0666459b9788813e032b83e7ba58578e48254e0a0e6", + "sha256:bde2aef6f44302dfb30320115b17d030798de8c4110e28d5cf6cf91a7a31074c" + ], + "version": "==0.9.4" + }, "markupsafe": { "hashes": [ "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", @@ -145,42 +155,19 @@ ], "version": "==1.1.1" }, - "paramiko": { - "hashes": [ - "sha256:99f0179bdc176281d21961a003ffdb2ec369daac1a1007241f53374e376576cf", - "sha256:f4b2edfa0d226b70bd4ca31ea7e389325990283da23465d572ed1f70a7583041" - ], - "version": "==2.6.0" - }, "pycparser": { "hashes": [ "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" ], "version": "==2.19" }, - "pynacl": { + "python-dateutil": { "hashes": [ - "sha256:05c26f93964373fc0abe332676cb6735f0ecad27711035b9472751faa8521255", - "sha256:0c6100edd16fefd1557da078c7a31e7b7d7a52ce39fdca2bec29d4f7b6e7600c", - "sha256:0d0a8171a68edf51add1e73d2159c4bc19fc0718e79dec51166e940856c2f28e", - "sha256:1c780712b206317a746ace34c209b8c29dbfd841dfbc02aa27f2084dd3db77ae", - "sha256:2424c8b9f41aa65bbdbd7a64e73a7450ebb4aa9ddedc6a081e7afcc4c97f7621", - "sha256:2d23c04e8d709444220557ae48ed01f3f1086439f12dbf11976e849a4926db56", - "sha256:30f36a9c70450c7878053fa1344aca0145fd47d845270b43a7ee9192a051bf39", - "sha256:37aa336a317209f1bb099ad177fef0da45be36a2aa664507c5d72015f956c310", - "sha256:4943decfc5b905748f0756fdd99d4f9498d7064815c4cf3643820c9028b711d1", - "sha256:57ef38a65056e7800859e5ba9e6091053cd06e1038983016effaffe0efcd594a", - "sha256:5bd61e9b44c543016ce1f6aef48606280e45f892a928ca7068fba30021e9b786", - "sha256:6482d3017a0c0327a49dddc8bd1074cc730d45db2ccb09c3bac1f8f32d1eb61b", - "sha256:7d3ce02c0784b7cbcc771a2da6ea51f87e8716004512493a2b69016326301c3b", - "sha256:a14e499c0f5955dcc3991f785f3f8e2130ed504fa3a7f44009ff458ad6bdd17f", - "sha256:a39f54ccbcd2757d1d63b0ec00a00980c0b382c62865b61a505163943624ab20", - "sha256:aabb0c5232910a20eec8563503c153a8e78bbf5459490c49ab31f6adf3f3a415", - "sha256:bd4ecb473a96ad0f90c20acba4f0bf0df91a4e03a1f4dd6a4bdc9ca75aa3a715", - "sha256:e2da3c13307eac601f3de04887624939aca8ee3c9488a0bb0eca4fb9401fc6b1", - "sha256:f67814c38162f4deb31f68d590771a29d5ae3b1bd64b75cf232308e5c74777e0" + "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", + "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" ], - "version": "==1.3.0" + "markers": "python_version >= '2.7'", + "version": "==2.8.0" }, "pyyaml": { "hashes": [ @@ -200,21 +187,36 @@ ], "version": "==5.1.2" }, + "s3transfer": { + "hashes": [ + "sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d", + "sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba" + ], + "version": "==0.2.1" + }, "six": { "hashes": [ "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" ], "version": "==1.12.0" + }, + "urllib3": { + "hashes": [ + "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398", + "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86" + ], + "markers": "python_version >= '3.4'", + "version": "==1.25.6" } }, "develop": { "ansible": { "hashes": [ - "sha256:e7e6de461b7d07cb4d8b2dd2a32b231af7c56e6bf39b851024671aaa52fd377e" + "sha256:8e9403e755ce8ef27b6066cdd7a4c567aa80ebe2fd90d0ff8efa0a725d246986" ], "index": "pypi", - "version": "==2.7.11" + "version": "==2.8.5" }, "ansible-lint": { "hashes": [ @@ -275,6 +277,7 @@ "sha256:4189e1ffed768bd0efd754a0abedebce19495ba2aa6b2f5e20f29ba80f81f9cb", "sha256:fa4e28166922feeb9b7b56134c1acc817a1bca36284a0035bc08a3dab1853a9f" ], + "index": "pypi", "version": "==1.9.242" }, "botocore": { @@ -282,6 +285,7 @@ "sha256:7af52e0aabaf4ba045e1a5832308e70e1ea4b499b71624857f09aed2ba5e667c", "sha256:dd62d63bcd3176c92775c52d3e879288f89bf0ac0039df14ea31f25d693acd6d" ], + "index": "pypi", "version": "==1.12.242" }, "cerberus": { diff --git a/bin/ansible-with-atl-env b/bin/ansible-with-atl-env index 580b4d4..29d5fee 100755 --- a/bin/ansible-with-atl-env +++ b/bin/ansible-with-atl-env @@ -14,8 +14,9 @@ source $ENV_FILE set +a # Use Ansible from virtualenv if provided -ansible-playbook -v \ - $ATL_DEPLOYMENT_REPOSITORY_CUSTOM_PARAMS \ - -i $INV \ - $PLAYBOOK \ - 2>&1 | tee --append $LOG_FILE +pipenv run \ + ansible-playbook -v \ + $ATL_DEPLOYMENT_REPOSITORY_CUSTOM_PARAMS \ + -i $INV \ + $PLAYBOOK \ + 2>&1 | tee --append $LOG_FILE diff --git a/bin/install-ansible b/bin/install-ansible index 953663f..4f3f81c 100755 --- a/bin/install-ansible +++ b/bin/install-ansible @@ -2,21 +2,13 @@ set -e -# Amazon Linux 2 packages Ansible separately, so enable the repo -. /etc/os-release -if [[ $ID == 'amzn' ]]; then - amazon-linux-extras enable ansible2 -fi - ./bin/pacapt install --noconfirm \ - ansible \ - python-boto3 \ - python-botocore + python3-dev \ + python3-pip + +pip3 install pipenv +pipenv sync if [[ $1 == "--dev" ]]; then - ./bin/pacapt install --noconfirm \ - python3-dev python3-pip - - pip3 install pipenv pipenv sync --dev fi From d67f579f2f72253c7fe1862c40e108b6e62868ca Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Fri, 4 Oct 2019 15:42:44 +1000 Subject: [PATCH 26/27] DCD-686: Add download of the backups. --- roles/load_backup_manifest/tasks/main.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/roles/load_backup_manifest/tasks/main.yml b/roles/load_backup_manifest/tasks/main.yml index 8a62e9d..45d7e1f 100644 --- a/roles/load_backup_manifest/tasks/main.yml +++ b/roles/load_backup_manifest/tasks/main.yml @@ -44,4 +44,26 @@ file: "{{ atl_backup_manifest_dest }}" name: atl_backup_manifest + - name: Define the DB and home dump destinations + set_fact: + atl_backup_db_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.db_dump | basename }}" + atl_backup_home_dest: "{{ atl_installer_temp }}/{{ atl_backup_manifest.shared_home_dump | basename }}" + + # FIXME: Here we fetch the backups. However we may wish to stream + # these directly from S3 to the target DB/FS to avoid requiring + # disk-space for the intermediate files. + - name: Fetch DB backup from S3 + aws_s3: + mode: get + bucket: "{{ atl_backup_manifest.db_dump | urlsplit('hostname') }}" + object: "{{ atl_backup_manifest.db_dump | urlsplit('path') }}" + dest: "{{ atl_backup_db_dest }}" + + - name: Fetch Home backup from S3 + aws_s3: + mode: get + bucket: "{{ atl_backup_manifest.shared_home_dump | urlsplit('hostname') }}" + object: "{{ atl_backup_manifest.shared_home_dump | urlsplit('path') }}" + dest: "{{ atl_backup_home_dest }}" + when: atl_backup_manifest_url is defined and atl_backup_manifest_url != '' From fd466f1230e9890be2d36d9612fb4464a4de4172 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 8 Oct 2019 10:19:39 +1100 Subject: [PATCH 27/27] DCD-686: Another role rename to match functionality. --- roles/{load_backup_manifest => fetch_backups}/tasks/main.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename roles/{load_backup_manifest => fetch_backups}/tasks/main.yml (100%) diff --git a/roles/load_backup_manifest/tasks/main.yml b/roles/fetch_backups/tasks/main.yml similarity index 100% rename from roles/load_backup_manifest/tasks/main.yml rename to roles/fetch_backups/tasks/main.yml