From cae394ebf1cd3bacd7417d0a073f1e9f58a8c35d Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Mon, 1 Mar 2021 13:52:01 +1100 Subject: [PATCH 1/6] DCD-1157: Fix incorrect conditional for using IAM request signing for authenticating with elasticsearch in bitbucket config --- roles/bitbucket_config/defaults/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/bitbucket_config/defaults/main.yaml b/roles/bitbucket_config/defaults/main.yaml index 9cdc3c7..90487ad 100644 --- a/roles/bitbucket_config/defaults/main.yaml +++ b/roles/bitbucket_config/defaults/main.yaml @@ -6,4 +6,4 @@ atl_bitbucket_properties: "{{ atl_bitbucket_properties_raw.split(' ') | reject(' atl_elasticsearch_username: "{{ lookup('env', 'ATL_ELASTICSEARCH_USERNAME') }}" atl_elasticsearch_password: "{{ lookup('env', 'ATL_ELASTICSEARCH_PASSWORD') }}" -elasticsearch_should_auth_with_iam: "{{ atl_elasticsearch_username is not defined and atl_aws_region is defined }}" \ No newline at end of file +elasticsearch_should_auth_with_iam: "{{ (atl_elasticsearch_username == '' or atl_elasticsearch_password == '') and atl_aws_region is defined }}" \ No newline at end of file From f42da4652bc949af96d06762f4749dce4a920549 Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Mon, 1 Mar 2021 13:58:10 +1100 Subject: [PATCH 2/6] DCD-1157: Add tests that bitbucket.properties has correct values for IAM elasticsearch authentication and basic authentication --- bitbucket-pipelines.yml | 10 ++++++- .../molecule/default/converge.yml | 3 +++ .../molecule/default/tests/test_default.py | 4 +++ .../molecule/iam_elasticsearch/Dockerfile.j2 | 14 ++++++++++ .../molecule/iam_elasticsearch/converge.yml | 23 ++++++++++++++++ .../molecule/iam_elasticsearch/molecule.yml | 27 +++++++++++++++++++ .../iam_elasticsearch/tests/test_default.py | 24 +++++++++++++++++ 7 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 roles/bitbucket_config/molecule/iam_elasticsearch/Dockerfile.j2 create mode 100644 roles/bitbucket_config/molecule/iam_elasticsearch/converge.yml create mode 100644 roles/bitbucket_config/molecule/iam_elasticsearch/molecule.yml create mode 100644 roles/bitbucket_config/molecule/iam_elasticsearch/tests/test_default.py diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 06a83de..cca342b 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -18,7 +18,7 @@ pipelines: - step: name: Pre Parallelization stage script: - - echo "Running tests in 37 batches" + - echo "Running tests in 38 batches" - step: name: Check if the template is up-to-date @@ -33,6 +33,14 @@ pipelines: fi - parallel: + - step: + name: bitbucket_config/iam_elasticsearch + services: + - docker + script: + - ./bin/install-ansible --dev + - cd roles/bitbucket_config + - pipenv run molecule test -s iam_elasticsearch - step: name: bitbucket_config/default services: diff --git a/roles/bitbucket_config/molecule/default/converge.yml b/roles/bitbucket_config/molecule/default/converge.yml index 2a1addf..64dcf3f 100644 --- a/roles/bitbucket_config/molecule/default/converge.yml +++ b/roles/bitbucket_config/molecule/default/converge.yml @@ -12,6 +12,9 @@ atl_jdbc_user: 'bb_db_user' atl_jdbc_password: 'molecule_password' + atl_elasticsearch_username: bitbucket + atl_elasticsearch_password: password + atl_bitbucket_properties_raw: "key1=val1 key2=val2 key3=val3" roles: diff --git a/roles/bitbucket_config/molecule/default/tests/test_default.py b/roles/bitbucket_config/molecule/default/tests/test_default.py index 770c3bb..024095f 100644 --- a/roles/bitbucket_config/molecule/default/tests/test_default.py +++ b/roles/bitbucket_config/molecule/default/tests/test_default.py @@ -15,6 +15,10 @@ def test_config_file(host): assert f.contains("jdbc.user=bb_db_user") assert f.contains("jdbc.password=molecule_password") + assert f.contains("plugin.search.elasticsearch.username=bitbucket") + assert f.contains("plugin.search.elasticsearch.password=password") + assert not f.contains("plugin.search.elasticsearch.aws.region") + assert f.contains("^key1=val1$") assert f.contains("^key2=val2$") assert f.contains("^key3=val3$") diff --git a/roles/bitbucket_config/molecule/iam_elasticsearch/Dockerfile.j2 b/roles/bitbucket_config/molecule/iam_elasticsearch/Dockerfile.j2 new file mode 100644 index 0000000..e6aa95d --- /dev/null +++ b/roles/bitbucket_config/molecule/iam_elasticsearch/Dockerfile.j2 @@ -0,0 +1,14 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi diff --git a/roles/bitbucket_config/molecule/iam_elasticsearch/converge.yml b/roles/bitbucket_config/molecule/iam_elasticsearch/converge.yml new file mode 100644 index 0000000..07accb2 --- /dev/null +++ b/roles/bitbucket_config/molecule/iam_elasticsearch/converge.yml @@ -0,0 +1,23 @@ +--- +- name: Converge + hosts: all + vars: + atl_product_family: "stash" + atl_product_edition: "bitbucket" + atl_product_user: "bitbucket" + atl_product_version: "6.3.1" + + atl_product_home: "{{ atl_shared_mountpoint }}/{{ atl_product_edition }}" + + atl_jdbc_user: 'bb_db_user' + atl_jdbc_password: 'molecule_password' + + atl_aws_region: us-east-2 + + atl_bitbucket_properties_raw: "key1=val1 key2=val2 key3=val3" + + roles: + - role: linux_common + - role: product_common + - role: product_install + - role: bitbucket_config diff --git a/roles/bitbucket_config/molecule/iam_elasticsearch/molecule.yml b/roles/bitbucket_config/molecule/iam_elasticsearch/molecule.yml new file mode 100644 index 0000000..400e984 --- /dev/null +++ b/roles/bitbucket_config/molecule/iam_elasticsearch/molecule.yml @@ -0,0 +1,27 @@ +--- +dependency: + name: galaxy +driver: + name: docker +platforms: + - name: amazon_linux2 + image: amazonlinux:2 + groups: + - aws_node_local + ulimits: + - nofile:262144:262144 + - name: ubuntu_lts + image: ubuntu:bionic + groups: + - aws_node_local + ulimits: + - nofile:262144:262144 +provisioner: + name: ansible + options: + skip-tags: runtime_pkg + inventory: + links: + group_vars: ../../../../group_vars/ +verifier: + name: testinfra diff --git a/roles/bitbucket_config/molecule/iam_elasticsearch/tests/test_default.py b/roles/bitbucket_config/molecule/iam_elasticsearch/tests/test_default.py new file mode 100644 index 0000000..56c334a --- /dev/null +++ b/roles/bitbucket_config/molecule/iam_elasticsearch/tests/test_default.py @@ -0,0 +1,24 @@ +import os + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +def test_config_file(host): + f = host.file('/media/atl/bitbucket/shared/bitbucket.properties') + assert f.exists + assert f.user == 'bitbucket' + + assert f.contains("jdbc.driver=org.postgresql.Driver") + assert f.contains("jdbc.user=bb_db_user") + assert f.contains("jdbc.password=molecule_password") + + assert not f.contains("plugin.search.elasticsearch.username") + assert not f.contains("plugin.search.elasticsearch.password") + assert not f.contains("plugin.search.elasticsearch.aws.region=us-east-2") + + assert f.contains("^key1=val1$") + assert f.contains("^key2=val2$") + assert f.contains("^key3=val3$") From c49c5a536e98edfa73e73b0dc71a4489f06719bc Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Mon, 1 Mar 2021 14:05:05 +1100 Subject: [PATCH 3/6] DCD-1157: Fix bitbucket pipelines batching --- bitbucket-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 6dcc467..5b98510 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -18,7 +18,7 @@ pipelines: - step: name: Pre Parallelization stage script: - - echo "Running tests in 38 batches" + - echo "Running tests in 39 batches" - step: name: Check if the template is up-to-date From cc401e8721c62b1782ddd63db256bc82bb87f94d Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Mon, 1 Mar 2021 14:32:50 +1100 Subject: [PATCH 4/6] DCD-1157: fix condition for using IAM authentication to elasticsearch in bitbucket.properties template --- roles/bitbucket_config/templates/bitbucket.properties.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/bitbucket_config/templates/bitbucket.properties.j2 b/roles/bitbucket_config/templates/bitbucket.properties.j2 index d978cf6..df7b88c 100644 --- a/roles/bitbucket_config/templates/bitbucket.properties.j2 +++ b/roles/bitbucket_config/templates/bitbucket.properties.j2 @@ -13,7 +13,7 @@ hazelcast.network.aws.tag.value={{ atl_aws_stack_name }} hazelcast.group.name={{ atl_aws_stack_name }} hazelcast.group.password={{ atl_aws_stack_name }} plugin.search.elasticsearch.baseurl={{ atl_elasticsearch_endpoint }} -{% if elasticsearch_should_auth_with_iam == 'true' %} +{% if elasticsearch_should_auth_with_iam %} plugin.search.elasticsearch.aws.region={{ atl_aws_region }} {% else %} plugin.search.elasticsearch.username={{ atl_elasticsearch_username }} From 69172295a1871eeb644cb525aaba1b7989044402 Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Mon, 1 Mar 2021 14:59:20 +1100 Subject: [PATCH 5/6] DCD-1157: Fix incorrect assertion in IAM athentication elasticsearch config for bitbucket_config role --- .../molecule/iam_elasticsearch/tests/test_default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/bitbucket_config/molecule/iam_elasticsearch/tests/test_default.py b/roles/bitbucket_config/molecule/iam_elasticsearch/tests/test_default.py index 56c334a..e0561c0 100644 --- a/roles/bitbucket_config/molecule/iam_elasticsearch/tests/test_default.py +++ b/roles/bitbucket_config/molecule/iam_elasticsearch/tests/test_default.py @@ -17,7 +17,7 @@ def test_config_file(host): assert not f.contains("plugin.search.elasticsearch.username") assert not f.contains("plugin.search.elasticsearch.password") - assert not f.contains("plugin.search.elasticsearch.aws.region=us-east-2") + assert f.contains("plugin.search.elasticsearch.aws.region=us-east-2") assert f.contains("^key1=val1$") assert f.contains("^key2=val2$") From 34d5a9ce10f4ec22f59975c5105eff5f2a4e2d73 Mon Sep 17 00:00:00 2001 From: Ben Partridge Date: Tue, 2 Mar 2021 10:41:41 +1100 Subject: [PATCH 6/6] DCD-1157: Remove unnecessary noise from IAM authentication elasticsearch molecule test --- .../molecule/iam_elasticsearch/converge.yml | 2 -- .../molecule/iam_elasticsearch/tests/test_default.py | 9 --------- 2 files changed, 11 deletions(-) diff --git a/roles/bitbucket_config/molecule/iam_elasticsearch/converge.yml b/roles/bitbucket_config/molecule/iam_elasticsearch/converge.yml index 07accb2..596fbfc 100644 --- a/roles/bitbucket_config/molecule/iam_elasticsearch/converge.yml +++ b/roles/bitbucket_config/molecule/iam_elasticsearch/converge.yml @@ -14,8 +14,6 @@ atl_aws_region: us-east-2 - atl_bitbucket_properties_raw: "key1=val1 key2=val2 key3=val3" - roles: - role: linux_common - role: product_common diff --git a/roles/bitbucket_config/molecule/iam_elasticsearch/tests/test_default.py b/roles/bitbucket_config/molecule/iam_elasticsearch/tests/test_default.py index e0561c0..1c541f5 100644 --- a/roles/bitbucket_config/molecule/iam_elasticsearch/tests/test_default.py +++ b/roles/bitbucket_config/molecule/iam_elasticsearch/tests/test_default.py @@ -9,16 +9,7 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( def test_config_file(host): f = host.file('/media/atl/bitbucket/shared/bitbucket.properties') assert f.exists - assert f.user == 'bitbucket' - - assert f.contains("jdbc.driver=org.postgresql.Driver") - assert f.contains("jdbc.user=bb_db_user") - assert f.contains("jdbc.password=molecule_password") assert not f.contains("plugin.search.elasticsearch.username") assert not f.contains("plugin.search.elasticsearch.password") assert f.contains("plugin.search.elasticsearch.aws.region=us-east-2") - - assert f.contains("^key1=val1$") - assert f.contains("^key2=val2$") - assert f.contains("^key3=val3$")